{
  "checkpoint": "checkpoint_tokens_000500020611.pt",
  "trainer_state": {
    "completed_steps": 20486,
    "train_tokens_seen": 500020611,
    "last_loss": 0.759232759475708
  },
  "checkpoint_args": {
    "data": "oeis_train_bfile_prefix4096.packed",
    "model_backend": "custom",
    "param_dtype": "fp32",
    "weight_update_mode": "bf16_live_fp32_master",
    "batch_size": 1,
    "seq_len": 4096,
    "steps": 5,
    "warmup_steps": 2,
    "target_tokens": 754629901,
    "max_steps": 0,
    "batch_mode": "bucketed",
    "pad_to_seq_len": false,
    "index_dir": "oeis_train_bfile_prefix4096.packed.idx",
    "bucket_tokens_per_batch": 16384,
    "bucket_token_budget_spec": "512:32768,*:24576",
    "bucket_max_batch_size": 512,
    "bucket_pad_multiple": 8,
    "bucket_pad_to_upper": true,
    "bucket_replacement": false,
    "bucket_repeat_epochs": true,
    "bucket_sampling": "token_mass",
    "vocab_mode": "oeis",
    "hidden_size": 1024,
    "intermediate_size": 3072,
    "num_hidden_layers": 28,
    "num_attention_heads": 16,
    "num_key_value_heads": 8,
    "head_dim": 0,
    "compile": true,
    "compile_mode": "reduce-overhead",
    "compile_dynamic": true,
    "compile_skip_dynamic_cudagraphs": true,
    "prewarm_bucket_shapes": true,
    "prewarm_bucket_passes": 2,
    "prewarm_restore_state": true,
    "prewarm_verify_restore": false,
    "prewarm_update_optimizer": false,
    "prewarm_materialize_optimizer_state": true,
    "gradient_checkpointing": false,
    "native_gqa": true,
    "amp": true,
    "lr": 0.0003,
    "optimizer_mode": "torch_muon_hybrid",
    "adamw_lr": 0.0001,
    "muon_lr": 0.01,
    "body_lr_mult": 1.0,
    "adamw_weight_decay": 0.01,
    "muon_weight_decay": 0.0,
    "muon_momentum": 0.95,
    "muon_ns_steps": 5,
    "muon_adjust_lr_fn": "",
    "no_muon_nesterov": false,
    "lr_schedule": "warmup_cosine_cooldown",
    "lr_total_tokens": 754629901,
    "lr_warmup_tokens": 0,
    "lr_warmup_fraction": 0.005,
    "lr_decay_end_fraction": 0.95,
    "lr_peak_factor": 1.0,
    "lr_min_factor": 0.1,
    "lr_final_factor": 0.0,
    "checkpoint_dir": "checkpoints",
    "checkpoint_every_tokens": 250000000,
    "checkpoint_every_steps": 0,
    "keep_last_checkpoints": 4,
    "resume": "",
    "init_from_hf_dir": "base_model",
    "save_final": true,
    "trim_final_batch": true,
    "allow_token_overshoot": false,
    "val_data": "oeis_val_decontam.jsonl",
    "val_format": "jsonl",
    "val_index_dir": "",
    "val_every_tokens": 250000000,
    "val_every_steps": 0,
    "val_batches": 128,
    "val_batch_size": 32,
    "val_max_examples": 0,
    "val_max_context_tokens": 4096,
    "oeis_eval_data": "",
    "oeis_eval_every_tokens": 0,
    "oeis_eval_every_steps": 0,
    "oeis_eval_batch_size": 64,
    "oeis_eval_max_examples": 0,
    "oeis_eval_max_new_tokens": 20,
    "oeis_eval_max_context_tokens": 0,
    "oeis_eval_collect_examples": 3,
    "oeis_eval_generation_backend": "legacy",
    "expected_loss_tokens": 754629901,
    "safe_preflight": true,
    "allow_synthetic_only": false,
    "allow_replacement_sampling": false,
    "preflight_only": false,
    "wandb": true,
    "wandb_project": "oeis-massive",
    "wandb_entity": "n8programs",
    "wandb_run_name": "nextterm-440m-bfile-only-1epoch-20260701-gqa-lrpeak100",
    "wandb_id": "nextterm440m-bfile-only-1epoch-20260701-gqa-lrpeak100",
    "wandb_resume": "allow",
    "wandb_mode": "online",
    "wandb_anonymous": "",
    "wandb_tags": "bfile-only,continued-pretraining,440m,no-synth,invlen-repair,lrpeak100",
    "log_every_steps": 10,
    "seed": 0,
    "report_json": "report.json"
  }
}