| { |
| "checkpoint": "checkpoint_tokens_000500020611.pt", |
| "trainer_state": { |
| "completed_steps": 20486, |
| "train_tokens_seen": 500020611, |
| "last_loss": 0.759232759475708 |
| }, |
| "checkpoint_args": { |
| "data": "oeis_train_bfile_prefix4096.packed", |
| "model_backend": "custom", |
| "param_dtype": "fp32", |
| "weight_update_mode": "bf16_live_fp32_master", |
| "batch_size": 1, |
| "seq_len": 4096, |
| "steps": 5, |
| "warmup_steps": 2, |
| "target_tokens": 754629901, |
| "max_steps": 0, |
| "batch_mode": "bucketed", |
| "pad_to_seq_len": false, |
| "index_dir": "oeis_train_bfile_prefix4096.packed.idx", |
| "bucket_tokens_per_batch": 16384, |
| "bucket_token_budget_spec": "512:32768,*:24576", |
| "bucket_max_batch_size": 512, |
| "bucket_pad_multiple": 8, |
| "bucket_pad_to_upper": true, |
| "bucket_replacement": false, |
| "bucket_repeat_epochs": true, |
| "bucket_sampling": "token_mass", |
| "vocab_mode": "oeis", |
| "hidden_size": 1024, |
| "intermediate_size": 3072, |
| "num_hidden_layers": 28, |
| "num_attention_heads": 16, |
| "num_key_value_heads": 8, |
| "head_dim": 0, |
| "compile": true, |
| "compile_mode": "reduce-overhead", |
| "compile_dynamic": true, |
| "compile_skip_dynamic_cudagraphs": true, |
| "prewarm_bucket_shapes": true, |
| "prewarm_bucket_passes": 2, |
| "prewarm_restore_state": true, |
| "prewarm_verify_restore": false, |
| "prewarm_update_optimizer": false, |
| "prewarm_materialize_optimizer_state": true, |
| "gradient_checkpointing": false, |
| "native_gqa": true, |
| "amp": true, |
| "lr": 0.0003, |
| "optimizer_mode": "torch_muon_hybrid", |
| "adamw_lr": 0.0001, |
| "muon_lr": 0.01, |
| "body_lr_mult": 1.0, |
| "adamw_weight_decay": 0.01, |
| "muon_weight_decay": 0.0, |
| "muon_momentum": 0.95, |
| "muon_ns_steps": 5, |
| "muon_adjust_lr_fn": "", |
| "no_muon_nesterov": false, |
| "lr_schedule": "warmup_cosine_cooldown", |
| "lr_total_tokens": 754629901, |
| "lr_warmup_tokens": 0, |
| "lr_warmup_fraction": 0.005, |
| "lr_decay_end_fraction": 0.95, |
| "lr_peak_factor": 1.0, |
| "lr_min_factor": 0.1, |
| "lr_final_factor": 0.0, |
| "checkpoint_dir": "checkpoints", |
| "checkpoint_every_tokens": 250000000, |
| "checkpoint_every_steps": 0, |
| "keep_last_checkpoints": 4, |
| "resume": "", |
| "init_from_hf_dir": "base_model", |
| "save_final": true, |
| "trim_final_batch": true, |
| "allow_token_overshoot": false, |
| "val_data": "oeis_val_decontam.jsonl", |
| "val_format": "jsonl", |
| "val_index_dir": "", |
| "val_every_tokens": 250000000, |
| "val_every_steps": 0, |
| "val_batches": 128, |
| "val_batch_size": 32, |
| "val_max_examples": 0, |
| "val_max_context_tokens": 4096, |
| "oeis_eval_data": "", |
| "oeis_eval_every_tokens": 0, |
| "oeis_eval_every_steps": 0, |
| "oeis_eval_batch_size": 64, |
| "oeis_eval_max_examples": 0, |
| "oeis_eval_max_new_tokens": 20, |
| "oeis_eval_max_context_tokens": 0, |
| "oeis_eval_collect_examples": 3, |
| "oeis_eval_generation_backend": "legacy", |
| "expected_loss_tokens": 754629901, |
| "safe_preflight": true, |
| "allow_synthetic_only": false, |
| "allow_replacement_sampling": false, |
| "preflight_only": false, |
| "wandb": true, |
| "wandb_project": "oeis-massive", |
| "wandb_entity": "n8programs", |
| "wandb_run_name": "nextterm-440m-bfile-only-1epoch-20260701-gqa-lrpeak100", |
| "wandb_id": "nextterm440m-bfile-only-1epoch-20260701-gqa-lrpeak100", |
| "wandb_resume": "allow", |
| "wandb_mode": "online", |
| "wandb_anonymous": "", |
| "wandb_tags": "bfile-only,continued-pretraining,440m,no-synth,invlen-repair,lrpeak100", |
| "log_every_steps": 10, |
| "seed": 0, |
| "report_json": "report.json" |
| } |
| } |
|
|
