| { |
| "step": 5568, |
| "val_bpb": 0.719042, |
| "model_config": { |
| "sequence_len": 2048, |
| "vocab_size": 32768, |
| "n_layer": 24, |
| "n_head": 12, |
| "n_kv_head": 12, |
| "n_embd": 1536, |
| "window_pattern": "SSSL" |
| }, |
| "device_batch_size": 32, |
| "max_seq_len": 2048, |
| "total_batch_size": 1048576, |
| "note": "DecoderStack-medium (d24) capture converted for nanochat. This meta was built after the fact from the run log (full_d24_w8.log); DecoderStack's capture writes only {step, code, weights} and does not emit a meta.json of its own. model_config is the nanochat GPTConfig equivalent of StackConfig -- the two architectures are the same model (both count 1,384,122,122 parameters). Sufficient for load/eval and for SFT (chat_sft builds a fresh optimizer and only optionally warm-starts it). NOT sufficient for pre-training resume: there is no dataloader_state_dict, loop_state, or optimizer state here, and DecoderStack's pre-tokenized binary loader has no state nanochat's tokenizing loader could consume -- the data order could not be continued regardless. device_batch_size/max_seq_len are the nanochat-shaped restatement of micro_batch_tokens=65,536 (= 32 x 2048) -- DecoderStack trains varlen, so its micro-batch is a token count, not a sequence count. TOKENIZER: these weights use the 32k vocab shipped in the ChrisMcCormick/climbmix_32k_8_170 dataset repo (mirrored under tokenizer/ in this repo). It is NOT the same vocabulary as ChrisMcCormick/nanochat-varlen-d24-2026-03-22 -- 31,474 of 32,759 ids differ. Pairing these weights with that tokenizer produces garbage. USER_CONFIG added 2026-08-01 (make_user_config.py): chat_sft reads the pretraining LRs from meta['user_config'], so without this block it would have silently inherited its own fallback unembedding_lr=0.004 instead of this run's 0.008. Values are copied from the `schedules` / `stack_config` blocks already in this file; base_train.py argparse fields this capture gives no evidence for are omitted rather than guessed. weight_decay=0.28 is the pre-scale argument that yields the recorded muon_weight_decay_peak of 0.059738.", |
| "stack_config": { |
| "n_layers": 24, |
| "d_model": 1536, |
| "d_vocab": 32768, |
| "d_smr_gate": 24, |
| "n_q_heads": 12, |
| "n_kv_heads": 12, |
| "n_o_heads": 12, |
| "d_qk": 128, |
| "d_vo": 128, |
| "seq_len": 2048, |
| "short_win_size": 768, |
| "full_ctxt_layers": [ |
| 3, |
| 7, |
| 11, |
| 15, |
| 19, |
| 23 |
| ], |
| "d_ve_gate": 12, |
| "ve_layers": [ |
| 1, |
| 3, |
| 5, |
| 7, |
| 9, |
| 11, |
| 13, |
| 15, |
| 17, |
| 19, |
| 21, |
| 23 |
| ], |
| "d_mlp": 6144, |
| "micro_batch_tokens": 65536, |
| "total_batch_size": 1048576, |
| "num_iterations": 5568, |
| "max_num_docs": 192, |
| "num_train_shards": 80, |
| "val_tokens": 10485760, |
| "val_loss_every": 250, |
| "save_steps": [ |
| 1950, |
| 5568 |
| ], |
| "world_size": 8, |
| "grad_accum_steps": 2, |
| "precision": "bf16", |
| "run_id": "2026-08-01_023219-d24" |
| }, |
| "schedules": { |
| "batch_lr_scale": 1.4142135623730951, |
| "muon_weight_decay_peak": 0.059738, |
| "warmup_steps": 40, |
| "warmdown_ratio": 0.65, |
| "final_lr_frac": 0.05, |
| "matrix_lr": 0.02, |
| "muon_momentum": "0.85->0.97 over 400 steps, cooled to 0.90", |
| "muon_beta2": 0.9, |
| "muon_ns_steps": 5, |
| "unembedding_lr": 0.008, |
| "embedding_lr": 0.3, |
| "value_embedding_lr_mult": 0.5, |
| "scalar_lr": 0.5, |
| "smear_lr": 0.2, |
| "adamw_eps": 1e-10 |
| }, |
| "results": { |
| "core_metric": 0.2517, |
| "min_val_bpb": 0.719042, |
| "peak_memory_mib": 71569, |
| "total_training_time_min": 110.74, |
| "gpu": "8x NVIDIA H100 80GB HBM3" |
| }, |
| "user_config": { |
| "depth": 24, |
| "aspect_ratio": 64, |
| "head_dim": 128, |
| "max_seq_len": 2048, |
| "window_pattern": "SSSL", |
| "num_iterations": 5568, |
| "device_batch_size": 32, |
| "total_batch_size": 1048576, |
| "embedding_lr": 0.3, |
| "unembedding_lr": 0.008, |
| "matrix_lr": 0.02, |
| "scalar_lr": 0.5, |
| "weight_decay": 0.28, |
| "warmup_steps": 40, |
| "warmdown_ratio": 0.65, |
| "final_lr_frac": 0.05, |
| "model_tag": "d24_decoderstack", |
| "run": "2026-08-01_023219-d24" |
| } |
| } |
|
|
