File size: 4,166 Bytes
11f07f9 3768187 11f07f9 3768187 11f07f9 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 | {
"step": 5568,
"val_bpb": 0.719042,
"model_config": {
"sequence_len": 2048,
"vocab_size": 32768,
"n_layer": 24,
"n_head": 12,
"n_kv_head": 12,
"n_embd": 1536,
"window_pattern": "SSSL"
},
"device_batch_size": 32,
"max_seq_len": 2048,
"total_batch_size": 1048576,
"note": "DecoderStack-medium (d24) capture converted for nanochat. This meta was built after the fact from the run log (full_d24_w8.log); DecoderStack's capture writes only {step, code, weights} and does not emit a meta.json of its own. model_config is the nanochat GPTConfig equivalent of StackConfig -- the two architectures are the same model (both count 1,384,122,122 parameters). Sufficient for load/eval and for SFT (chat_sft builds a fresh optimizer and only optionally warm-starts it). NOT sufficient for pre-training resume: there is no dataloader_state_dict, loop_state, or optimizer state here, and DecoderStack's pre-tokenized binary loader has no state nanochat's tokenizing loader could consume -- the data order could not be continued regardless. device_batch_size/max_seq_len are the nanochat-shaped restatement of micro_batch_tokens=65,536 (= 32 x 2048) -- DecoderStack trains varlen, so its micro-batch is a token count, not a sequence count. TOKENIZER: these weights use the 32k vocab shipped in the ChrisMcCormick/climbmix_32k_8_170 dataset repo (mirrored under tokenizer/ in this repo). It is NOT the same vocabulary as ChrisMcCormick/nanochat-varlen-d24-2026-03-22 -- 31,474 of 32,759 ids differ. Pairing these weights with that tokenizer produces garbage. USER_CONFIG added 2026-08-01 (make_user_config.py): chat_sft reads the pretraining LRs from meta['user_config'], so without this block it would have silently inherited its own fallback unembedding_lr=0.004 instead of this run's 0.008. Values are copied from the `schedules` / `stack_config` blocks already in this file; base_train.py argparse fields this capture gives no evidence for are omitted rather than guessed. weight_decay=0.28 is the pre-scale argument that yields the recorded muon_weight_decay_peak of 0.059738.",
"stack_config": {
"n_layers": 24,
"d_model": 1536,
"d_vocab": 32768,
"d_smr_gate": 24,
"n_q_heads": 12,
"n_kv_heads": 12,
"n_o_heads": 12,
"d_qk": 128,
"d_vo": 128,
"seq_len": 2048,
"short_win_size": 768,
"full_ctxt_layers": [
3,
7,
11,
15,
19,
23
],
"d_ve_gate": 12,
"ve_layers": [
1,
3,
5,
7,
9,
11,
13,
15,
17,
19,
21,
23
],
"d_mlp": 6144,
"micro_batch_tokens": 65536,
"total_batch_size": 1048576,
"num_iterations": 5568,
"max_num_docs": 192,
"num_train_shards": 80,
"val_tokens": 10485760,
"val_loss_every": 250,
"save_steps": [
1950,
5568
],
"world_size": 8,
"grad_accum_steps": 2,
"precision": "bf16",
"run_id": "2026-08-01_023219-d24"
},
"schedules": {
"batch_lr_scale": 1.4142135623730951,
"muon_weight_decay_peak": 0.059738,
"warmup_steps": 40,
"warmdown_ratio": 0.65,
"final_lr_frac": 0.05,
"matrix_lr": 0.02,
"muon_momentum": "0.85->0.97 over 400 steps, cooled to 0.90",
"muon_beta2": 0.9,
"muon_ns_steps": 5,
"unembedding_lr": 0.008,
"embedding_lr": 0.3,
"value_embedding_lr_mult": 0.5,
"scalar_lr": 0.5,
"smear_lr": 0.2,
"adamw_eps": 1e-10
},
"results": {
"core_metric": 0.2517,
"min_val_bpb": 0.719042,
"peak_memory_mib": 71569,
"total_training_time_min": 110.74,
"gpu": "8x NVIDIA H100 80GB HBM3"
},
"user_config": {
"depth": 24,
"aspect_ratio": 64,
"head_dim": 128,
"max_seq_len": 2048,
"window_pattern": "SSSL",
"num_iterations": 5568,
"device_batch_size": 32,
"total_batch_size": 1048576,
"embedding_lr": 0.3,
"unembedding_lr": 0.008,
"matrix_lr": 0.02,
"scalar_lr": 0.5,
"weight_decay": 0.28,
"warmup_steps": 40,
"warmdown_ratio": 0.65,
"final_lr_frac": 0.05,
"model_tag": "d24_decoderstack",
"run": "2026-08-01_023219-d24"
}
}
|