periodic checkpoint @ step 9500

README.md

@@ -9,7 +9,7 @@ tags:
 
 # d
 
-In-training periodic checkpoint (step 9400). Architecture:
+In-training periodic checkpoint (step 9500). Architecture:
 Kimi K3 mini port (`KimiLinearForCausalLM`).
 
 **Not a finished model** -- intermediate weights uploaded during training for safekeeping/diffing.

k3_2b-latest.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:559c4b104a19c76a0a262d5a37f070edb0d00daf50cb28b1298c63c37b0dc48c
+size 10478273275

metrics.jsonl

@@ -1043,3 +1043,14 @@
 {"wall": 6731.29, "step": 9390, "loss": 2.178069536884626, "lm_loss": 2.178069536884626, "mtp_loss": 0.0, "lr": 0.025, "grad_norm": 0.3765924274921417, "tokens": 2769223680, "adamw_lr": 0.0004, "mem_alloc_gb": 19.520028591156006, "mem_reserv_gb": 84.498046875, "loss_smoothed": 2.177011124541362, "seq_len": 4096}
 {"wall": 6938.625, "eval_loss": 1.8792177885770798, "eval_ppl": 6.548380374908447, "step": 9400}
 {"wall": 6938.628, "step": 9400, "loss": 2.335595414042473, "lm_loss": 2.335595414042473, "mtp_loss": 0.0, "lr": 0.025, "grad_norm": 0.317325621843338, "tokens": 2772172800, "adamw_lr": 0.0004, "mem_alloc_gb": 19.520028591156006, "mem_reserv_gb": 84.498046875, "loss_smoothed": 2.242513418197632, "seq_len": 4096}
+{"wall": 7216.982, "step": 9410, "loss": 2.1877023528019586, "lm_loss": 2.1877023528019586, "mtp_loss": 0.0, "lr": 0.025, "grad_norm": 0.2942897379398346, "tokens": 2775121920, "adamw_lr": 0.0004, "mem_alloc_gb": 19.520028591156006, "mem_reserv_gb": 86.861328125, "loss_smoothed": 2.159415241082509, "seq_len": 4096}
+{"wall": 7421.941, "step": 9420, "loss": 2.172256112098694, "lm_loss": 2.172256112098694, "mtp_loss": 0.0, "lr": 0.025, "grad_norm": 0.33291739225387573, "tokens": 2778071040, "adamw_lr": 0.0004, "mem_alloc_gb": 19.520028591156006, "mem_reserv_gb": 86.861328125, "loss_smoothed": 2.2003966410954794, "seq_len": 4096}
+{"wall": 7627.04, "step": 9430, "loss": 2.3482660204172134, "lm_loss": 2.3482660204172134, "mtp_loss": 0.0, "lr": 0.025, "grad_norm": 0.4067702889442444, "tokens": 2781020160, "adamw_lr": 0.0004, "mem_alloc_gb": 19.520028591156006, "mem_reserv_gb": 86.861328125, "loss_smoothed": 2.178508657713731, "seq_len": 4096}
+{"wall": 7832.234, "step": 9440, "loss": 2.06701996922493, "lm_loss": 2.06701996922493, "mtp_loss": 0.0, "lr": 0.025, "grad_norm": 0.28016197681427, "tokens": 2783969280, "adamw_lr": 0.0004, "mem_alloc_gb": 19.520028591156006, "mem_reserv_gb": 86.861328125, "loss_smoothed": 2.2110639572143556, "seq_len": 4096}
+{"wall": 8037.191, "step": 9450, "loss": 2.184415469566981, "lm_loss": 2.184415469566981, "mtp_loss": 0.0, "lr": 0.025, "grad_norm": 0.3530008792877197, "tokens": 2786918400, "adamw_lr": 0.0004, "mem_alloc_gb": 19.520028591156006, "mem_reserv_gb": 86.861328125, "loss_smoothed": 2.2258059963583947, "seq_len": 4096}
+{"wall": 8242.009, "step": 9460, "loss": 2.1902506947517395, "lm_loss": 2.1902506947517395, "mtp_loss": 0.0, "lr": 0.025, "grad_norm": 0.2605612874031067, "tokens": 2789867520, "adamw_lr": 0.0004, "mem_alloc_gb": 19.520028591156006, "mem_reserv_gb": 86.861328125, "loss_smoothed": 2.1953834712505342, "seq_len": 4096}
+{"wall": 8446.872, "step": 9470, "loss": 2.1020779808362327, "lm_loss": 2.1020779808362327, "mtp_loss": 0.0, "lr": 0.025, "grad_norm": 0.3889986276626587, "tokens": 2792816640, "adamw_lr": 0.0004, "mem_alloc_gb": 19.520028591156006, "mem_reserv_gb": 86.861328125, "loss_smoothed": 2.192288875579834, "seq_len": 4096}
+{"wall": 8651.85, "step": 9480, "loss": 2.216184342900912, "lm_loss": 2.216184342900912, "mtp_loss": 0.0, "lr": 0.025, "grad_norm": 0.42791157960891724, "tokens": 2795765760, "adamw_lr": 0.0004, "mem_alloc_gb": 19.520028591156006, "mem_reserv_gb": 86.861328125, "loss_smoothed": 2.2066777693728605, "seq_len": 4096}
+{"wall": 8857.371, "step": 9490, "loss": 2.1327615032593408, "lm_loss": 2.1327615032593408, "mtp_loss": 0.0, "lr": 0.025, "grad_norm": 0.35273656249046326, "tokens": 2798714880, "adamw_lr": 0.0004, "mem_alloc_gb": 19.520028591156006, "mem_reserv_gb": 86.861328125, "loss_smoothed": 2.235764626165231, "seq_len": 4096}
+{"wall": 9064.738, "eval_loss": 1.8863441795110703, "eval_ppl": 6.595213890075684, "step": 9500}
+{"wall": 9064.74, "step": 9500, "loss": 2.1489363312721252, "lm_loss": 2.1489363312721252, "mtp_loss": 0.0, "lr": 0.025, "grad_norm": 0.29979124665260315, "tokens": 2801664000, "adamw_lr": 0.0004, "mem_alloc_gb": 19.520028591156006, "mem_reserv_gb": 86.861328125, "loss_smoothed": 2.2166685223579408, "seq_len": 4096}