periodic checkpoint @ step 10300

README.md

@@ -9,7 +9,7 @@ tags:
 
 # d
 
-In-training periodic checkpoint (step 10200). Architecture:
+In-training periodic checkpoint (step 10300). Architecture:
 Kimi K3 mini port (`KimiLinearForCausalLM`).
 
 **Not a finished model** -- intermediate weights uploaded during training for safekeeping/diffing.

k3_2b-latest.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9c1ff2ea2ed9a8c9f694ea1d58822650a89d64640b0cd2da09ee87fc1c0f8991
+size 10478273275

metrics.jsonl

@@ -1131,3 +1131,14 @@
 {"wall": 23750.784, "step": 10190, "loss": 2.114508872230848, "lm_loss": 2.114508872230848, "mtp_loss": 0.0, "lr": 0.025, "grad_norm": 0.3238677978515625, "tokens": 3005153280, "adamw_lr": 0.0004, "mem_alloc_gb": 19.520028591156006, "mem_reserv_gb": 84.380859375, "loss_smoothed": 2.1369449625412624, "seq_len": 4096}
 {"wall": 23957.907, "eval_loss": 1.883776217699051, "eval_ppl": 6.578299045562744, "step": 10200}
 {"wall": 23957.909, "step": 10200, "loss": 2.281770423054695, "lm_loss": 2.281770423054695, "mtp_loss": 0.0, "lr": 0.025, "grad_norm": 0.36528244614601135, "tokens": 3008102400, "adamw_lr": 0.0004, "mem_alloc_gb": 19.520028591156006, "mem_reserv_gb": 84.380859375, "loss_smoothed": 2.230571264276902, "seq_len": 4096}
+{"wall": 24244.027, "step": 10210, "loss": 2.2656367421150208, "lm_loss": 2.2656367421150208, "mtp_loss": 0.0, "lr": 0.025, "grad_norm": 0.3323168456554413, "tokens": 3011051520, "adamw_lr": 0.0004, "mem_alloc_gb": 19.520028591156006, "mem_reserv_gb": 84.263671875, "loss_smoothed": 2.2053902325530848, "seq_len": 4096}
+{"wall": 24449.115, "step": 10220, "loss": 2.153836896022161, "lm_loss": 2.153836896022161, "mtp_loss": 0.0, "lr": 0.025, "grad_norm": 0.29876238107681274, "tokens": 3014000640, "adamw_lr": 0.0004, "mem_alloc_gb": 19.520028591156006, "mem_reserv_gb": 84.263671875, "loss_smoothed": 2.1383815951645375, "seq_len": 4096}
+{"wall": 24653.437, "step": 10230, "loss": 2.2027230709791183, "lm_loss": 2.2027230709791183, "mtp_loss": 0.0, "lr": 0.025, "grad_norm": 0.4377819299697876, "tokens": 3016949760, "adamw_lr": 0.0004, "mem_alloc_gb": 19.520028591156006, "mem_reserv_gb": 84.263671875, "loss_smoothed": 2.1574684105813504, "seq_len": 4096}
+{"wall": 24858.265, "step": 10240, "loss": 2.070614521702131, "lm_loss": 2.070614521702131, "mtp_loss": 0.0, "lr": 0.025, "grad_norm": 0.31369268894195557, "tokens": 3019898880, "adamw_lr": 0.0004, "mem_alloc_gb": 19.520028591156006, "mem_reserv_gb": 84.263671875, "loss_smoothed": 2.180688039958477, "seq_len": 4096}
+{"wall": 25063.35, "step": 10250, "loss": 2.203572099407514, "lm_loss": 2.203572099407514, "mtp_loss": 0.0, "lr": 0.025, "grad_norm": 0.3951571583747864, "tokens": 3022848000, "adamw_lr": 0.0004, "mem_alloc_gb": 19.520028591156006, "mem_reserv_gb": 84.263671875, "loss_smoothed": 2.163116140663624, "seq_len": 4096}
+{"wall": 25267.688, "step": 10260, "loss": 2.2659132977326712, "lm_loss": 2.2659132977326712, "mtp_loss": 0.0, "lr": 0.025, "grad_norm": 0.3866041898727417, "tokens": 3025797120, "adamw_lr": 0.0004, "mem_alloc_gb": 19.520028591156006, "mem_reserv_gb": 84.263671875, "loss_smoothed": 2.159140620628993, "seq_len": 4096}
+{"wall": 25472.764, "step": 10270, "loss": 2.288241297006607, "lm_loss": 2.288241297006607, "mtp_loss": 0.0, "lr": 0.025, "grad_norm": 0.37408870458602905, "tokens": 3028746240, "adamw_lr": 0.0004, "mem_alloc_gb": 19.520028591156006, "mem_reserv_gb": 84.263671875, "loss_smoothed": 2.1731113865971565, "seq_len": 4096}
+{"wall": 25677.176, "step": 10280, "loss": 2.0418743987878165, "lm_loss": 2.0418743987878165, "mtp_loss": 0.0, "lr": 0.025, "grad_norm": 0.38568007946014404, "tokens": 3031695360, "adamw_lr": 0.0004, "mem_alloc_gb": 19.520028591156006, "mem_reserv_gb": 84.263671875, "loss_smoothed": 2.165077544997136, "seq_len": 4096}
+{"wall": 25882.344, "step": 10290, "loss": 2.158773437142372, "lm_loss": 2.158773437142372, "mtp_loss": 0.0, "lr": 0.025, "grad_norm": 0.34138041734695435, "tokens": 3034644480, "adamw_lr": 0.0004, "mem_alloc_gb": 19.520028591156006, "mem_reserv_gb": 84.263671875, "loss_smoothed": 2.2402724474668503, "seq_len": 4096}
+{"wall": 26089.518, "eval_loss": 1.8766836673021317, "eval_ppl": 6.5318074226379395, "step": 10300}
+{"wall": 26089.519, "step": 10300, "loss": 2.0133808155854545, "lm_loss": 2.0133808155854545, "mtp_loss": 0.0, "lr": 0.025, "grad_norm": 0.3389778435230255, "tokens": 3037593600, "adamw_lr": 0.0004, "mem_alloc_gb": 19.520028591156006, "mem_reserv_gb": 84.263671875, "loss_smoothed": 2.1546207197010516, "seq_len": 4096}