python-124m-muon-5b-20261021-v1: 3000012443 training targets

README.md

@@ -11,7 +11,7 @@ tags:
 # Simple LLM Python 124M / 5B
 
 Run: `python-124m-muon-5b-20261021-v1`. This is an **intermediate** base-model checkpoint, freshly initialized.
-Processed **2,750,029,614 nonpadding training targets** in **46,880 optimizer updates**.
+Processed **3,000,012,443 nonpadding training targets** in **51,214 optimizer updates**.
 The exact target is 5,000,000,000. This model is not instruction-tuned.
 
 124,389,632 parameters; 14 layers, width 768, 12 attention heads, RMSNorm,

metrics.jsonl

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ddcce92c81019a608eeba85cef0194d9c82fceb28ae2509cab63bbd68db5e4d0
-size 75744653
+oid sha256:2ea67a66d22701d8b216e12f4a9a85ca51cff528d10b6ebbe0efec89812a324e
+size 82752200

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:abcf556a9434e6fc77196a76841a237b65f3b974127b47b5130e6157fb09850a
+oid sha256:00b85d47c4be410669df45899bf95af410f958d77b0987ae00fc5f341cc2cc9b
 size 598238584

provenance.json

@@ -49,20 +49,20 @@
   "torch_version": "2.6.0+cu124",
   "gpu": "NVIDIA H100 80GB HBM3",
   "parameters": 124389632,
-  "resume_from": "/runs/python-124m-muon-5b-20261021-v1-s012/step-000042519.pt",
-  "step": 46880,
-  "tokens": 2750029614,
+  "resume_from": "/runs/python-124m-muon-5b-20261021-v1-s013/step-000046880.pt",
+  "step": 51214,
+  "tokens": 3000012443,
   "run_id": "python-124m-muon-5b-20261021-v1",
-  "checkpoint": "/runs/python-124m-muon-5b-20261021-v1-s013/step-000046880.pt",
-  "weights_sha256": "abcf556a9434e6fc77196a76841a237b65f3b974127b47b5130e6157fb09850a",
+  "checkpoint": "/runs/python-124m-muon-5b-20261021-v1-s014/step-000051214.pt",
+  "weights_sha256": "00b85d47c4be410669df45899bf95af410f958d77b0987ae00fc5f341cc2cc9b",
   "weight_roundtrip_exact": true,
   "cpu_logits_roundtrip_exact": true,
   "loss_token_exposures": {
-    "python_implementation": 1787518112,
-    "python_tests_examples": 412503978,
-    "python_documentation": 275002110,
-    "cs_math": 137501054,
-    "general_education": 82501544,
-    "books": 55002816
+    "python_implementation": 1949998511,
+    "python_tests_examples": 450001516,
+    "python_documentation": 299999767,
+    "cs_math": 150000858,
+    "general_education": 90006253,
+    "books": 60005538
   }
 }

runtime-cost-estimate.json

@@ -1,11 +1,11 @@
 {
   "kind": "GPU compute estimate, not measured invoice",
-  "training_worker_reported_seconds": 17408.777889444,
+  "training_worker_reported_seconds": 18846.089904152002,
   "h100_usd_per_second": 0.001097,
   "excludes": [
     "GPU work outside recorded training segments",
     "CPU preparation and evaluation",
     "storage and transfer"
   ],
-  "training_gpu_usd_estimate": 152.77943475776058
+  "training_gpu_usd_estimate": 165.393284998838
 }

validation.jsonl

@@ -119,3 +119,14 @@
 {"step": 46000, "tokens": 2699872835, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.6516034398742783, "python_tests_examples": 1.484993995577513, "python_documentation": 2.572842087115055, "cs_math": 2.638648783390031, "general_education": 3.704827610137638, "books": 3.8900709382420082}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.6516034398742783, "python_tests_examples": 1.484993995577513, "python_documentation": 2.572842087115055, "cs_math": 2.638648783390031, "general_education": 3.704827610137638, "books": 3.8900709382420082}, "weighted_loss": 1.8744542302048843}
 {"step": 46500, "tokens": 2728888163, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.6491192107410588, "python_tests_examples": 1.4691654664063583, "python_documentation": 2.5695742893452858, "cs_math": 2.664499644567642, "general_education": 3.692125639604816, "books": 3.8290243232799543}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.6491192107410588, "python_tests_examples": 1.4691654664063583, "python_documentation": 2.5695742893452858, "cs_math": 2.664499644567642, "general_education": 3.692125639604816, "books": 3.8290243232799543}, "weighted_loss": 1.8698289737592964}
 {"step": 46880, "tokens": 2750029614, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.6467544658541027, "python_tests_examples": 1.4647498506032444, "python_documentation": 2.572555247388413, "cs_math": 2.648651639761941, "general_education": 3.69639633080658, "books": 3.806166974120848}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.6467544658541027, "python_tests_examples": 1.4647498506032444, "python_documentation": 2.572555247388413, "cs_math": 2.648651639761941, "general_education": 3.69639633080658, "books": 3.806166974120848}, "weighted_loss": 1.866806216529206}
+{"step": 46880, "tokens": 2750029614, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.6467022709071193, "python_tests_examples": 1.4647627163544379, "python_documentation": 2.572469600432031, "cs_math": 2.6486947637389417, "general_education": 3.6963313257025985, "books": 3.8061652607397467}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.6467022709071193, "python_tests_examples": 1.4647627163544379, "python_documentation": 2.572469600432031, "cs_math": 2.6486947637389417, "general_education": 3.6963313257025985, "books": 3.8061652607397467}, "weighted_loss": 1.8667658267588163}
+{"step": 47000, "tokens": 2756985176, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.639379275588841, "python_tests_examples": 1.4702677663292656, "python_documentation": 2.5599512525721915, "cs_math": 2.624311742172163, "general_education": 3.690775189340678, "books": 3.8031458644127976}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.639379275588841, "python_tests_examples": 1.4702677663292656, "python_documentation": 2.5599512525721915, "cs_math": 2.624311742172163, "general_education": 3.690775189340678, "books": 3.8031458644127976}, "weighted_loss": 1.8601335794164402}
+{"step": 47500, "tokens": 2786528695, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.6614750608690003, "python_tests_examples": 1.4754512953566314, "python_documentation": 2.5693570941617665, "cs_math": 2.631362817249529, "general_education": 3.679419247921367, "books": 3.831070580061593}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.6614750608690003, "python_tests_examples": 1.4754512953566314, "python_documentation": 2.5693570941617665, "cs_math": 2.631362817249529, "general_education": 3.679419247921367, "books": 3.831070580061593}, "weighted_loss": 1.8767843231858707}
+{"step": 48000, "tokens": 2815195401, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.6261803692542767, "python_tests_examples": 1.4611837231351057, "python_documentation": 2.5484189725319917, "cs_math": 2.620219709564023, "general_education": 3.6827778476176793, "books": 3.8207707989844844}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.6261803692542767, "python_tests_examples": 1.4611837231351057, "python_documentation": 2.5484189725319917, "cs_math": 2.620219709564023, "general_education": 3.6827778476176793, "books": 3.8207707989844844}, "weighted_loss": 1.848946432625166}
+{"step": 48500, "tokens": 2844170669, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.6178605330274833, "python_tests_examples": 1.4587491350219386, "python_documentation": 2.5503576803927106, "cs_math": 2.623273289159314, "general_education": 3.6657157679920966, "books": 3.7972821260022593}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.6178605330274833, "python_tests_examples": 1.4587491350219386, "python_documentation": 2.5503576803927106, "cs_math": 2.623273289159314, "general_education": 3.6657157679920966, "books": 3.7972821260022593}, "weighted_loss": 1.8425382647781998}
+{"step": 49000, "tokens": 2872819106, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.6245528395879096, "python_tests_examples": 1.4730214331564067, "python_documentation": 2.5541811883665715, "cs_math": 2.6043313170773423, "general_education": 3.669172517870058, "books": 3.793951819540671}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.6245528395879096, "python_tests_examples": 1.4730214331564067, "python_documentation": 2.5541811883665715, "cs_math": 2.6043313170773423, "general_education": 3.669172517870058, "books": 3.793951819540671}, "weighted_loss": 1.8485014573230416}
+{"step": 49500, "tokens": 2902458878, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.6358722342434773, "python_tests_examples": 1.462259784872081, "python_documentation": 2.56843079454404, "cs_math": 2.620988020801461, "general_education": 3.677050421563169, "books": 3.8220571369800824}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.6358722342434773, "python_tests_examples": 1.462259784872081, "python_documentation": 2.56843079454404, "cs_math": 2.620988020801461, "general_education": 3.677050421563169, "books": 3.8220571369800824}, "weighted_loss": 1.857301055870046}
+{"step": 50000, "tokens": 2931488718, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.6067324080013066, "python_tests_examples": 1.4412295205342038, "python_documentation": 2.5374203226963474, "cs_math": 2.5789156712368233, "general_education": 3.6433104257039046, "books": 3.7850507019166475}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.6067324080013066, "python_tests_examples": 1.4412295205342038, "python_documentation": 2.5374203226963474, "cs_math": 2.5789156712368233, "general_education": 3.6433104257039046, "books": 3.7850507019166475}, "weighted_loss": 1.828248635921906}
+{"step": 50500, "tokens": 2959496384, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.608132598431615, "python_tests_examples": 1.4316472895963646, "python_documentation": 2.5325119666261378, "cs_math": 2.613458658322964, "general_education": 3.649876971160361, "books": 3.847978563637746}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.608132598431615, "python_tests_examples": 1.4316472895963646, "python_documentation": 2.5325119666261378, "cs_math": 2.613458658322964, "general_education": 3.649876971160361, "books": 3.847978563637746}, "weighted_loss": 1.8304132924063323}
+{"step": 51000, "tokens": 2987950618, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.6319590964781707, "python_tests_examples": 1.4570949555070054, "python_documentation": 2.5565334545764364, "cs_math": 2.5855854818249195, "general_education": 3.657231114387861, "books": 3.7673285006035977}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.6319590964781707, "python_tests_examples": 1.4570949555070054, "python_documentation": 2.5565334545764364, "cs_math": 2.5855854818249195, "general_education": 3.657231114387861, "books": 3.7673285006035977}, "weighted_loss": 1.8493337790294593}
+{"step": 51214, "tokens": 3000012443, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.6138033852584228, "python_tests_examples": 1.4585360756879255, "python_documentation": 2.5455657028268335, "cs_math": 2.604423032794273, "general_education": 3.651095990644877, "books": 3.77585911669828}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.6138033852584228, "python_tests_examples": 1.4585360756879255, "python_documentation": 2.5455657028268335, "cs_math": 2.604423032794273, "general_education": 3.651095990644877, "books": 3.77585911669828}, "weighted_loss": 1.8375803957468726}