python-124m-muon-5b-20261021-v1: 1500065139 training targets

README.md

@@ -11,7 +11,7 @@ tags:
 # Simple LLM Python 124M / 5B
 
 Run: `python-124m-muon-5b-20261021-v1`. This is an **intermediate** base-model checkpoint, freshly initialized.
-Processed **1,250,043,430 nonpadding training targets** in **20,819 optimizer updates**.
+Processed **1,500,065,139 nonpadding training targets** in **25,134 optimizer updates**.
 The exact target is 5,000,000,000. This model is not instruction-tuned.
 
 124,389,632 parameters; 14 layers, width 768, 12 attention heads, RMSNorm,

metrics.jsonl

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:fe3adad5156fb7fd04a929740f261a6c2b6c0e5d5010615326cee2934b83880f
-size 33610823
+oid sha256:c31c2362c2423768a517e0f58344f6ba00cfe74ac3ceca8ef6dc84117a4255a2
+size 40585201

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:084abd8bfcc152474b883737a749be51945bdd1e9d0a43f1d0a75d02b57ec8a2
+oid sha256:99ce99f9dd43f0ddfda6d9e99110bcefb819d72cf8378a7948e3d06503f6f26e
 size 598238584

provenance.json

@@ -49,20 +49,20 @@
   "torch_version": "2.6.0+cu124",
   "gpu": "NVIDIA H100 80GB HBM3",
   "parameters": 124389632,
-  "resume_from": "/runs/python-124m-muon-5b-20261021-v1-s006/step-000016446.pt",
-  "step": 20819,
-  "tokens": 1250043430,
+  "resume_from": "/runs/python-124m-muon-5b-20261021-v1-s007/step-000020819.pt",
+  "step": 25134,
+  "tokens": 1500065139,
   "run_id": "python-124m-muon-5b-20261021-v1",
-  "checkpoint": "/runs/python-124m-muon-5b-20261021-v1-s007/step-000020819.pt",
-  "weights_sha256": "084abd8bfcc152474b883737a749be51945bdd1e9d0a43f1d0a75d02b57ec8a2",
+  "checkpoint": "/runs/python-124m-muon-5b-20261021-v1-s008/step-000025134.pt",
+  "weights_sha256": "99ce99f9dd43f0ddfda6d9e99110bcefb819d72cf8378a7948e3d06503f6f26e",
   "weight_roundtrip_exact": true,
   "cpu_logits_roundtrip_exact": true,
   "loss_token_exposures": {
-    "python_implementation": 812527466,
-    "python_tests_examples": 187505735,
-    "python_documentation": 125004796,
-    "cs_math": 62501707,
-    "general_education": 37501065,
-    "books": 25002661
+    "python_implementation": 975031828,
+    "python_tests_examples": 225010687,
+    "python_documentation": 150004046,
+    "cs_math": 75003081,
+    "general_education": 45006772,
+    "books": 30008725
   }
 }

runtime-cost-estimate.json

@@ -1,11 +1,11 @@
 {
   "kind": "GPU compute estimate, not measured invoice",
-  "training_worker_reported_seconds": 8451.06975036,
+  "training_worker_reported_seconds": 9925.474066897,
   "h100_usd_per_second": 0.001097,
   "excludes": [
     "GPU work outside recorded training segments",
     "CPU preparation and evaluation",
     "storage and transfer"
   ],
-  "training_gpu_usd_estimate": 74.16658812915936
+  "training_gpu_usd_estimate": 87.10596041108809
 }

validation.jsonl

@@ -55,3 +55,14 @@
 {"step": 20000, "tokens": 1202829235, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.8195488412400105, "python_tests_examples": 1.6549985057441632, "python_documentation": 2.7682803908320976, "cs_math": 2.855537448354571, "general_education": 3.951401117957689, "books": 4.087104636168625}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.8195488412400105, "python_tests_examples": 1.6549985057441632, "python_documentation": 2.7682803908320976, "cs_math": 2.855537448354571, "general_education": 3.951401117957689, "books": 4.087104636168625}, "weighted_loss": 2.050845560430673}
 {"step": 20500, "tokens": 1231597289, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.8230518750977858, "python_tests_examples": 1.641852645034251, "python_documentation": 2.757211575758287, "cs_math": 2.9303158158619897, "general_education": 3.940093853837251, "books": 4.0772245839636545}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.8230518750977858, "python_tests_examples": 1.641852645034251, "python_documentation": 2.757211575758287, "cs_math": 2.9303158158619897, "general_education": 3.940093853837251, "books": 4.0772245839636545}, "weighted_loss": 2.053245871232017}
 {"step": 20819, "tokens": 1250043430, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.8134269705343335, "python_tests_examples": 1.6431167731638394, "python_documentation": 2.7631289270946664, "cs_math": 2.931464609876069, "general_education": 3.9224856050651864, "books": 4.08663177344478}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.8134269705343335, "python_tests_examples": 1.6431167731638394, "python_documentation": 2.7631289270946664, "cs_math": 2.931464609876069, "general_education": 3.9224856050651864, "books": 4.08663177344478}, "weighted_loss": 2.047488373646014}
+{"step": 20819, "tokens": 1250043430, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.8134269705343335, "python_tests_examples": 1.6431167731638394, "python_documentation": 2.7631289270946664, "cs_math": 2.931464609876069, "general_education": 3.9224856050651864, "books": 4.08663177344478}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.8134269705343335, "python_tests_examples": 1.6431167731638394, "python_documentation": 2.7631289270946664, "cs_math": 2.931464609876069, "general_education": 3.9224856050651864, "books": 4.08663177344478}, "weighted_loss": 2.047488373646014}
+{"step": 21000, "tokens": 1260766304, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.882422600740141, "python_tests_examples": 1.6866194222269926, "python_documentation": 2.797401507012038, "cs_math": 2.8983131346516577, "general_education": 3.9631784796920306, "books": 4.1020163289635665}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.882422600740141, "python_tests_examples": 1.6866194222269926, "python_documentation": 2.797401507012038, "cs_math": 2.8983131346516577, "general_education": 3.9631784796920306, "books": 4.1020163289635665}, "weighted_loss": 2.1021590922189595}
+{"step": 21500, "tokens": 1289336997, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.8136476529929875, "python_tests_examples": 1.6389241846046052, "python_documentation": 2.764635415907759, "cs_math": 2.8469049487086413, "general_education": 3.9192050746301703, "books": 4.029935737699964}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.8136476529929875, "python_tests_examples": 1.6389241846046052, "python_documentation": 2.764635415907759, "cs_math": 2.8469049487086413, "general_education": 3.9192050746301703, "books": 4.029935737699964}, "weighted_loss": 2.0416932581552447}
+{"step": 22000, "tokens": 1318668934, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.833382329573898, "python_tests_examples": 1.649568415165966, "python_documentation": 2.7616877844554915, "cs_math": 2.864320183509569, "general_education": 3.904631296062873, "books": 4.126818982489824}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.833382329573898, "python_tests_examples": 1.649568415165966, "python_documentation": 2.7616877844554915, "cs_math": 2.864320183509569, "general_education": 3.904631296062873, "books": 4.126818982489824}, "weighted_loss": 2.058193882650639}
+{"step": 22500, "tokens": 1346203175, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.8054147494652306, "python_tests_examples": 1.636037399444906, "python_documentation": 2.762533861199212, "cs_math": 3.288824270209026, "general_education": 3.926987012410063, "books": 4.109962069093029}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.8054147494652306, "python_tests_examples": 1.636037399444906, "python_documentation": 2.762533861199212, "cs_math": 3.288824270209026, "general_education": 3.926987012410063, "books": 4.109962069093029}, "weighted_loss": 2.0596286484536708}
+{"step": 23000, "tokens": 1375135872, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.787296452462399, "python_tests_examples": 1.6285698735422154, "python_documentation": 2.7461240135548346, "cs_math": 2.8530820032701207, "general_education": 3.893942036416225, "books": 4.02095759026914}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.787296452462399, "python_tests_examples": 1.6285698735422154, "python_documentation": 2.7461240135548346, "cs_math": 2.8530820032701207, "general_education": 3.893942036416225, "books": 4.02095759026914}, "weighted_loss": 2.020532089548751}
+{"step": 23500, "tokens": 1403717997, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.7952488828815003, "python_tests_examples": 1.6253046045480386, "python_documentation": 2.7479297945059935, "cs_math": 2.8703087680616544, "general_education": 3.884099496469049, "books": 3.998890467704}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.7952488828815003, "python_tests_examples": 1.6253046045480386, "python_documentation": 2.7479297945059935, "cs_math": 2.8703087680616544, "general_education": 3.884099496469049, "books": 3.998890467704}, "weighted_loss": 2.0255166766570145}
+{"step": 24000, "tokens": 1432418425, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.7686402057100945, "python_tests_examples": 1.6096693997471638, "python_documentation": 2.7134478635599883, "cs_math": 2.8159821645605554, "general_education": 3.8711856056621134, "books": 4.009074157692712}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.7686402057100945, "python_tests_examples": 1.6096693997471638, "python_documentation": 2.7134478635599883, "cs_math": 2.8159821645605554, "general_education": 3.8711856056621134, "books": 4.009074157692712}, "weighted_loss": 1.9995274895813804}
+{"step": 24500, "tokens": 1462684944, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.7934891613066972, "python_tests_examples": 1.6076885935941783, "python_documentation": 2.7254708976246254, "cs_math": 2.800588991551641, "general_education": 3.8950732961405246, "books": 4.095124763690478}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.7934891613066972, "python_tests_examples": 1.6076885935941783, "python_documentation": 2.7254708976246254, "cs_math": 2.800588991551641, "general_education": 3.8950732961405246, "books": 4.095124763690478}, "weighted_loss": 2.01825247738655}
+{"step": 25000, "tokens": 1492026967, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.806598180626368, "python_tests_examples": 1.6217702912916119, "python_documentation": 2.7367518695776885, "cs_math": 2.8490930317757166, "general_education": 3.8581286404899005, "books": 3.9950269532361795}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.806598180626368, "python_tests_examples": 1.6217702912916119, "python_documentation": 2.7367518695776885, "cs_math": 2.8490930317757166, "general_education": 3.8581286404899005, "books": 3.9950269532361795}, "weighted_loss": 2.029328597926856}
+{"step": 25134, "tokens": 1500065139, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.9026956493929437, "python_tests_examples": 1.7113816452590394, "python_documentation": 2.8754445534067337, "cs_math": 2.9621716956660684, "general_education": 4.001712479204489, "books": 4.094768780210336}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.9026956493929437, "python_tests_examples": 1.7113816452590394, "python_documentation": 2.8754445534067337, "cs_math": 2.9621716956660684, "general_education": 4.001712479204489, "books": 4.094768780210336}, "weighted_loss": 2.1310592089985874}