python-124m-muon-5b-20261021-v1: 1750010347 training targets

README.md

@@ -11,7 +11,7 @@ tags:
 # Simple LLM Python 124M / 5B
 
 Run: `python-124m-muon-5b-20261021-v1`. This is an **intermediate** base-model checkpoint, freshly initialized.
-Processed **1,500,065,139 nonpadding training targets** in **25,134 optimizer updates**.
+Processed **1,750,010,347 nonpadding training targets** in **29,477 optimizer updates**.
 The exact target is 5,000,000,000. This model is not instruction-tuned.
 
 124,389,632 parameters; 14 layers, width 768, 12 attention heads, RMSNorm,

metrics.jsonl

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c31c2362c2423768a517e0f58344f6ba00cfe74ac3ceca8ef6dc84117a4255a2
-size 40585201
+oid sha256:c0dd656f8d3d03c4148bdb72d54553a4f436c8d5fb2b40662bd246df1d621dc1
+size 47605051

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:99ce99f9dd43f0ddfda6d9e99110bcefb819d72cf8378a7948e3d06503f6f26e
+oid sha256:0791d9601f6e02040b31d09b17080852e80eabe9a211da90eec1c612b61cd076
 size 598238584

provenance.json

@@ -49,20 +49,20 @@
   "torch_version": "2.6.0+cu124",
   "gpu": "NVIDIA H100 80GB HBM3",
   "parameters": 124389632,
-  "resume_from": "/runs/python-124m-muon-5b-20261021-v1-s007/step-000020819.pt",
-  "step": 25134,
-  "tokens": 1500065139,
+  "resume_from": "/runs/python-124m-muon-5b-20261021-v1-s008/step-000025134.pt",
+  "step": 29477,
+  "tokens": 1750010347,
   "run_id": "python-124m-muon-5b-20261021-v1",
-  "checkpoint": "/runs/python-124m-muon-5b-20261021-v1-s008/step-000025134.pt",
-  "weights_sha256": "99ce99f9dd43f0ddfda6d9e99110bcefb819d72cf8378a7948e3d06503f6f26e",
+  "checkpoint": "/runs/python-124m-muon-5b-20261021-v1-s009/step-000029477.pt",
+  "weights_sha256": "0791d9601f6e02040b31d09b17080852e80eabe9a211da90eec1c612b61cd076",
   "weight_roundtrip_exact": true,
   "cpu_logits_roundtrip_exact": true,
   "loss_token_exposures": {
-    "python_implementation": 975031828,
-    "python_tests_examples": 225010687,
-    "python_documentation": 150004046,
-    "cs_math": 75003081,
-    "general_education": 45006772,
-    "books": 30008725
+    "python_implementation": 1137504011,
+    "python_tests_examples": 262504036,
+    "python_documentation": 174999404,
+    "cs_math": 87500573,
+    "general_education": 52501748,
+    "books": 35000575
   }
 }

runtime-cost-estimate.json

@@ -1,11 +1,11 @@
 {
   "kind": "GPU compute estimate, not measured invoice",
-  "training_worker_reported_seconds": 9925.474066897,
+  "training_worker_reported_seconds": 11565.736927455,
   "h100_usd_per_second": 0.001097,
   "excludes": [
     "GPU work outside recorded training segments",
     "CPU preparation and evaluation",
     "storage and transfer"
   ],
-  "training_gpu_usd_estimate": 87.10596041108809
+  "training_gpu_usd_estimate": 101.50090727534509
 }

validation.jsonl

@@ -66,3 +66,13 @@
 {"step": 24500, "tokens": 1462684944, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.7934891613066972, "python_tests_examples": 1.6076885935941783, "python_documentation": 2.7254708976246254, "cs_math": 2.800588991551641, "general_education": 3.8950732961405246, "books": 4.095124763690478}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.7934891613066972, "python_tests_examples": 1.6076885935941783, "python_documentation": 2.7254708976246254, "cs_math": 2.800588991551641, "general_education": 3.8950732961405246, "books": 4.095124763690478}, "weighted_loss": 2.01825247738655}
 {"step": 25000, "tokens": 1492026967, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.806598180626368, "python_tests_examples": 1.6217702912916119, "python_documentation": 2.7367518695776885, "cs_math": 2.8490930317757166, "general_education": 3.8581286404899005, "books": 3.9950269532361795}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.806598180626368, "python_tests_examples": 1.6217702912916119, "python_documentation": 2.7367518695776885, "cs_math": 2.8490930317757166, "general_education": 3.8581286404899005, "books": 3.9950269532361795}, "weighted_loss": 2.029328597926856}
 {"step": 25134, "tokens": 1500065139, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.9026956493929437, "python_tests_examples": 1.7113816452590394, "python_documentation": 2.8754445534067337, "cs_math": 2.9621716956660684, "general_education": 4.001712479204489, "books": 4.094768780210336}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.9026956493929437, "python_tests_examples": 1.7113816452590394, "python_documentation": 2.8754445534067337, "cs_math": 2.9621716956660684, "general_education": 4.001712479204489, "books": 4.094768780210336}, "weighted_loss": 2.1310592089985874}
+{"step": 25134, "tokens": 1500065139, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.9026956493929437, "python_tests_examples": 1.7113816452590394, "python_documentation": 2.8754445534067337, "cs_math": 2.9621716956660684, "general_education": 4.001712479204489, "books": 4.094768780210336}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.9026956493929437, "python_tests_examples": 1.7113816452590394, "python_documentation": 2.8754445534067337, "cs_math": 2.9621716956660684, "general_education": 4.001712479204489, "books": 4.094768780210336}, "weighted_loss": 2.1310592089985874}
+{"step": 25500, "tokens": 1520876053, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.7926663177893423, "python_tests_examples": 1.6243597628820778, "python_documentation": 2.7476645225161525, "cs_math": 2.978623538164787, "general_education": 3.9159063079452934, "books": 4.095203354577838}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.7926663177893423, "python_tests_examples": 1.6243597628820778, "python_documentation": 2.7476645225161525, "cs_math": 2.978623538164787, "general_education": 3.9159063079452934, "books": 4.095203354577838}, "weighted_loss": 2.031965956485154}
+{"step": 26000, "tokens": 1549543290, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.7801283355030186, "python_tests_examples": 1.5968990419932103, "python_documentation": 2.7203067559115204, "cs_math": 2.893907865869423, "general_education": 3.878497257694033, "books": 4.01891057198479}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.7801283355030186, "python_tests_examples": 1.5968990419932103, "python_documentation": 2.7203067559115204, "cs_math": 2.893907865869423, "general_education": 3.878497257694033, "books": 4.01891057198479}, "weighted_loss": 2.010077472431084}
+{"step": 26500, "tokens": 1578268315, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.7879225422594391, "python_tests_examples": 1.6075021221423473, "python_documentation": 2.7300201195721567, "cs_math": 2.792680152562876, "general_education": 3.874709540129418, "books": 4.003693356686895}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.7879225422594391, "python_tests_examples": 1.6075021221423473, "python_documentation": 2.7300201195721567, "cs_math": 2.792680152562876, "general_education": 3.874709540129418, "books": 4.003693356686895}, "weighted_loss": 2.0122261437129674}
+{"step": 27000, "tokens": 1607323902, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.8042014060640117, "python_tests_examples": 1.619506073875421, "python_documentation": 2.7276876524224067, "cs_math": 2.837299437167873, "general_education": 3.8703949451321975, "books": 4.050724830264092}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.8042014060640117, "python_tests_examples": 1.619506073875421, "python_documentation": 2.7276876524224067, "cs_math": 2.837299437167873, "general_education": 3.8703949451321975, "books": 4.050724830264092}, "weighted_loss": 2.0274169070828028}
+{"step": 27500, "tokens": 1635899048, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.7725933315575124, "python_tests_examples": 1.6090353957455559, "python_documentation": 2.7394131883225885, "cs_math": 2.822554137927651, "general_education": 3.8836554761503654, "books": 4.169520102361804}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.7725933315575124, "python_tests_examples": 1.6090353957455559, "python_documentation": 2.7394131883225885, "cs_math": 2.822554137927651, "general_education": 3.8836554761503654, "books": 4.169520102361804}, "weighted_loss": 2.008510066934605}
+{"step": 28000, "tokens": 1664597353, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.77915740295185, "python_tests_examples": 1.59835466986829, "python_documentation": 2.767492920739809, "cs_math": 2.7828748612306153, "general_education": 3.864995001500991, "books": 3.9935997905441103}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.77915740295185, "python_tests_examples": 1.59835466986829, "python_documentation": 2.767492920739809, "cs_math": 2.7828748612306153, "general_education": 3.864995001500991, "books": 3.9935997905441103}, "weighted_loss": 2.0079203933903695}
+{"step": 28500, "tokens": 1692478231, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.7574180564067199, "python_tests_examples": 1.5896200781995016, "python_documentation": 2.7066559490340487, "cs_math": 2.7939421301146874, "general_education": 3.846267246600883, "books": 3.988833050094938}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.7574180564067199, "python_tests_examples": 1.5896200781995016, "python_documentation": 2.7066559490340487, "cs_math": 2.7939421301146874, "general_education": 3.846267246600883, "books": 3.988833050094938}, "weighted_loss": 1.9862921282033577}
+{"step": 29000, "tokens": 1722016887, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.7802874831173077, "python_tests_examples": 1.5980181085352476, "python_documentation": 2.729425466859814, "cs_math": 2.79540980915752, "general_education": 3.8600788159303407, "books": 4.008424649184839}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.7802874831173077, "python_tests_examples": 1.5980181085352476, "python_documentation": 2.729425466859814, "cs_math": 2.79540980915752, "general_education": 3.8600788159303407, "books": 4.008424649184839}, "weighted_loss": 2.0055734749120013}
+{"step": 29477, "tokens": 1750010347, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.7467950020957397, "python_tests_examples": 1.5897608021121512, "python_documentation": 2.6845406729359866, "cs_math": 2.7847005012754265, "general_education": 3.850954334300287, "books": 4.067085262930438}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.7467950020957397, "python_tests_examples": 1.5897608021121512, "python_documentation": 2.6845406729359866, "cs_math": 2.7847005012754265, "general_education": 3.850954334300287, "books": 4.067085262930438}, "weighted_loss": 1.9784402993240409}