python-124m-muon-5b-20261021-v1: 500044156 training targets

.gitattributes

@@ -35,3 +35,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
 training-curves.png filter=lfs diff=lfs merge=lfs -text
 validation-curves.png filter=lfs diff=lfs merge=lfs -text
+metrics.jsonl filter=lfs diff=lfs merge=lfs -text

README.md

@@ -11,7 +11,7 @@ tags:
 # Simple LLM Python 124M / 5B
 
 Run: `python-124m-muon-5b-20261021-v1`. This is an **intermediate** base-model checkpoint, freshly initialized.
-Processed **250,051,061 nonpadding training targets** in **3,492 optimizer updates**.
+Processed **500,044,156 nonpadding training targets** in **7,739 optimizer updates**.
 The exact target is 5,000,000,000. This model is not instruction-tuned.
 
 124,389,632 parameters; 14 layers, width 768, 12 attention heads, RMSNorm,

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:eeb38d5c42e571001fa732d6c420bb9db632c7f0e2d3670877090eb4de9ddbec
+oid sha256:c364510402aed86abcfb3624b17a0494e93089ad9b549c4ca785ac61fb58b83c
 size 598238584

provenance.json

@@ -49,20 +49,20 @@
   "torch_version": "2.6.0+cu124",
   "gpu": "NVIDIA H100 80GB HBM3",
   "parameters": 124389632,
-  "resume_from": "/runs/python-124m-muon-5b-20261021-v1-s002/step-000000549.pt",
-  "step": 3492,
-  "tokens": 250051061,
+  "resume_from": "/runs/python-124m-muon-5b-20261021-v1-s003/step-000003492.pt",
+  "step": 7739,
+  "tokens": 500044156,
   "run_id": "python-124m-muon-5b-20261021-v1",
-  "checkpoint": "/runs/python-124m-muon-5b-20261021-v1-s003/step-000003492.pt",
-  "weights_sha256": "eeb38d5c42e571001fa732d6c420bb9db632c7f0e2d3670877090eb4de9ddbec",
+  "checkpoint": "/runs/python-124m-muon-5b-20261021-v1-s004/step-000007739.pt",
+  "weights_sha256": "c364510402aed86abcfb3624b17a0494e93089ad9b549c4ca785ac61fb58b83c",
   "weight_roundtrip_exact": true,
   "cpu_logits_roundtrip_exact": true,
   "loss_token_exposures": {
-    "python_implementation": 162530375,
-    "python_tests_examples": 37508199,
-    "python_documentation": 25004987,
-    "cs_math": 12502399,
-    "general_education": 7501644,
-    "books": 5003457
+    "python_implementation": 325019540,
+    "python_tests_examples": 75010988,
+    "python_documentation": 50004745,
+    "cs_math": 25005168,
+    "general_education": 15001035,
+    "books": 10002680
   }
 }

runtime-cost-estimate.json

@@ -1,11 +1,11 @@
 {
   "kind": "GPU compute estimate, not measured invoice",
-  "training_worker_reported_seconds": 2091.3677939050003,
+  "training_worker_reported_seconds": 4307.891061891,
   "h100_usd_per_second": 0.001097,
   "excludes": [
     "GPU work outside recorded training segments",
     "CPU preparation and evaluation",
     "storage and transfer"
   ],
-  "training_gpu_usd_estimate": 18.353843759310283
+  "training_gpu_usd_estimate": 37.806051959155425
 }

validation.jsonl

@@ -12,3 +12,14 @@
 {"step": 2500, "tokens": 184107656, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 2.1924578516790976, "python_tests_examples": 2.022210039841007, "python_documentation": 3.43803907747578, "cs_math": 3.147088807277527, "general_education": 4.217252623405235, "books": 4.347502121851156}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 2.1924578516790976, "python_tests_examples": 2.022210039841007, "python_documentation": 3.43803907747578, "cs_math": 3.147088807277527, "general_education": 4.217252623405235, "books": 4.347502121851156}, "weighted_loss": 2.4430550788181993}
 {"step": 3000, "tokens": 217806656, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 2.068694597510086, "python_tests_examples": 1.918312337532877, "python_documentation": 3.380592492439255, "cs_math": 3.08924052137883, "general_education": 4.137618646272441, "books": 4.255268273750112}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 2.068694597510086, "python_tests_examples": 1.918312337532877, "python_documentation": 3.380592492439255, "cs_math": 3.08924052137883, "general_education": 4.137618646272441, "books": 4.255268273750112}, "weighted_loss": 2.33415353918753}
 {"step": 3492, "tokens": 250051061, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 2.0556095562875134, "python_tests_examples": 1.9033838904744125, "python_documentation": 3.356063952920466, "cs_math": 3.2277762661045117, "general_education": 4.149975508770645, "books": 4.269131061287444}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 2.0556095562875134, "python_tests_examples": 1.9033838904744125, "python_documentation": 3.356063952920466, "cs_math": 3.2277762661045117, "general_education": 4.149975508770645, "books": 4.269131061287444}, "weighted_loss": 2.328530890244186}
+{"step": 3492, "tokens": 250051061, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 2.0555805144808352, "python_tests_examples": 1.9033394301683813, "python_documentation": 3.356108079190304, "cs_math": 3.227825874372679, "general_education": 4.150068663833321, "books": 4.2691129489454465}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 2.0555805144808352, "python_tests_examples": 1.9033394301683813, "python_documentation": 3.356108079190304, "cs_math": 3.227825874372679, "general_education": 4.150068663833321, "books": 4.2691129489454465}, "weighted_loss": 2.328514669469373}
+{"step": 3500, "tokens": 250689263, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 2.0735829775672268, "python_tests_examples": 1.9018731976605134, "python_documentation": 3.34724153220845, "cs_math": 3.1957152504192443, "general_education": 4.138100642569044, "books": 4.241033362682663}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 2.0735829775672268, "python_tests_examples": 1.9018731976605134, "python_documentation": 3.34724153220845, "cs_math": 3.1957152504192443, "general_education": 4.138100642569044, "books": 4.241033362682663}, "weighted_loss": 2.3365835173403062}
+{"step": 4000, "tokens": 283060428, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 2.0354083803662046, "python_tests_examples": 1.8549883225016235, "python_documentation": 3.130226218259442, "cs_math": 3.1470833069180864, "general_education": 4.197534637586692, "books": 4.380953744223545}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 2.0354083803662046, "python_tests_examples": 1.8549883225016235, "python_documentation": 3.130226218259442, "cs_math": 3.1470833069180864, "general_education": 4.197534637586692, "books": 4.380953744223545}, "weighted_loss": 2.285185596797197}
+{"step": 4500, "tokens": 312478073, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 2.0503928708790116, "python_tests_examples": 1.8547063482518618, "python_documentation": 3.0390099372056114, "cs_math": 3.3040542850031094, "general_education": 4.154135087672262, "books": 4.261745634853236}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 2.0503928708790116, "python_tests_examples": 1.8547063482518618, "python_documentation": 3.0390099372056114, "cs_math": 3.3040542850031094, "general_education": 4.154135087672262, "books": 4.261745634853236}, "weighted_loss": 2.289923991607086}
+{"step": 5000, "tokens": 341143218, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 2.0144936650683274, "python_tests_examples": 1.834447479373386, "python_documentation": 2.9856509178803794, "cs_math": 3.038316841526908, "general_education": 4.1000303154137185, "books": 4.248347954134631}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 2.0144936650683274, "python_tests_examples": 1.834447479373386, "python_documentation": 2.9856509178803794, "cs_math": 3.038316841526908, "general_education": 4.1000303154137185, "books": 4.248347954134631}, "weighted_loss": 2.2430368066099082}
+{"step": 5500, "tokens": 370117057, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 2.013591982024698, "python_tests_examples": 1.8352118621069122, "python_documentation": 2.99600414829098, "cs_math": 3.075333256106162, "general_education": 4.1229966894398595, "books": 4.297999465364176}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 2.013591982024698, "python_tests_examples": 1.8352118621069122, "python_documentation": 2.99600414829098, "cs_math": 3.075333256106162, "general_education": 4.1229966894398595, "books": 4.297999465364176}, "weighted_loss": 2.247133535256976}
+{"step": 6000, "tokens": 399042307, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.973133626291731, "python_tests_examples": 1.7973139822257844, "python_documentation": 2.954126155327438, "cs_math": 2.988975753681561, "general_education": 4.078878806978985, "books": 4.250536836566654}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.973133626291731, "python_tests_examples": 1.7973139822257844, "python_documentation": 2.954126155327438, "cs_math": 2.988975753681561, "general_education": 4.078878806978985, "books": 4.250536836566654}, "weighted_loss": 2.2043724585810174}
+{"step": 6500, "tokens": 427727758, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.9693918560385164, "python_tests_examples": 1.7866897363664287, "python_documentation": 2.9405420648739984, "cs_math": 2.9657696623507532, "general_education": 4.079703859373367, "books": 4.264720867830421}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.9693918560385164, "python_tests_examples": 1.7866897363664287, "python_documentation": 2.9405420648739984, "cs_math": 2.9657696623507532, "general_education": 4.079703859373367, "books": 4.264720867830421}, "weighted_loss": 2.198136389622747}
+{"step": 7000, "tokens": 456372695, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.9531801376163918, "python_tests_examples": 1.7893383780664776, "python_documentation": 2.9155548204269097, "cs_math": 3.125105384520615, "general_education": 4.056566197248992, "books": 4.171409649500405}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.9531801376163918, "python_tests_examples": 1.7893383780664776, "python_documentation": 2.9155548204269097, "cs_math": 3.125105384520615, "general_education": 4.056566197248992, "books": 4.171409649500405}, "weighted_loss": 2.1909037763368255}
+{"step": 7500, "tokens": 486193519, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.949939646881978, "python_tests_examples": 1.7539362380201835, "python_documentation": 2.8834913661762975, "cs_math": 3.0106634796019436, "general_education": 4.041560746654249, "books": 4.2517702006261135}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.949939646881978, "python_tests_examples": 1.7539362380201835, "python_documentation": 2.8834913661762975, "cs_math": 3.0106634796019436, "general_education": 4.041560746654249, "books": 4.2517702006261135}, "weighted_loss": 2.1757157431861898}
+{"step": 7739, "tokens": 500044156, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.9559222801322604, "python_tests_examples": 1.776924798225013, "python_documentation": 2.89655862147543, "cs_math": 2.983571789368241, "general_education": 4.077060617154236, "books": 4.229515550403194}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.9559222801322604, "python_tests_examples": 1.776924798225013, "python_documentation": 2.89655862147543, "cs_math": 2.983571789368241, "general_education": 4.077060617154236, "books": 4.229515550403194}, "weighted_loss": 2.183624782958367}