python-124m-muon-5b-20261021-v1: 2750029614 training targets

README.md

@@ -11,7 +11,7 @@ tags:
 # Simple LLM Python 124M / 5B
 
 Run: `python-124m-muon-5b-20261021-v1`. This is an **intermediate** base-model checkpoint, freshly initialized.
-Processed **2,500,025,813 nonpadding training targets** in **42,519 optimizer updates**.
+Processed **2,750,029,614 nonpadding training targets** in **46,880 optimizer updates**.
 The exact target is 5,000,000,000. This model is not instruction-tuned.
 
 124,389,632 parameters; 14 layers, width 768, 12 attention heads, RMSNorm,

metrics.jsonl

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e8c69bacee5259d530ee64b94a5db49ccad3a89bc04f4d42ac7faf30f8e66e2e
-size 68693730
+oid sha256:ddcce92c81019a608eeba85cef0194d9c82fceb28ae2509cab63bbd68db5e4d0
+size 75744653

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:fe81807ad6958176c3836f364259ef9f1158beb2ed30a3da25c7d613a2337c71
+oid sha256:abcf556a9434e6fc77196a76841a237b65f3b974127b47b5130e6157fb09850a
 size 598238584

provenance.json

@@ -47,22 +47,22 @@
     "tokenizer": "00e32adc13dab74f5a029c63304b4591bd48f7cd6b086b22d4624d244667b4da"
   },
   "torch_version": "2.6.0+cu124",
-  "gpu": "NVIDIA H200",
+  "gpu": "NVIDIA H100 80GB HBM3",
   "parameters": 124389632,
-  "resume_from": "/runs/python-124m-muon-5b-20261021-v1-s011/step-000038226.pt",
-  "step": 42519,
-  "tokens": 2500025813,
+  "resume_from": "/runs/python-124m-muon-5b-20261021-v1-s012/step-000042519.pt",
+  "step": 46880,
+  "tokens": 2750029614,
   "run_id": "python-124m-muon-5b-20261021-v1",
-  "checkpoint": "/runs/python-124m-muon-5b-20261021-v1-s012/step-000042519.pt",
-  "weights_sha256": "fe81807ad6958176c3836f364259ef9f1158beb2ed30a3da25c7d613a2337c71",
+  "checkpoint": "/runs/python-124m-muon-5b-20261021-v1-s013/step-000046880.pt",
+  "weights_sha256": "abcf556a9434e6fc77196a76841a237b65f3b974127b47b5130e6157fb09850a",
   "weight_roundtrip_exact": true,
   "cpu_logits_roundtrip_exact": true,
   "loss_token_exposures": {
-    "python_implementation": 1625006416,
-    "python_tests_examples": 375009387,
-    "python_documentation": 250001615,
-    "cs_math": 125003765,
-    "general_education": 75000584,
-    "books": 50004046
+    "python_implementation": 1787518112,
+    "python_tests_examples": 412503978,
+    "python_documentation": 275002110,
+    "cs_math": 137501054,
+    "general_education": 82501544,
+    "books": 55002816
   }
 }

runtime-cost-estimate.json

@@ -1,11 +1,11 @@
 {
   "kind": "GPU compute estimate, not measured invoice",
-  "training_worker_reported_seconds": 15882.104739504,
+  "training_worker_reported_seconds": 17408.777889444,
   "h100_usd_per_second": 0.001097,
   "excludes": [
     "GPU work outside recorded training segments",
     "CPU preparation and evaluation",
     "storage and transfer"
   ],
-  "training_gpu_usd_estimate": 139.38135119388713
+  "training_gpu_usd_estimate": 152.77943475776058
 }

validation.jsonl

@@ -109,3 +109,13 @@
 {"step": 42000, "tokens": 2470760694, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.6584197833332812, "python_tests_examples": 1.5135724393726473, "python_documentation": 2.6091162778057058, "cs_math": 2.7192630648603546, "general_education": 3.733178821266971, "books": 3.8689184870439757}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.6584197833332812, "python_tests_examples": 1.5135724393726473, "python_documentation": 2.6091162778057058, "cs_math": 2.7192630648603546, "general_education": 3.733178821266971, "books": 3.8689184870439757}, "weighted_loss": 1.891257240475007}
 {"step": 42500, "tokens": 2499010434, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.672793460030488, "python_tests_examples": 1.5085760987370946, "python_documentation": 2.6524121191564776, "cs_math": 2.6837753270545597, "general_education": 3.755705071581761, "books": 3.930422787980831}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.672793460030488, "python_tests_examples": 1.5085760987370946, "python_documentation": 2.6524121191564776, "cs_math": 2.6837753270545597, "general_education": 3.755705071581761, "books": 3.930422787980831}, "weighted_loss": 1.9043117500058264}
 {"step": 42519, "tokens": 2500025813, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.6665285634459845, "python_tests_examples": 1.5116764890828391, "python_documentation": 2.6236265730134023, "cs_math": 2.671596395204467, "general_education": 3.7442769424823514, "books": 3.91341999350956}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.6665285634459845, "python_tests_examples": 1.5116764890828391, "python_documentation": 2.6236265730134023, "cs_math": 2.671596395204467, "general_education": 3.7442769424823514, "books": 3.91341999350956}, "weighted_loss": 1.896534224808541}
+{"step": 42519, "tokens": 2500025813, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.6665285634459845, "python_tests_examples": 1.5116764890828391, "python_documentation": 2.6236265730134023, "cs_math": 2.671596395204467, "general_education": 3.7442769424823514, "books": 3.91341999350956}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.6665285634459845, "python_tests_examples": 1.5116764890828391, "python_documentation": 2.6236265730134023, "cs_math": 2.671596395204467, "general_education": 3.7442769424823514, "books": 3.91341999350956}, "weighted_loss": 1.896534224808541}
+{"step": 43000, "tokens": 2527971083, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.662284612737962, "python_tests_examples": 1.4964215858475767, "python_documentation": 2.600178602194451, "cs_math": 2.7010346768881215, "general_education": 3.724890235249983, "books": 3.9124972920963574}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.662284612737962, "python_tests_examples": 1.4964215858475767, "python_documentation": 2.600178602194451, "cs_math": 2.7010346768881215, "general_education": 3.724890235249983, "books": 3.9124972920963574}, "weighted_loss": 1.8900144831200896}
+{"step": 43500, "tokens": 2556315179, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.6593065286970807, "python_tests_examples": 1.5066770522567425, "python_documentation": 2.600363345639107, "cs_math": 2.6659042873906227, "general_education": 3.7102325935298004, "books": 3.885079257506833}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.6593065286970807, "python_tests_examples": 1.5066770522567425, "python_documentation": 2.600363345639107, "cs_math": 2.6659042873906227, "general_education": 3.7102325935298004, "books": 3.885079257506833}, "weighted_loss": 1.8868909133810865}
+{"step": 44000, "tokens": 2585733432, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.65876134067965, "python_tests_examples": 1.4965265943672301, "python_documentation": 2.590752870897446, "cs_math": 2.655538734468028, "general_education": 3.701792870486619, "books": 3.851070253224629}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.65876134067965, "python_tests_examples": 1.4965265943672301, "python_documentation": 2.590752870897446, "cs_math": 2.655538734468028, "general_education": 3.701792870486619, "books": 3.851070253224629}, "weighted_loss": 1.8826012755890942}
+{"step": 44500, "tokens": 2614214744, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.6427071035977403, "python_tests_examples": 1.489512216835279, "python_documentation": 2.5802467922486985, "cs_math": 2.6597350916065308, "general_education": 3.6928744418986894, "books": 3.8242913897436672}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.6427071035977403, "python_tests_examples": 1.489512216835279, "python_documentation": 2.5802467922486985, "cs_math": 2.6597350916065308, "general_education": 3.6928744418986894, "books": 3.8242913897436672}, "weighted_loss": 1.8694699447208536}
+{"step": 45000, "tokens": 2643742535, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.6504271266344603, "python_tests_examples": 1.489330347210624, "python_documentation": 2.5932837460476086, "cs_math": 2.6417932825022854, "general_education": 3.755251737032295, "books": 3.8289579849712263}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.6504271266344603, "python_tests_examples": 1.489330347210624, "python_documentation": 2.5932837460476086, "cs_math": 2.6417932825022854, "general_education": 3.755251737032295, "books": 3.8289579849712263}, "weighted_loss": 1.8768319349342613}
+{"step": 45500, "tokens": 2671594082, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.6420125507057544, "python_tests_examples": 1.4934213985181783, "python_documentation": 2.575857575461065, "cs_math": 2.6424521159424, "general_education": 3.7008286913537, "books": 3.8692998818696074}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.6420125507057544, "python_tests_examples": 1.4934213985181783, "python_documentation": 2.575857575461065, "cs_math": 2.6424521159424, "general_education": 3.7008286913537, "books": 3.8692998818696074}, "weighted_loss": 1.8694405894576969}
+{"step": 46000, "tokens": 2699872835, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.6516034398742783, "python_tests_examples": 1.484993995577513, "python_documentation": 2.572842087115055, "cs_math": 2.638648783390031, "general_education": 3.704827610137638, "books": 3.8900709382420082}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.6516034398742783, "python_tests_examples": 1.484993995577513, "python_documentation": 2.572842087115055, "cs_math": 2.638648783390031, "general_education": 3.704827610137638, "books": 3.8900709382420082}, "weighted_loss": 1.8744542302048843}
+{"step": 46500, "tokens": 2728888163, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.6491192107410588, "python_tests_examples": 1.4691654664063583, "python_documentation": 2.5695742893452858, "cs_math": 2.664499644567642, "general_education": 3.692125639604816, "books": 3.8290243232799543}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.6491192107410588, "python_tests_examples": 1.4691654664063583, "python_documentation": 2.5695742893452858, "cs_math": 2.664499644567642, "general_education": 3.692125639604816, "books": 3.8290243232799543}, "weighted_loss": 1.8698289737592964}
+{"step": 46880, "tokens": 2750029614, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.6467544658541027, "python_tests_examples": 1.4647498506032444, "python_documentation": 2.572555247388413, "cs_math": 2.648651639761941, "general_education": 3.69639633080658, "books": 3.806166974120848}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.6467544658541027, "python_tests_examples": 1.4647498506032444, "python_documentation": 2.572555247388413, "cs_math": 2.648651639761941, "general_education": 3.69639633080658, "books": 3.806166974120848}, "weighted_loss": 1.866806216529206}