python-124m-muon-5b-20261021-v1: 1000070440 training targets

README.md

@@ -11,7 +11,7 @@ tags:
 # Simple LLM Python 124M / 5B
 
 Run: `python-124m-muon-5b-20261021-v1`. This is an **intermediate** base-model checkpoint, freshly initialized.
-Processed **750,036,273 nonpadding training targets** in **12,072 optimizer updates**.
+Processed **1,000,070,440 nonpadding training targets** in **16,446 optimizer updates**.
 The exact target is 5,000,000,000. This model is not instruction-tuned.
 
 124,389,632 parameters; 14 layers, width 768, 12 attention heads, RMSNorm,

metrics.jsonl

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:62e3ce38dd6fecc79cf8165c53fe22f2ddc6634c93d7dc9be7a2cd92b71a0564
-size 19475495
+oid sha256:7bc3dacc96821bbd8a93dd183962e33f815b6c42a3dc5a27e2a79872dbf1a51b
+size 26543001

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a5719e7c548a8ffef180c6813e37458f582a18c1eb94493d5e036932dea37da6
+oid sha256:128ac2d57ca86eef82fadbcdda3857a541036a6b17e75c063d9f93bddc5d4eec
 size 598238584

provenance.json

@@ -47,22 +47,22 @@
     "tokenizer": "00e32adc13dab74f5a029c63304b4591bd48f7cd6b086b22d4624d244667b4da"
   },
   "torch_version": "2.6.0+cu124",
-  "gpu": "NVIDIA H200",
+  "gpu": "NVIDIA H100 80GB HBM3",
   "parameters": 124389632,
-  "resume_from": "/runs/python-124m-muon-5b-20261021-v1-s004/step-000007739.pt",
-  "step": 12072,
-  "tokens": 750036273,
+  "resume_from": "/runs/python-124m-muon-5b-20261021-v1-s005-retry-001/step-000012072.pt",
+  "step": 16446,
+  "tokens": 1000070440,
   "run_id": "python-124m-muon-5b-20261021-v1",
-  "checkpoint": "/runs/python-124m-muon-5b-20261021-v1-s005-retry-001/step-000012072.pt",
-  "weights_sha256": "a5719e7c548a8ffef180c6813e37458f582a18c1eb94493d5e036932dea37da6",
+  "checkpoint": "/runs/python-124m-muon-5b-20261021-v1-s006/step-000016446.pt",
+  "weights_sha256": "128ac2d57ca86eef82fadbcdda3857a541036a6b17e75c063d9f93bddc5d4eec",
   "weight_roundtrip_exact": true,
   "cpu_logits_roundtrip_exact": true,
   "loss_token_exposures": {
-    "python_implementation": 487521404,
-    "python_tests_examples": 112505163,
-    "python_documentation": 75003002,
-    "cs_math": 37501721,
-    "general_education": 22501446,
-    "books": 15003537
+    "python_implementation": 650035375,
+    "python_tests_examples": 150015147,
+    "python_documentation": 100005889,
+    "cs_math": 50005950,
+    "general_education": 30001554,
+    "books": 20006525
   }
 }

runtime-cost-estimate.json

@@ -1,11 +1,11 @@
 {
   "kind": "GPU compute estimate, not measured invoice",
-  "training_worker_reported_seconds": 5735.615135963,
+  "training_worker_reported_seconds": 6999.435524978,
   "h100_usd_per_second": 0.001097,
   "excludes": [
     "GPU work outside recorded training segments",
     "CPU preparation and evaluation",
     "storage and transfer"
   ],
-  "training_gpu_usd_estimate": 50.335758433211296
+  "training_gpu_usd_estimate": 61.42704616720693
 }

validation.jsonl

@@ -34,3 +34,13 @@
 {"step": 11500, "tokens": 716867733, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.9013674905849665, "python_tests_examples": 1.7188571261247234, "python_documentation": 2.8495332417955663, "cs_math": 2.980339026751044, "general_education": 3.9957867598410317, "books": 4.128864759524754}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.9013674905849665, "python_tests_examples": 1.7188571261247234, "python_documentation": 2.8495332417955663, "cs_math": 2.980339026751044, "general_education": 3.9957867598410317, "books": 4.128864759524754}, "weighted_loss": 2.1301386113017715}
 {"step": 12000, "tokens": 745807584, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.9118772415864633, "python_tests_examples": 1.7364030514951017, "python_documentation": 2.831054179453419, "cs_math": 2.927207559844683, "general_education": 3.984384398731678, "books": 4.080954188179626}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.9118772415864633, "python_tests_examples": 1.7364030514951017, "python_documentation": 2.831054179453419, "cs_math": 2.927207559844683, "general_education": 3.984384398731678, "books": 4.080954188179626}, "weighted_loss": 2.133797076418585}
 {"step": 12072, "tokens": 750036273, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.8791453879206281, "python_tests_examples": 1.7069534428977058, "python_documentation": 2.8507127404455876, "cs_math": 2.9244520415402597, "general_education": 4.020535449075895, "books": 4.112124923362367}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.8791453879206281, "python_tests_examples": 1.7069534428977058, "python_documentation": 2.8507127404455876, "cs_math": 2.9244520415402597, "general_education": 4.020535449075895, "books": 4.112124923362367}, "weighted_loss": 2.11163995664416}
+{"step": 12072, "tokens": 750036273, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.8791453879206281, "python_tests_examples": 1.7069534428977058, "python_documentation": 2.8507127404455876, "cs_math": 2.9244520415402597, "general_education": 4.020535449075895, "books": 4.112124923362367}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.8791453879206281, "python_tests_examples": 1.7069534428977058, "python_documentation": 2.8507127404455876, "cs_math": 2.9244520415402597, "general_education": 4.020535449075895, "books": 4.112124923362367}, "weighted_loss": 2.11163995664416}
+{"step": 12500, "tokens": 774761244, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.8896572397196596, "python_tests_examples": 1.711922849401653, "python_documentation": 2.8439367355775147, "cs_math": 2.929396798276988, "general_education": 4.0076192544237905, "books": 4.08792873821072}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.8896572397196596, "python_tests_examples": 1.711922849401653, "python_documentation": 2.8439367355775147, "cs_math": 2.929396798276988, "general_education": 4.0076192544237905, "books": 4.08792873821072}, "weighted_loss": 2.1179162990965557}
+{"step": 13000, "tokens": 803273983, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.8692352060883712, "python_tests_examples": 1.7108534788956131, "python_documentation": 2.8258234414766767, "cs_math": 2.9569605807637815, "general_education": 4.015485374227873, "books": 4.213441275225843}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.8692352060883712, "python_tests_examples": 1.7108534788956131, "python_documentation": 2.8258234414766767, "cs_math": 2.9569605807637815, "general_education": 4.015485374227873, "books": 4.213441275225843}, "weighted_loss": 2.106794665708993}
+{"step": 13500, "tokens": 832517627, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.8662839820860846, "python_tests_examples": 1.689093073228711, "python_documentation": 2.8314096961157955, "cs_math": 2.9604265273527512, "general_education": 3.9957214023842593, "books": 4.114930017596196}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.8662839820860846, "python_tests_examples": 1.689093073228711, "python_documentation": 2.8314096961157955, "cs_math": 2.9604265273527512, "general_education": 3.9957214023842593, "books": 4.114930017596196}, "weighted_loss": 2.0997810877429304}
+{"step": 14000, "tokens": 861240875, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.885992129443007, "python_tests_examples": 1.7053687545452991, "python_documentation": 2.8060647995242287, "cs_math": 2.8955932201883803, "general_education": 3.969068211240147, "books": 4.138763419048485}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.885992129443007, "python_tests_examples": 1.7053687545452991, "python_documentation": 2.8060647995242287, "cs_math": 2.8955932201883803, "general_education": 3.969068211240147, "books": 4.138763419048485}, "weighted_loss": 2.1089336529997658}
+{"step": 14500, "tokens": 889582677, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.849676747210904, "python_tests_examples": 1.691045555238832, "python_documentation": 2.8008440348602424, "cs_math": 2.8846094038056576, "general_education": 3.9555553621423947, "books": 4.090755672342319}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.849676747210904, "python_tests_examples": 1.691045555238832, "python_documentation": 2.8008440348602424, "cs_math": 2.8846094038056576, "general_education": 3.9555553621423947, "books": 4.090755672342319}, "weighted_loss": 2.0807433669603377}
+{"step": 15000, "tokens": 918819917, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.857062785865969, "python_tests_examples": 1.6778109293374126, "python_documentation": 2.790859091843084, "cs_math": 3.0896510901316425, "general_education": 3.9526452198589577, "books": 4.128502029130586}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.857062785865969, "python_tests_examples": 1.6778109293374126, "python_documentation": 2.790859091843084, "cs_math": 3.0896510901316425, "general_education": 3.9526452198589577, "books": 4.128502029130586}, "weighted_loss": 2.0934803110827627}
+{"step": 15500, "tokens": 947077614, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.8594222856914986, "python_tests_examples": 1.6748200916824856, "python_documentation": 2.7956311417561435, "cs_math": 2.985422776822322, "general_education": 3.950244911336941, "books": 4.0800848224163495}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.8594222856914986, "python_tests_examples": 1.6748200916824856, "python_documentation": 2.7956311417561435, "cs_math": 2.985422776822322, "general_education": 3.950244911336941, "books": 4.0800848224163495}, "weighted_loss": 2.0887907962570127}
+{"step": 16000, "tokens": 974882707, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.854393777780219, "python_tests_examples": 1.6702474014735955, "python_documentation": 2.8046355190348904, "cs_math": 2.890126670573612, "general_education": 3.955185241842711, "books": 4.10421407147964}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.854393777780219, "python_tests_examples": 1.6702474014735955, "python_documentation": 2.8046355190348904, "cs_math": 2.890126670573612, "general_education": 3.955185241842711, "books": 4.10421407147964}, "weighted_loss": 2.0816027898952254}
+{"step": 16446, "tokens": 1000070440, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.825571772312484, "python_tests_examples": 1.6630119510485815, "python_documentation": 2.7780784043286477, "cs_math": 2.880713280455128, "general_education": 3.928751808829358, "books": 4.058927748896857}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.825571772312484, "python_tests_examples": 1.6630119510485815, "python_documentation": 2.7780784043286477, "cs_math": 2.880713280455128, "general_education": 3.928751808829358, "books": 4.058927748896857}, "weighted_loss": 2.0569580583588407}