python-124m-muon-5b-20261021-v1: 2250050419 training targets

README.md

@@ -11,7 +11,7 @@ tags:
 # Simple LLM Python 124M / 5B
 
 Run: `python-124m-muon-5b-20261021-v1`. This is an **intermediate** base-model checkpoint, freshly initialized.
-Processed **2,000,044,496 nonpadding training targets** in **33,868 optimizer updates**.
+Processed **2,250,050,419 nonpadding training targets** in **38,226 optimizer updates**.
 The exact target is 5,000,000,000. This model is not instruction-tuned.
 
 124,389,632 parameters; 14 layers, width 768, 12 attention heads, RMSNorm,

metrics.jsonl

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e6cd519d5606148c3d1b6ffe880a83dd48f4f625fef03fa5460df8d9656fe2ed
-size 54705200
+oid sha256:1add255d7dabbf68fcbcc5a4f0d225e35615ea7871fb24b73ad0123e91dba9cf
+size 61751279

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ee7e5b6464d223c25099ca4bcbb92272022cc8e39500f187ee6a436cf4d2c563
+oid sha256:b8f082de3af9df6b5b66c8d2216328282bffef67cbb7c48f8ecbad0fc13812f6
 size 598238584

provenance.json

@@ -47,22 +47,22 @@
     "tokenizer": "00e32adc13dab74f5a029c63304b4591bd48f7cd6b086b22d4624d244667b4da"
   },
   "torch_version": "2.6.0+cu124",
-  "gpu": "NVIDIA H200",
+  "gpu": "NVIDIA H100 80GB HBM3",
   "parameters": 124389632,
-  "resume_from": "/runs/python-124m-muon-5b-20261021-v1-s009/step-000029477.pt",
-  "step": 33868,
-  "tokens": 2000044496,
+  "resume_from": "/runs/python-124m-muon-5b-20261021-v1-s010/step-000033868.pt",
+  "step": 38226,
+  "tokens": 2250050419,
   "run_id": "python-124m-muon-5b-20261021-v1",
-  "checkpoint": "/runs/python-124m-muon-5b-20261021-v1-s010/step-000033868.pt",
-  "weights_sha256": "ee7e5b6464d223c25099ca4bcbb92272022cc8e39500f187ee6a436cf4d2c563",
+  "checkpoint": "/runs/python-124m-muon-5b-20261021-v1-s011/step-000038226.pt",
+  "weights_sha256": "b8f082de3af9df6b5b66c8d2216328282bffef67cbb7c48f8ecbad0fc13812f6",
   "weight_roundtrip_exact": true,
   "cpu_logits_roundtrip_exact": true,
   "loss_token_exposures": {
-    "python_implementation": 1300025113,
-    "python_tests_examples": 300009224,
-    "python_documentation": 200003637,
-    "cs_math": 100002345,
-    "general_education": 60002388,
-    "books": 40001789
+    "python_implementation": 1462523246,
+    "python_tests_examples": 337505307,
+    "python_documentation": 225003533,
+    "cs_math": 112507475,
+    "general_education": 67502535,
+    "books": 45008323
   }
 }

runtime-cost-estimate.json

@@ -1,11 +1,11 @@
 {
   "kind": "GPU compute estimate, not measured invoice",
-  "training_worker_reported_seconds": 13010.55108537,
+  "training_worker_reported_seconds": 14531.632408324,
   "h100_usd_per_second": 0.001097,
   "excludes": [
     "GPU work outside recorded training segments",
     "CPU preparation and evaluation",
     "storage and transfer"
   ],
-  "training_gpu_usd_estimate": 114.18059632520713
+  "training_gpu_usd_estimate": 127.52960601545144
 }

validation.jsonl

@@ -87,3 +87,14 @@
 {"step": 33000, "tokens": 1950659059, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.759009503710785, "python_tests_examples": 1.5768835676583983, "python_documentation": 2.6973614729414646, "cs_math": 2.7902642995630775, "general_education": 3.836614855989556, "books": 3.9957197493656986}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.759009503710785, "python_tests_examples": 1.5768835676583983, "python_documentation": 2.6973614729414646, "cs_math": 2.7902642995630775, "general_education": 3.836614855989556, "books": 3.9957197493656986}, "weighted_loss": 1.984150915500071}
 {"step": 33500, "tokens": 1978433425, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.7402662197187007, "python_tests_examples": 1.5568571703293375, "python_documentation": 2.6798317536078136, "cs_math": 2.8720228350778405, "general_education": 3.801153109742646, "books": 3.9190878620001133}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.7402662197187007, "python_tests_examples": 1.5568571703293375, "python_documentation": 2.6798317536078136, "cs_math": 2.8720228350778405, "general_education": 3.801153109742646, "books": 3.9190878620001133}, "weighted_loss": 1.9687022860135113}
 {"step": 33868, "tokens": 2000044496, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.7381127430775816, "python_tests_examples": 1.5702197333558823, "python_documentation": 2.6799776620561504, "cs_math": 2.7968400148889274, "general_education": 3.8338019965578227, "books": 3.9896411339274924}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.7381127430775816, "python_tests_examples": 1.5702197333558823, "python_documentation": 2.6799776620561504, "cs_math": 2.7968400148889274, "general_education": 3.8338019965578227, "books": 3.9896411339274924}, "weighted_loss": 1.967952892529156}
+{"step": 33868, "tokens": 2000044496, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.7381127430775816, "python_tests_examples": 1.5702197333558823, "python_documentation": 2.6799776620561504, "cs_math": 2.7968400148889274, "general_education": 3.8338019965578227, "books": 3.9896411339274924}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.7381127430775816, "python_tests_examples": 1.5702197333558823, "python_documentation": 2.6799776620561504, "cs_math": 2.7968400148889274, "general_education": 3.8338019965578227, "books": 3.9896411339274924}, "weighted_loss": 1.967952892529156}
+{"step": 34000, "tokens": 2007497643, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.7218440710249343, "python_tests_examples": 1.568370602255054, "python_documentation": 2.6677751610638363, "cs_math": 2.7504215881845053, "general_education": 3.815885616067522, "books": 3.9240893280451337}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.7218440710249343, "python_tests_examples": 1.568370602255054, "python_documentation": 2.6677751610638363, "cs_math": 2.7504215881845053, "general_education": 3.815885616067522, "books": 3.9240893280451337}, "weighted_loss": 1.9517111870630026}
+{"step": 34500, "tokens": 2036166375, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.7350403056086743, "python_tests_examples": 1.5604548168354715, "python_documentation": 2.6711974555880436, "cs_math": 2.7210872658295613, "general_education": 3.8200174930246265, "books": 3.9224512595620373}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.7350403056086743, "python_tests_examples": 1.5604548168354715, "python_documentation": 2.6711974555880436, "cs_math": 2.7210872658295613, "general_education": 3.8200174930246265, "books": 3.9224512595620373}, "weighted_loss": 1.958068080003221}
+{"step": 35000, "tokens": 2064344121, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.7245661001236468, "python_tests_examples": 1.5618139542400573, "python_documentation": 2.6664266093870075, "cs_math": 2.759829652998887, "general_education": 3.8043715160759097, "books": 3.952070162636266}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.7245661001236468, "python_tests_examples": 1.5618139542400573, "python_documentation": 2.6664266093870075, "cs_math": 2.759829652998887, "general_education": 3.8043715160759097, "books": 3.952070162636266}, "weighted_loss": 1.9530467505400269}
+{"step": 35500, "tokens": 2092624173, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.7204966908623232, "python_tests_examples": 1.536562869951058, "python_documentation": 2.651238069885758, "cs_math": 2.760038909320543, "general_education": 3.799250490768446, "books": 3.940742592771743}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.7204966908623232, "python_tests_examples": 1.536562869951058, "python_documentation": 2.651238069885758, "cs_math": 2.760038909320543, "general_education": 3.799250490768446, "books": 3.940742592771743}, "weighted_loss": 1.9447253985862598}
+{"step": 36000, "tokens": 2121190811, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.7225749261752683, "python_tests_examples": 1.5399761998988302, "python_documentation": 2.6484602335034366, "cs_math": 2.7072314807098965, "general_education": 3.797050368056565, "books": 3.9519888455691996}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.7225749261752683, "python_tests_examples": 1.5399761998988302, "python_documentation": 2.6484602335034366, "cs_math": 2.7072314807098965, "general_education": 3.797050368056565, "books": 3.9519888455691996}, "weighted_loss": 1.9438290173376684}
+{"step": 36500, "tokens": 2149557693, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.7250041968255712, "python_tests_examples": 1.549382829736641, "python_documentation": 2.6688400077246692, "cs_math": 2.732723910650674, "general_education": 3.7874556086503204, "books": 3.99883148931898}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.7250041968255712, "python_tests_examples": 1.549382829736641, "python_documentation": 2.6688400077246692, "cs_math": 2.732723910650674, "general_education": 3.7874556086503204, "books": 3.99883148931898}, "weighted_loss": 1.950780646748007}
+{"step": 37000, "tokens": 2178437154, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.7086719983858065, "python_tests_examples": 1.5385944321959677, "python_documentation": 2.647395186575279, "cs_math": 2.7108559115361786, "general_education": 3.774505922945183, "books": 3.906747592524407}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.7086719983858065, "python_tests_examples": 1.5385944321959677, "python_documentation": 2.647395186575279, "cs_math": 2.7108559115361786, "general_education": 3.774505922945183, "books": 3.906747592524407}, "weighted_loss": 1.93307840755335}
+{"step": 37500, "tokens": 2207296022, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.718658994094128, "python_tests_examples": 1.5534102917892585, "python_documentation": 2.6596528264321972, "cs_math": 2.7355120834156215, "general_education": 3.7837945164747793, "books": 3.9525511810541536}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.718658994094128, "python_tests_examples": 1.5534102917892585, "python_documentation": 2.6596528264321972, "cs_math": 2.7355120834156215, "general_education": 3.7837945164747793, "books": 3.9525511810541536}, "weighted_loss": 1.9454456358588992}
+{"step": 38000, "tokens": 2237107539, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.7060511874648254, "python_tests_examples": 1.497887448604929, "python_documentation": 2.6335174030365485, "cs_math": 2.74082284404164, "general_education": 3.7670152071382885, "books": 3.9221956192942127}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.7060511874648254, "python_tests_examples": 1.497887448604929, "python_documentation": 2.6335174030365485, "cs_math": 2.74082284404164, "general_education": 3.7670152071382885, "books": 3.9221956192942127}, "weighted_loss": 1.9254636402486456}
+{"step": 38226, "tokens": 2250050419, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.7127098837603656, "python_tests_examples": 1.5213255602600189, "python_documentation": 2.6389322857640507, "cs_math": 2.7310883885499884, "general_education": 3.7891177747461477, "books": 3.9279855452469072}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.7127098837603656, "python_tests_examples": 1.5213255602600189, "python_documentation": 2.6389322857640507, "cs_math": 2.7310883885499884, "general_education": 3.7891177747461477, "books": 3.9279855452469072}, "weighted_loss": 1.9341411506344675}