python-124m-muon-5b-20261021-v1: 4000050828 training targets

README.md

@@ -11,7 +11,7 @@ tags:
 # Simple LLM Python 124M / 5B
 
 Run: `python-124m-muon-5b-20261021-v1`. This is an **intermediate** base-model checkpoint, freshly initialized.
-Processed **3,750,066,616 nonpadding training targets** in **64,314 optimizer updates**.
+Processed **4,000,050,828 nonpadding training targets** in **68,695 optimizer updates**.
 The exact target is 5,000,000,000. This model is not instruction-tuned.
 
 124,389,632 parameters; 14 layers, width 768, 12 attention heads, RMSNorm,

metrics.jsonl

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:fa276fc4d7ecd87aac6204da24d76ea578083a3dc63e877f89c8acbc06074705
-size 103927559
+oid sha256:ae5382bf286a446fa472e7c4cf6b9d914623e4797f2c42e203a3bc84cffd3326
+size 111005287

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:cd7a50d995f9a94a7964d8bdbd2c231acdad8959fb7d4b699697d0b0c3aa089f
+oid sha256:598fac790da185d17ed86ca1e0dd669cbafadc452e2cf6259b23a7e1837d9057
 size 598238584

provenance.json

@@ -49,20 +49,20 @@
   "torch_version": "2.6.0+cu124",
   "gpu": "NVIDIA H100 80GB HBM3",
   "parameters": 124389632,
-  "resume_from": "/runs/python-124m-muon-5b-20261021-v1-s016/step-000059962.pt",
-  "step": 64314,
-  "tokens": 3750066616,
+  "resume_from": "/runs/python-124m-muon-5b-20261021-v1-s017/step-000064314.pt",
+  "step": 68695,
+  "tokens": 4000050828,
   "run_id": "python-124m-muon-5b-20261021-v1",
-  "checkpoint": "/runs/python-124m-muon-5b-20261021-v1-s017/step-000064314.pt",
-  "weights_sha256": "cd7a50d995f9a94a7964d8bdbd2c231acdad8959fb7d4b699697d0b0c3aa089f",
+  "checkpoint": "/runs/python-124m-muon-5b-20261021-v1-s018/step-000068695.pt",
+  "weights_sha256": "598fac790da185d17ed86ca1e0dd669cbafadc452e2cf6259b23a7e1837d9057",
   "weight_roundtrip_exact": true,
   "cpu_logits_roundtrip_exact": true,
   "loss_token_exposures": {
-    "python_implementation": 2437535103,
-    "python_tests_examples": 562507869,
-    "python_documentation": 375005667,
-    "cs_math": 187507695,
-    "general_education": 112501827,
-    "books": 75008455
+    "python_implementation": 2600025355,
+    "python_tests_examples": 600009749,
+    "python_documentation": 400003840,
+    "cs_math": 200002118,
+    "general_education": 120003954,
+    "books": 80005812
   }
 }

runtime-cost-estimate.json

@@ -1,11 +1,11 @@
 {
   "kind": "GPU compute estimate, not measured invoice",
-  "training_worker_reported_seconds": 22879.924163712,
+  "training_worker_reported_seconds": 24398.398226481,
   "h100_usd_per_second": 0.001097,
   "excludes": [
     "GPU work outside recorded training segments",
     "CPU preparation and evaluation",
     "storage and transfer"
   ],
-  "training_gpu_usd_estimate": 200.79421446073653
+  "training_gpu_usd_estimate": 214.12034283559728
 }

validation.jsonl

@@ -163,3 +163,14 @@
 {"step": 63500, "tokens": 3702737597, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.516607835221889, "python_tests_examples": 1.3611787423449861, "python_documentation": 2.427677897506438, "cs_math": 2.4860427546174817, "general_education": 3.5177311704709457, "books": 3.6650031919147663}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.516607835221889, "python_tests_examples": 1.3611787423449861, "python_documentation": 2.427677897506438, "cs_math": 2.4860427546174817, "general_education": 3.5177311704709457, "books": 3.6650031919147663}, "weighted_loss": 1.7358738306799173}
 {"step": 64000, "tokens": 3731599719, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.5109427931446224, "python_tests_examples": 1.3646128265771917, "python_documentation": 2.4148546415911403, "cs_math": 2.4706570600933246, "general_education": 3.5207767645011443, "books": 3.6565846233290804}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.5109427931446224, "python_tests_examples": 1.3646128265771917, "python_documentation": 2.4148546415911403, "cs_math": 2.4706570600933246, "general_education": 3.5207767645011443, "books": 3.6565846233290804}, "weighted_loss": 1.7305780520959795}
 {"step": 64314, "tokens": 3750066616, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.496936217922308, "python_tests_examples": 1.3603376824416726, "python_documentation": 2.403399152475429, "cs_math": 2.4564876530741824, "general_education": 3.5112581308356, "books": 3.671319605612263}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.496936217922308, "python_tests_examples": 1.3603376824416726, "python_documentation": 2.403399152475429, "cs_math": 2.4564876530741824, "general_education": 3.5112581308356, "books": 3.671319605612263}, "weighted_loss": 1.7189876279543164}
+{"step": 64314, "tokens": 3750066616, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.496936217922308, "python_tests_examples": 1.3603376824416726, "python_documentation": 2.403399152475429, "cs_math": 2.4564876530741824, "general_education": 3.5112581308356, "books": 3.671319605612263}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.496936217922308, "python_tests_examples": 1.3603376824416726, "python_documentation": 2.403399152475429, "cs_math": 2.4564876530741824, "general_education": 3.5112581308356, "books": 3.671319605612263}, "weighted_loss": 1.7189876279543164}
+{"step": 64500, "tokens": 3760345368, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.500103266566246, "python_tests_examples": 1.356005694491338, "python_documentation": 2.4172506988865847, "cs_math": 2.490685122574665, "general_education": 3.5005094319835774, "books": 3.679229798795144}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.500103266566246, "python_tests_examples": 1.356005694491338, "python_documentation": 2.4172506988865847, "cs_math": 2.490685122574665, "general_education": 3.5005094319835774, "books": 3.679229798795144}, "weighted_loss": 1.7233271823945624}
+{"step": 65000, "tokens": 3788909641, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.4957971950865634, "python_tests_examples": 1.342331277926567, "python_documentation": 2.4152636570341373, "cs_math": 2.4848459097241022, "general_education": 3.513104520044097, "books": 3.6883881901103095}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.4957971950865634, "python_tests_examples": 1.342331277926567, "python_documentation": 2.4152636570341373, "cs_math": 2.4848459097241022, "general_education": 3.513104520044097, "books": 3.6883881901103095}, "weighted_loss": 1.718547429088399}
+{"step": 65500, "tokens": 3817391531, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.4954207322238189, "python_tests_examples": 1.3409164852357291, "python_documentation": 2.406550116396212, "cs_math": 2.4544539138838615, "general_education": 3.4996535814043237, "books": 3.677618219183874}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.4954207322238189, "python_tests_examples": 1.3409164852357291, "python_documentation": 2.406550116396212, "cs_math": 2.4544539138838615, "general_education": 3.4996535814043237, "books": 3.677618219183874}, "weighted_loss": 1.7150806278904631}
+{"step": 66000, "tokens": 3846632520, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.493805734968299, "python_tests_examples": 1.321275860213168, "python_documentation": 2.3991306378582586, "cs_math": 2.440924076370371, "general_education": 3.4929930103665314, "books": 3.6386969284392143}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.493805734968299, "python_tests_examples": 1.321275860213168, "python_documentation": 2.3991306378582586, "cs_math": 2.440924076370371, "general_education": 3.4929930103665314, "books": 3.6386969284392143}, "weighted_loss": 1.7086881032454941}
+{"step": 66500, "tokens": 3875469461, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.5029609389430505, "python_tests_examples": 1.3333023071915757, "python_documentation": 2.3931957283477208, "cs_math": 2.4556076891277585, "general_education": 3.484122709024476, "books": 3.637914827079183}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.5029609389430505, "python_tests_examples": 1.3333023071915757, "python_documentation": 2.3931957283477208, "cs_math": 2.4556076891277585, "general_education": 3.484122709024476, "books": 3.637914827079183}, "weighted_loss": 1.716301891495197}
+{"step": 67000, "tokens": 3903409466, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.490468331221429, "python_tests_examples": 1.3359449031573516, "python_documentation": 2.391926861985238, "cs_math": 2.4553309815699036, "general_education": 3.482346779213611, "books": 3.6425159064209423}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.490468331221429, "python_tests_examples": 1.3359449031573516, "python_documentation": 2.391926861985238, "cs_math": 2.4553309815699036, "general_education": 3.482346779213611, "books": 3.6425159064209423}, "weighted_loss": 1.708476107549378}
+{"step": 67500, "tokens": 3931458537, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.475126932094841, "python_tests_examples": 1.33692925172583, "python_documentation": 2.3880407356892257, "cs_math": 2.434466655594477, "general_education": 3.4896869606610768, "books": 3.637430412359201}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.475126932094841, "python_tests_examples": 1.33692925172583, "python_documentation": 2.3880407356892257, "cs_math": 2.434466655594477, "general_education": 3.4896869606610768, "books": 3.637430412359201}, "weighted_loss": 1.6973385170361839}
+{"step": 68000, "tokens": 3960163873, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.491658087374827, "python_tests_examples": 1.3296636027108473, "python_documentation": 2.3974830933903113, "cs_math": 2.427966778466422, "general_education": 3.476407078937841, "books": 3.64512588462247}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.491658087374827, "python_tests_examples": 1.3296636027108473, "python_documentation": 2.3974830933903113, "cs_math": 2.427966778466422, "general_education": 3.476407078937841, "books": 3.64512588462247}, "weighted_loss": 1.7073686755232016}
+{"step": 68500, "tokens": 3989329448, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.4878267151674192, "python_tests_examples": 1.3381985103004923, "python_documentation": 2.3813420782779957, "cs_math": 2.4293124916263187, "general_education": 3.4613178178039776, "books": 3.5946787210921385}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.4878267151674192, "python_tests_examples": 1.3381985103004923, "python_documentation": 2.3813420782779957, "cs_math": 2.4293124916263187, "general_education": 3.4613178178039776, "books": 3.5946787210921385}, "weighted_loss": 1.7031500827689738}
+{"step": 68695, "tokens": 4000050828, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.4843846831903258, "python_tests_examples": 1.3291989281511627, "python_documentation": 2.372109309818256, "cs_math": 2.408725010838461, "general_education": 3.469720559747707, "books": 3.5792315186490185}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.4843846831903258, "python_tests_examples": 1.3291989281511627, "python_documentation": 2.372109309818256, "cs_math": 2.408725010838461, "general_education": 3.469720559747707, "books": 3.5792315186490185}, "weighted_loss": 1.6975533119855464}