python-124m-muon-5b-20261021-v1: 3750066616 training targets

README.md

@@ -11,7 +11,7 @@ tags:
 # Simple LLM Python 124M / 5B
 
 Run: `python-124m-muon-5b-20261021-v1`. This is an **intermediate** base-model checkpoint, freshly initialized.
-Processed **3,500,002,820 nonpadding training targets** in **59,962 optimizer updates**.
+Processed **3,750,066,616 nonpadding training targets** in **64,314 optimizer updates**.
 The exact target is 5,000,000,000. This model is not instruction-tuned.
 
 124,389,632 parameters; 14 layers, width 768, 12 attention heads, RMSNorm,

metrics.jsonl

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:011a6c210020adeff56053a51cb5e69c0aaab398479e9b875440cc57c21fe6ef
-size 96894059
+oid sha256:fa276fc4d7ecd87aac6204da24d76ea578083a3dc63e877f89c8acbc06074705
+size 103927559

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ec6a6d4863359857391cdd3fcecb1128126c18000f640af17487fc4a73f2bd55
+oid sha256:cd7a50d995f9a94a7964d8bdbd2c231acdad8959fb7d4b699697d0b0c3aa089f
 size 598238584

provenance.json

@@ -49,20 +49,20 @@
   "torch_version": "2.6.0+cu124",
   "gpu": "NVIDIA H100 80GB HBM3",
   "parameters": 124389632,
-  "resume_from": "/runs/python-124m-muon-5b-20261021-v1-s015-retry-001/step-000055578.pt",
-  "step": 59962,
-  "tokens": 3500002820,
+  "resume_from": "/runs/python-124m-muon-5b-20261021-v1-s016/step-000059962.pt",
+  "step": 64314,
+  "tokens": 3750066616,
   "run_id": "python-124m-muon-5b-20261021-v1",
-  "checkpoint": "/runs/python-124m-muon-5b-20261021-v1-s016/step-000059962.pt",
-  "weights_sha256": "ec6a6d4863359857391cdd3fcecb1128126c18000f640af17487fc4a73f2bd55",
+  "checkpoint": "/runs/python-124m-muon-5b-20261021-v1-s017/step-000064314.pt",
+  "weights_sha256": "cd7a50d995f9a94a7964d8bdbd2c231acdad8959fb7d4b699697d0b0c3aa089f",
   "weight_roundtrip_exact": true,
   "cpu_logits_roundtrip_exact": true,
   "loss_token_exposures": {
-    "python_implementation": 2274996049,
-    "python_tests_examples": 525000419,
-    "python_documentation": 349999178,
-    "cs_math": 174999629,
-    "general_education": 105001083,
-    "books": 70006462
+    "python_implementation": 2437535103,
+    "python_tests_examples": 562507869,
+    "python_documentation": 375005667,
+    "cs_math": 187507695,
+    "general_education": 112501827,
+    "books": 75008455
   }
 }

runtime-cost-estimate.json

@@ -1,11 +1,11 @@
 {
   "kind": "GPU compute estimate, not measured invoice",
-  "training_worker_reported_seconds": 21568.491258613,
+  "training_worker_reported_seconds": 22879.924163712,
   "h100_usd_per_second": 0.001097,
   "excludes": [
     "GPU work outside recorded training segments",
     "CPU preparation and evaluation",
     "storage and transfer"
   ],
-  "training_gpu_usd_estimate": 189.28507928558773
+  "training_gpu_usd_estimate": 200.79421446073653
 }

validation.jsonl

@@ -152,3 +152,14 @@
 {"step": 59000, "tokens": 3444340752, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.5613102577114222, "python_tests_examples": 1.3965378647389544, "python_documentation": 2.469179269847765, "cs_math": 2.5288061007205345, "general_education": 3.5913385373597784, "books": 3.7487922490842474}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.5613102577114222, "python_tests_examples": 1.3965378647389544, "python_documentation": 2.469179269847765, "cs_math": 2.5288061007205345, "general_education": 3.5913385373597784, "books": 3.7487922490842474}, "weighted_loss": 1.780406580346549}
 {"step": 59500, "tokens": 3472633266, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.565028378602422, "python_tests_examples": 1.3982575599716787, "python_documentation": 2.475228831347491, "cs_math": 2.6625630188350464, "general_education": 3.5657097601378624, "books": 3.6792614658854097}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.565028378602422, "python_tests_examples": 1.3982575599716787, "python_documentation": 2.475228831347491, "cs_math": 2.6625630188350464, "general_education": 3.5657097601378624, "books": 3.6792614658854097}, "weighted_loss": 1.7882146362856715}
 {"step": 59962, "tokens": 3500002820, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.5447765558799367, "python_tests_examples": 1.3814372709276486, "python_documentation": 2.453733376294258, "cs_math": 2.527201465021461, "general_education": 3.5492423770881225, "books": 3.7050187810936785}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.5447765558799367, "python_tests_examples": 1.3814372709276486, "python_documentation": 2.453733376294258, "cs_math": 2.527201465021461, "general_education": 3.5492423770881225, "books": 3.7050187810936785}, "weighted_loss": 1.7636314097761223}
+{"step": 59962, "tokens": 3500002820, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.5447765558799367, "python_tests_examples": 1.3814372709276486, "python_documentation": 2.453733376294258, "cs_math": 2.527201465021461, "general_education": 3.5492423770881225, "books": 3.7050187810936785}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.5447765558799367, "python_tests_examples": 1.3814372709276486, "python_documentation": 2.453733376294258, "cs_math": 2.527201465021461, "general_education": 3.5492423770881225, "books": 3.7050187810936785}, "weighted_loss": 1.7636314097761223}
+{"step": 60000, "tokens": 3502067669, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.5355878438240227, "python_tests_examples": 1.379576797259741, "python_documentation": 2.447834325175176, "cs_math": 2.494671192375936, "general_education": 3.546626747529538, "books": 3.701115341038685}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.5355878438240227, "python_tests_examples": 1.379576797259741, "python_documentation": 2.447834325175176, "cs_math": 2.494671192375936, "general_education": 3.546626747529538, "books": 3.701115341038685}, "weighted_loss": 1.75500671945755}
+{"step": 60500, "tokens": 3531005885, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.5467051199058033, "python_tests_examples": 1.3553301048372812, "python_documentation": 2.4531356626539673, "cs_math": 2.5351288139994588, "general_education": 3.570274790279619, "books": 3.742649602690529}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.5467051199058033, "python_tests_examples": 1.3553301048372812, "python_documentation": 2.4531356626539673, "cs_math": 2.5351288139994588, "general_education": 3.570274790279619, "books": 3.742649602690529}, "weighted_loss": 1.7626890863919331}
+{"step": 61000, "tokens": 3560078100, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.5331599022553701, "python_tests_examples": 1.3671638331468887, "python_documentation": 2.437152580461761, "cs_math": 2.4960589176096186, "general_education": 3.5311243535458283, "books": 3.6655921952246837}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.5331599022553701, "python_tests_examples": 1.3671638331468887, "python_documentation": 2.437152580461761, "cs_math": 2.4960589176096186, "general_education": 3.5311243535458283, "books": 3.6655921952246837}, "weighted_loss": 1.7493922898755494}
+{"step": 61500, "tokens": 3588567431, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.5285036797365525, "python_tests_examples": 1.3777452396488707, "python_documentation": 2.437761310912972, "cs_math": 2.4864490091616767, "general_education": 3.5347110227534317, "books": 3.704191907181714}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.5285036797365525, "python_tests_examples": 1.3777452396488707, "python_documentation": 2.437761310912972, "cs_math": 2.4864490091616767, "general_education": 3.5347110227534317, "books": 3.704191907181714}, "weighted_loss": 1.7484129281517082}
+{"step": 62000, "tokens": 3617519309, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.5372752767227302, "python_tests_examples": 1.375532369298733, "python_documentation": 2.4464931623085797, "cs_math": 2.5157105736891316, "general_education": 3.5418823761947724, "books": 3.682259917080274}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.5372752767227302, "python_tests_examples": 1.375532369298733, "python_documentation": 2.4464931623085797, "cs_math": 2.5157105736891316, "general_education": 3.5418823761947724, "books": 3.682259917080274}, "weighted_loss": 1.7558952998073478}
+{"step": 62500, "tokens": 3645671076, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.5223175790041406, "python_tests_examples": 1.3579598591796513, "python_documentation": 2.4347455548411703, "cs_math": 2.5170421266831196, "general_education": 3.5219737085253024, "books": 3.6564040577098433}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.5223175790041406, "python_tests_examples": 1.3579598591796513, "python_documentation": 2.4347455548411703, "cs_math": 2.5170421266831196, "general_education": 3.5219737085253024, "books": 3.6564040577098433}, "weighted_loss": 1.741314359457868}
+{"step": 63000, "tokens": 3674916203, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.536258862293152, "python_tests_examples": 1.3709512337373946, "python_documentation": 2.436224623895435, "cs_math": 2.5026778553056417, "general_education": 3.5383254656515915, "books": 3.688502990451609}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.536258862293152, "python_tests_examples": 1.3709512337373946, "python_documentation": 2.436224623895435, "cs_math": 2.5026778553056417, "general_education": 3.5383254656515915, "books": 3.688502990451609}, "weighted_loss": 1.7528871244845634}
+{"step": 63500, "tokens": 3702737597, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.516607835221889, "python_tests_examples": 1.3611787423449861, "python_documentation": 2.427677897506438, "cs_math": 2.4860427546174817, "general_education": 3.5177311704709457, "books": 3.6650031919147663}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.516607835221889, "python_tests_examples": 1.3611787423449861, "python_documentation": 2.427677897506438, "cs_math": 2.4860427546174817, "general_education": 3.5177311704709457, "books": 3.6650031919147663}, "weighted_loss": 1.7358738306799173}
+{"step": 64000, "tokens": 3731599719, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.5109427931446224, "python_tests_examples": 1.3646128265771917, "python_documentation": 2.4148546415911403, "cs_math": 2.4706570600933246, "general_education": 3.5207767645011443, "books": 3.6565846233290804}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.5109427931446224, "python_tests_examples": 1.3646128265771917, "python_documentation": 2.4148546415911403, "cs_math": 2.4706570600933246, "general_education": 3.5207767645011443, "books": 3.6565846233290804}, "weighted_loss": 1.7305780520959795}
+{"step": 64314, "tokens": 3750066616, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.496936217922308, "python_tests_examples": 1.3603376824416726, "python_documentation": 2.403399152475429, "cs_math": 2.4564876530741824, "general_education": 3.5112581308356, "books": 3.671319605612263}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.496936217922308, "python_tests_examples": 1.3603376824416726, "python_documentation": 2.403399152475429, "cs_math": 2.4564876530741824, "general_education": 3.5112581308356, "books": 3.671319605612263}, "weighted_loss": 1.7189876279543164}