python-124m-muon-5b-20261021-v1: 750036273 training targets

README.md

@@ -11,7 +11,7 @@ tags:
 # Simple LLM Python 124M / 5B
 
 Run: `python-124m-muon-5b-20261021-v1`. This is an **intermediate** base-model checkpoint, freshly initialized.
-Processed **500,044,156 nonpadding training targets** in **7,739 optimizer updates**.
+Processed **750,036,273 nonpadding training targets** in **12,072 optimizer updates**.
 The exact target is 5,000,000,000. This model is not instruction-tuned.
 
 124,389,632 parameters; 14 layers, width 768, 12 attention heads, RMSNorm,

metrics.jsonl

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:321b1ea178fa5880721a9ddf774532ede32f5b90a5ea8e7e9e38d7072fa9c840
-size 12478127
+oid sha256:62e3ce38dd6fecc79cf8165c53fe22f2ddc6634c93d7dc9be7a2cd92b71a0564
+size 19475495

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c364510402aed86abcfb3624b17a0494e93089ad9b549c4ca785ac61fb58b83c
+oid sha256:a5719e7c548a8ffef180c6813e37458f582a18c1eb94493d5e036932dea37da6
 size 598238584

provenance.json

@@ -47,22 +47,22 @@
     "tokenizer": "00e32adc13dab74f5a029c63304b4591bd48f7cd6b086b22d4624d244667b4da"
   },
   "torch_version": "2.6.0+cu124",
-  "gpu": "NVIDIA H100 80GB HBM3",
+  "gpu": "NVIDIA H200",
   "parameters": 124389632,
-  "resume_from": "/runs/python-124m-muon-5b-20261021-v1-s003/step-000003492.pt",
-  "step": 7739,
-  "tokens": 500044156,
+  "resume_from": "/runs/python-124m-muon-5b-20261021-v1-s004/step-000007739.pt",
+  "step": 12072,
+  "tokens": 750036273,
   "run_id": "python-124m-muon-5b-20261021-v1",
-  "checkpoint": "/runs/python-124m-muon-5b-20261021-v1-s004/step-000007739.pt",
-  "weights_sha256": "c364510402aed86abcfb3624b17a0494e93089ad9b549c4ca785ac61fb58b83c",
+  "checkpoint": "/runs/python-124m-muon-5b-20261021-v1-s005-retry-001/step-000012072.pt",
+  "weights_sha256": "a5719e7c548a8ffef180c6813e37458f582a18c1eb94493d5e036932dea37da6",
   "weight_roundtrip_exact": true,
   "cpu_logits_roundtrip_exact": true,
   "loss_token_exposures": {
-    "python_implementation": 325019540,
-    "python_tests_examples": 75010988,
-    "python_documentation": 50004745,
-    "cs_math": 25005168,
-    "general_education": 15001035,
-    "books": 10002680
+    "python_implementation": 487521404,
+    "python_tests_examples": 112505163,
+    "python_documentation": 75003002,
+    "cs_math": 37501721,
+    "general_education": 22501446,
+    "books": 15003537
   }
 }

runtime-cost-estimate.json

@@ -1,11 +1,11 @@
 {
   "kind": "GPU compute estimate, not measured invoice",
-  "training_worker_reported_seconds": 4307.891061891,
+  "training_worker_reported_seconds": 5735.615135963,
   "h100_usd_per_second": 0.001097,
   "excludes": [
     "GPU work outside recorded training segments",
     "CPU preparation and evaluation",
     "storage and transfer"
   ],
-  "training_gpu_usd_estimate": 37.806051959155425
+  "training_gpu_usd_estimate": 50.335758433211296
 }

validation.jsonl

@@ -23,3 +23,14 @@
 {"step": 7000, "tokens": 456372695, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.9531801376163918, "python_tests_examples": 1.7893383780664776, "python_documentation": 2.9155548204269097, "cs_math": 3.125105384520615, "general_education": 4.056566197248992, "books": 4.171409649500405}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.9531801376163918, "python_tests_examples": 1.7893383780664776, "python_documentation": 2.9155548204269097, "cs_math": 3.125105384520615, "general_education": 4.056566197248992, "books": 4.171409649500405}, "weighted_loss": 2.1909037763368255}
 {"step": 7500, "tokens": 486193519, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.949939646881978, "python_tests_examples": 1.7539362380201835, "python_documentation": 2.8834913661762975, "cs_math": 3.0106634796019436, "general_education": 4.041560746654249, "books": 4.2517702006261135}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.949939646881978, "python_tests_examples": 1.7539362380201835, "python_documentation": 2.8834913661762975, "cs_math": 3.0106634796019436, "general_education": 4.041560746654249, "books": 4.2517702006261135}, "weighted_loss": 2.1757157431861898}
 {"step": 7739, "tokens": 500044156, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.9559222801322604, "python_tests_examples": 1.776924798225013, "python_documentation": 2.89655862147543, "cs_math": 2.983571789368241, "general_education": 4.077060617154236, "books": 4.229515550403194}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.9559222801322604, "python_tests_examples": 1.776924798225013, "python_documentation": 2.89655862147543, "cs_math": 2.983571789368241, "general_education": 4.077060617154236, "books": 4.229515550403194}, "weighted_loss": 2.183624782958367}
+{"step": 7739, "tokens": 500044156, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.9558636641098135, "python_tests_examples": 1.7768634434539108, "python_documentation": 2.8966163870040993, "cs_math": 2.9835976359870426, "general_education": 4.077043272149668, "books": 4.229472609265283}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.9558636641098135, "python_tests_examples": 1.7768634434539108, "python_documentation": 2.8966163870040993, "cs_math": 2.9835976359870426, "general_education": 4.077043272149668, "books": 4.229472609265283}, "weighted_loss": 2.1835831690390233}
+{"step": 8000, "tokens": 515147420, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.9344671249632284, "python_tests_examples": 1.7571344301973597, "python_documentation": 2.9059551179592247, "cs_math": 2.964384997567286, "general_education": 4.037741806805928, "books": 4.095613195337264}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.9344671249632284, "python_tests_examples": 1.7571344301973597, "python_documentation": 2.9059551179592247, "cs_math": 2.964384997567286, "general_education": 4.037741806805928, "books": 4.095613195337264}, "weighted_loss": 2.162833075540912}
+{"step": 8500, "tokens": 543920440, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.9421330646644737, "python_tests_examples": 1.7933335696728492, "python_documentation": 2.8981683660129134, "cs_math": 2.957613935575463, "general_education": 4.0425241716565345, "books": 4.218511676303839}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.9421330646644737, "python_tests_examples": 1.7933335696728492, "python_documentation": 2.8981683660129134, "cs_math": 2.957613935575463, "general_education": 4.0425241716565345, "books": 4.218511676303839}, "weighted_loss": 2.1747300195386727}
+{"step": 9000, "tokens": 572472555, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.9091652514493596, "python_tests_examples": 1.7298035158671552, "python_documentation": 2.8780407452177346, "cs_math": 2.9812023622512966, "general_education": 4.039666276436217, "books": 4.229009608001554}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.9091652514493596, "python_tests_examples": 1.7298035158671552, "python_documentation": 2.8780407452177346, "cs_math": 2.9812023622512966, "general_education": 4.039666276436217, "books": 4.229009608001554}, "weighted_loss": 2.143062313909613}
+{"step": 9500, "tokens": 601512600, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.911113693467738, "python_tests_examples": 1.7240659858825191, "python_documentation": 2.8625171036727357, "cs_math": 2.948460134413718, "general_education": 4.015230230948537, "books": 4.135078575961906}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.911113693467738, "python_tests_examples": 1.7240659858825191, "python_documentation": 2.8625171036727357, "cs_math": 2.948460134413718, "general_education": 4.015230230948537, "books": 4.135078575961906}, "weighted_loss": 2.1376669941720614}
+{"step": 10000, "tokens": 630023633, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.8984249795128982, "python_tests_examples": 1.7340796050599236, "python_documentation": 2.881032922447183, "cs_math": 3.0059936798696927, "general_education": 4.0039803525022775, "books": 4.143642290771952}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.8984249795128982, "python_tests_examples": 1.7340796050599236, "python_documentation": 2.881032922447183, "cs_math": 3.0059936798696927, "general_education": 4.0039803525022775, "books": 4.143642290771952}, "weighted_loss": 2.1354834100710827}
+{"step": 10500, "tokens": 658977162, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.912675448150471, "python_tests_examples": 1.7332242361250154, "python_documentation": 2.8558312064521867, "cs_math": 2.997984610834551, "general_education": 4.0155387500954065, "books": 4.166696393194188}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.912675448150471, "python_tests_examples": 1.7332242361250154, "python_documentation": 2.8558312064521867, "cs_math": 2.997984610834551, "general_education": 4.0155387500954065, "books": 4.166696393194188}, "weighted_loss": 2.1425051182702504}
+{"step": 11000, "tokens": 687715468, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.8900111572758103, "python_tests_examples": 1.707760685669567, "python_documentation": 2.844471527332552, "cs_math": 2.9288557650064098, "general_education": 4.0120199653996576, "books": 4.130792971963541}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.8900111572758103, "python_tests_examples": 1.707760685669567, "python_documentation": 2.844471527332552, "cs_math": 2.9288557650064098, "general_education": 4.0120199653996576, "books": 4.130792971963541}, "weighted_loss": 2.118537754464548}
+{"step": 11500, "tokens": 716867733, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.9013674905849665, "python_tests_examples": 1.7188571261247234, "python_documentation": 2.8495332417955663, "cs_math": 2.980339026751044, "general_education": 3.9957867598410317, "books": 4.128864759524754}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.9013674905849665, "python_tests_examples": 1.7188571261247234, "python_documentation": 2.8495332417955663, "cs_math": 2.980339026751044, "general_education": 3.9957867598410317, "books": 4.128864759524754}, "weighted_loss": 2.1301386113017715}
+{"step": 12000, "tokens": 745807584, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.9118772415864633, "python_tests_examples": 1.7364030514951017, "python_documentation": 2.831054179453419, "cs_math": 2.927207559844683, "general_education": 3.984384398731678, "books": 4.080954188179626}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.9118772415864633, "python_tests_examples": 1.7364030514951017, "python_documentation": 2.831054179453419, "cs_math": 2.927207559844683, "general_education": 3.984384398731678, "books": 4.080954188179626}, "weighted_loss": 2.133797076418585}
+{"step": 12072, "tokens": 750036273, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.8791453879206281, "python_tests_examples": 1.7069534428977058, "python_documentation": 2.8507127404455876, "cs_math": 2.9244520415402597, "general_education": 4.020535449075895, "books": 4.112124923362367}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.8791453879206281, "python_tests_examples": 1.7069534428977058, "python_documentation": 2.8507127404455876, "cs_math": 2.9244520415402597, "general_education": 4.020535449075895, "books": 4.112124923362367}, "weighted_loss": 2.11163995664416}