python-124m-muon-5b-20261021-v1: 3250020404 training targets

README.md

@@ -11,7 +11,7 @@ tags:
 # Simple LLM Python 124M / 5B
 
 Run: `python-124m-muon-5b-20261021-v1`. This is an **intermediate** base-model checkpoint, freshly initialized.
-Processed **3,000,012,443 nonpadding training targets** in **51,214 optimizer updates**.
+Processed **3,250,020,404 nonpadding training targets** in **55,578 optimizer updates**.
 The exact target is 5,000,000,000. This model is not instruction-tuned.
 
 124,389,632 parameters; 14 layers, width 768, 12 attention heads, RMSNorm,

metrics.jsonl

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2ea67a66d22701d8b216e12f4a9a85ca51cff528d10b6ebbe0efec89812a324e
-size 82752200
+oid sha256:4571519e038fe0a4f7747a5ae25c6ff8ae24d298ad5145ac9a7ca819450c101f
+size 89809575

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:00b85d47c4be410669df45899bf95af410f958d77b0987ae00fc5f341cc2cc9b
+oid sha256:4a1dd2c5c5daf33829c336c57c09f67a1b016a6ba8ef445815764dbb305e1d20
 size 598238584

provenance.json

@@ -49,20 +49,20 @@
   "torch_version": "2.6.0+cu124",
   "gpu": "NVIDIA H100 80GB HBM3",
   "parameters": 124389632,
-  "resume_from": "/runs/python-124m-muon-5b-20261021-v1-s013/step-000046880.pt",
-  "step": 51214,
-  "tokens": 3000012443,
+  "resume_from": "/runs/python-124m-muon-5b-20261021-v1-s015/step-000052000.pt",
+  "step": 55578,
+  "tokens": 3250020404,
   "run_id": "python-124m-muon-5b-20261021-v1",
-  "checkpoint": "/runs/python-124m-muon-5b-20261021-v1-s014/step-000051214.pt",
-  "weights_sha256": "00b85d47c4be410669df45899bf95af410f958d77b0987ae00fc5f341cc2cc9b",
+  "checkpoint": "/runs/python-124m-muon-5b-20261021-v1-s015-retry-001/step-000055578.pt",
+  "weights_sha256": "4a1dd2c5c5daf33829c336c57c09f67a1b016a6ba8ef445815764dbb305e1d20",
   "weight_roundtrip_exact": true,
   "cpu_logits_roundtrip_exact": true,
   "loss_token_exposures": {
-    "python_implementation": 1949998511,
-    "python_tests_examples": 450001516,
-    "python_documentation": 299999767,
-    "cs_math": 150000858,
-    "general_education": 90006253,
-    "books": 60005538
+    "python_implementation": 2112506782,
+    "python_tests_examples": 487502611,
+    "python_documentation": 325001565,
+    "cs_math": 162501588,
+    "general_education": 97501257,
+    "books": 65006601
   }
 }

runtime-cost-estimate.json

@@ -1,11 +1,11 @@
 {
   "kind": "GPU compute estimate, not measured invoice",
-  "training_worker_reported_seconds": 18846.089904152002,
+  "training_worker_reported_seconds": 20103.543753661,
   "h100_usd_per_second": 0.001097,
   "excludes": [
     "GPU work outside recorded training segments",
     "CPU preparation and evaluation",
     "storage and transfer"
   ],
-  "training_gpu_usd_estimate": 165.393284998838
+  "training_gpu_usd_estimate": 176.42869998212896
 }

validation.jsonl

@@ -130,3 +130,15 @@
 {"step": 50500, "tokens": 2959496384, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.608132598431615, "python_tests_examples": 1.4316472895963646, "python_documentation": 2.5325119666261378, "cs_math": 2.613458658322964, "general_education": 3.649876971160361, "books": 3.847978563637746}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.608132598431615, "python_tests_examples": 1.4316472895963646, "python_documentation": 2.5325119666261378, "cs_math": 2.613458658322964, "general_education": 3.649876971160361, "books": 3.847978563637746}, "weighted_loss": 1.8304132924063323}
 {"step": 51000, "tokens": 2987950618, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.6319590964781707, "python_tests_examples": 1.4570949555070054, "python_documentation": 2.5565334545764364, "cs_math": 2.5855854818249195, "general_education": 3.657231114387861, "books": 3.7673285006035977}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.6319590964781707, "python_tests_examples": 1.4570949555070054, "python_documentation": 2.5565334545764364, "cs_math": 2.5855854818249195, "general_education": 3.657231114387861, "books": 3.7673285006035977}, "weighted_loss": 1.8493337790294593}
 {"step": 51214, "tokens": 3000012443, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.6138033852584228, "python_tests_examples": 1.4585360756879255, "python_documentation": 2.5455657028268335, "cs_math": 2.604423032794273, "general_education": 3.651095990644877, "books": 3.77585911669828}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.6138033852584228, "python_tests_examples": 1.4585360756879255, "python_documentation": 2.5455657028268335, "cs_math": 2.604423032794273, "general_education": 3.651095990644877, "books": 3.77585911669828}, "weighted_loss": 1.8375803957468726}
+{"step": 51214, "tokens": 3000012443, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.6137010160233307, "python_tests_examples": 1.458501179386325, "python_documentation": 2.5455380427472165, "cs_math": 2.604314327051353, "general_education": 3.651078771594475, "books": 3.775759851012299}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.6137010160233307, "python_tests_examples": 1.458501179386325, "python_documentation": 2.5455380427472165, "cs_math": 2.604314327051353, "general_education": 3.651078771594475, "books": 3.775759851012299}, "weighted_loss": 1.8374979181184832}
+{"step": 51500, "tokens": 3016724592, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.6034513130321784, "python_tests_examples": 1.437988450144515, "python_documentation": 2.5211493151177935, "cs_math": 2.5864445037201937, "general_education": 3.6253119151262547, "books": 3.75844733435062}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.6034513130321784, "python_tests_examples": 1.437988450144515, "python_documentation": 2.5211493151177935, "cs_math": 2.5864445037201937, "general_education": 3.6253119151262547, "books": 3.75844733435062}, "weighted_loss": 1.8233070818311823}
+{"step": 52000, "tokens": 3045336887, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.6043040496561631, "python_tests_examples": 1.4483298594207037, "python_documentation": 2.5372484659269983, "cs_math": 2.6074036782994585, "general_education": 3.65528083850296, "books": 3.781882389926915}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.6043040496561631, "python_tests_examples": 1.4483298594207037, "python_documentation": 2.5372484659269983, "cs_math": 2.6074036782994585, "general_education": 3.65528083850296, "books": 3.781882389926915}, "weighted_loss": 1.8294382146509114}
+{"step": 52000, "tokens": 3045336887, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.6043040496561631, "python_tests_examples": 1.4483298594207037, "python_documentation": 2.5372484659269983, "cs_math": 2.6074036782994585, "general_education": 3.65528083850296, "books": 3.781882389926915}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.6043040496561631, "python_tests_examples": 1.4483298594207037, "python_documentation": 2.5372484659269983, "cs_math": 2.6074036782994585, "general_education": 3.65528083850296, "books": 3.781882389926915}, "weighted_loss": 1.8294382146509114}
+{"step": 52500, "tokens": 3073905333, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.571113025455162, "python_tests_examples": 1.4347262522730506, "python_documentation": 2.5261504118238234, "cs_math": 2.572007841527797, "general_education": 3.642434247079037, "books": 3.810134083417566}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.571113025455162, "python_tests_examples": 1.4347262522730506, "python_documentation": 2.5261504118238234, "cs_math": 2.572007841527797, "general_education": 3.642434247079037, "books": 3.810134083417566}, "weighted_loss": 1.8031235467263076}
+{"step": 53000, "tokens": 3102412542, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.5871403477324377, "python_tests_examples": 1.4292991977226015, "python_documentation": 2.5076811204190306, "cs_math": 2.5543194129521902, "general_education": 3.6306779802757947, "books": 3.7737931255738686}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.5871403477324377, "python_tests_examples": 1.4292991977226015, "python_documentation": 2.5076811204190306, "cs_math": 2.5543194129521902, "general_education": 3.6306779802757947, "books": 3.7737931255738686}, "weighted_loss": 1.8089163902937386}
+{"step": 53500, "tokens": 3130745748, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.5817610052463946, "python_tests_examples": 1.4366600935302856, "python_documentation": 2.509474411013377, "cs_math": 2.574290898152003, "general_education": 3.6422494436205946, "books": 3.807328680775139}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.5817610052463946, "python_tests_examples": 1.4366600935302856, "python_documentation": 2.509474411013377, "cs_math": 2.574290898152003, "general_education": 3.6422494436205946, "books": 3.807328680775139}, "weighted_loss": 1.8087197103727577}
+{"step": 54000, "tokens": 3158815529, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.5928886828548283, "python_tests_examples": 1.4301428364790856, "python_documentation": 2.509200310396645, "cs_math": 2.5626211896030595, "general_education": 3.6161134824577927, "books": 3.7600273687120587}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.5928886828548283, "python_tests_examples": 1.4301428364790856, "python_documentation": 2.509200310396645, "cs_math": 2.5626211896030595, "general_education": 3.6161134824577927, "books": 3.7600273687120587}, "weighted_loss": 1.812634111695294}
+{"step": 54500, "tokens": 3188134001, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.5929771372178658, "python_tests_examples": 1.4297204620050488, "python_documentation": 2.5152511030133966, "cs_math": 2.5944611677424385, "general_education": 3.6225384442723163, "books": 3.7269349890514185}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.5929771372178658, "python_tests_examples": 1.4297204620050488, "python_documentation": 2.5152511030133966, "cs_math": 2.5944611677424385, "general_education": 3.6225384442723163, "books": 3.7269349890514185}, "weighted_loss": 1.8143562302900296}
+{"step": 55000, "tokens": 3217131091, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.5625315793460448, "python_tests_examples": 1.427025033704201, "python_documentation": 2.4945047995350222, "cs_math": 2.551059848155503, "general_education": 3.605662266616817, "books": 3.7320141816782204}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.5625315793460448, "python_tests_examples": 1.427025033704201, "python_documentation": 2.4945047995350222, "cs_math": 2.551059848155503, "general_education": 3.605662266616817, "books": 3.7320141816782204}, "weighted_loss": 1.7895129056239056}
+{"step": 55500, "tokens": 3245727964, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.5580064021150448, "python_tests_examples": 1.4257301893339225, "python_documentation": 2.4808555694992207, "cs_math": 2.5460227793064174, "general_education": 3.5927341956030046, "books": 3.7304924479643873}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.5580064021150448, "python_tests_examples": 1.4257301893339225, "python_documentation": 2.4808555694992207, "cs_math": 2.5460227793064174, "general_education": 3.5927341956030046, "books": 3.7304924479643873}, "weighted_loss": 1.7843422605174883}
+{"step": 55578, "tokens": 3250020404, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.5688506460597964, "python_tests_examples": 1.4291687882825896, "python_documentation": 2.4936180137144413, "cs_math": 2.7126572116453707, "general_education": 3.6291224016963137, "books": 3.7664474305436424}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.5688506460597964, "python_tests_examples": 1.4291687882825896, "python_documentation": 2.4936180137144413, "cs_math": 2.7126572116453707, "general_education": 3.6291224016963137, "books": 3.7664474305436424}, "weighted_loss": 1.8033255207967311}