python-124m-muon-5b-20261021-v1: 2000044496 training targets

README.md

@@ -11,7 +11,7 @@ tags:
 # Simple LLM Python 124M / 5B
 
 Run: `python-124m-muon-5b-20261021-v1`. This is an **intermediate** base-model checkpoint, freshly initialized.
-Processed **1,750,010,347 nonpadding training targets** in **29,477 optimizer updates**.
+Processed **2,000,044,496 nonpadding training targets** in **33,868 optimizer updates**.
 The exact target is 5,000,000,000. This model is not instruction-tuned.
 
 124,389,632 parameters; 14 layers, width 768, 12 attention heads, RMSNorm,

metrics.jsonl

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c0dd656f8d3d03c4148bdb72d54553a4f436c8d5fb2b40662bd246df1d621dc1
-size 47605051
+oid sha256:e6cd519d5606148c3d1b6ffe880a83dd48f4f625fef03fa5460df8d9656fe2ed
+size 54705200

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0791d9601f6e02040b31d09b17080852e80eabe9a211da90eec1c612b61cd076
+oid sha256:ee7e5b6464d223c25099ca4bcbb92272022cc8e39500f187ee6a436cf4d2c563
 size 598238584

provenance.json

@@ -47,22 +47,22 @@
     "tokenizer": "00e32adc13dab74f5a029c63304b4591bd48f7cd6b086b22d4624d244667b4da"
   },
   "torch_version": "2.6.0+cu124",
-  "gpu": "NVIDIA H100 80GB HBM3",
+  "gpu": "NVIDIA H200",
   "parameters": 124389632,
-  "resume_from": "/runs/python-124m-muon-5b-20261021-v1-s008/step-000025134.pt",
-  "step": 29477,
-  "tokens": 1750010347,
+  "resume_from": "/runs/python-124m-muon-5b-20261021-v1-s009/step-000029477.pt",
+  "step": 33868,
+  "tokens": 2000044496,
   "run_id": "python-124m-muon-5b-20261021-v1",
-  "checkpoint": "/runs/python-124m-muon-5b-20261021-v1-s009/step-000029477.pt",
-  "weights_sha256": "0791d9601f6e02040b31d09b17080852e80eabe9a211da90eec1c612b61cd076",
+  "checkpoint": "/runs/python-124m-muon-5b-20261021-v1-s010/step-000033868.pt",
+  "weights_sha256": "ee7e5b6464d223c25099ca4bcbb92272022cc8e39500f187ee6a436cf4d2c563",
   "weight_roundtrip_exact": true,
   "cpu_logits_roundtrip_exact": true,
   "loss_token_exposures": {
-    "python_implementation": 1137504011,
-    "python_tests_examples": 262504036,
-    "python_documentation": 174999404,
-    "cs_math": 87500573,
-    "general_education": 52501748,
-    "books": 35000575
+    "python_implementation": 1300025113,
+    "python_tests_examples": 300009224,
+    "python_documentation": 200003637,
+    "cs_math": 100002345,
+    "general_education": 60002388,
+    "books": 40001789
   }
 }

runtime-cost-estimate.json

@@ -1,11 +1,11 @@
 {
   "kind": "GPU compute estimate, not measured invoice",
-  "training_worker_reported_seconds": 11565.736927455,
+  "training_worker_reported_seconds": 13010.55108537,
   "h100_usd_per_second": 0.001097,
   "excludes": [
     "GPU work outside recorded training segments",
     "CPU preparation and evaluation",
     "storage and transfer"
   ],
-  "training_gpu_usd_estimate": 101.50090727534509
+  "training_gpu_usd_estimate": 114.18059632520713
 }

validation.jsonl

@@ -76,3 +76,14 @@
 {"step": 28500, "tokens": 1692478231, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.7574180564067199, "python_tests_examples": 1.5896200781995016, "python_documentation": 2.7066559490340487, "cs_math": 2.7939421301146874, "general_education": 3.846267246600883, "books": 3.988833050094938}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.7574180564067199, "python_tests_examples": 1.5896200781995016, "python_documentation": 2.7066559490340487, "cs_math": 2.7939421301146874, "general_education": 3.846267246600883, "books": 3.988833050094938}, "weighted_loss": 1.9862921282033577}
 {"step": 29000, "tokens": 1722016887, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.7802874831173077, "python_tests_examples": 1.5980181085352476, "python_documentation": 2.729425466859814, "cs_math": 2.79540980915752, "general_education": 3.8600788159303407, "books": 4.008424649184839}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.7802874831173077, "python_tests_examples": 1.5980181085352476, "python_documentation": 2.729425466859814, "cs_math": 2.79540980915752, "general_education": 3.8600788159303407, "books": 4.008424649184839}, "weighted_loss": 2.0055734749120013}
 {"step": 29477, "tokens": 1750010347, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.7467950020957397, "python_tests_examples": 1.5897608021121512, "python_documentation": 2.6845406729359866, "cs_math": 2.7847005012754265, "general_education": 3.850954334300287, "books": 4.067085262930438}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.7467950020957397, "python_tests_examples": 1.5897608021121512, "python_documentation": 2.6845406729359866, "cs_math": 2.7847005012754265, "general_education": 3.850954334300287, "books": 4.067085262930438}, "weighted_loss": 1.9784402993240409}
+{"step": 29477, "tokens": 1750010347, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.7467950020957397, "python_tests_examples": 1.5897608021121512, "python_documentation": 2.6845406729359866, "cs_math": 2.7847005012754265, "general_education": 3.850954334300287, "books": 4.067085262930438}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.7467950020957397, "python_tests_examples": 1.5897608021121512, "python_documentation": 2.6845406729359866, "cs_math": 2.7847005012754265, "general_education": 3.850954334300287, "books": 4.067085262930438}, "weighted_loss": 1.9784402993240409}
+{"step": 29500, "tokens": 1751283178, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.7506204723265468, "python_tests_examples": 1.5822468261443003, "python_documentation": 2.695578254739497, "cs_math": 2.793008068636141, "general_education": 3.851033364958564, "books": 4.010159893028972}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.7506204723265468, "python_tests_examples": 1.5822468261443003, "python_documentation": 2.695578254739497, "cs_math": 2.793008068636141, "general_education": 3.851033364958564, "books": 4.010159893028972}, "weighted_loss": 1.9801827586489937}
+{"step": 30000, "tokens": 1780115997, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.7462845485028053, "python_tests_examples": 1.580393427558499, "python_documentation": 2.703077479726032, "cs_math": 2.807583857567328, "general_education": 3.8358382433349263, "books": 4.03439420281388}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.7462845485028053, "python_tests_examples": 1.580393427558499, "python_documentation": 2.703077479726032, "cs_math": 2.807583857567328, "general_education": 3.8358382433349263, "books": 4.03439420281388}, "weighted_loss": 1.9785939428678934}
+{"step": 30500, "tokens": 1808072206, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.7594301102195073, "python_tests_examples": 1.5712926581670874, "python_documentation": 2.7051107980869586, "cs_math": 2.7716356662738475, "general_education": 3.82137829544564, "books": 3.9732124173129897}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.7594301102195073, "python_tests_examples": 1.5712926581670874, "python_documentation": 2.7051107980869586, "cs_math": 2.7716356662738475, "general_education": 3.82137829544564, "books": 3.9732124173129897}, "weighted_loss": 1.9825219306997601}
+{"step": 31000, "tokens": 1836670641, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.7493851293380516, "python_tests_examples": 1.5767311094938015, "python_documentation": 2.69112109462696, "cs_math": 2.7533468950755875, "general_education": 3.8262904058410285, "books": 4.016261623881992}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.7493851293380516, "python_tests_examples": 1.5767311094938015, "python_documentation": 2.69112109462696, "cs_math": 2.7533468950755875, "general_education": 3.8262904058410285, "books": 4.016261623881992}, "weighted_loss": 1.9755033993631497}
+{"step": 31500, "tokens": 1864848892, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.744497996837419, "python_tests_examples": 1.5722828014781758, "python_documentation": 2.6849569591368985, "cs_math": 2.7510063092309998, "general_education": 3.8496743195184338, "books": 4.010365411188315}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.744497996837419, "python_tests_examples": 1.5722828014781758, "python_documentation": 2.6849569591368985, "cs_math": 2.7510063092309998, "general_education": 3.8496743195184338, "books": 4.010365411188315}, "weighted_loss": 1.9715096673506078}
+{"step": 32000, "tokens": 1893563574, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.7461878901870935, "python_tests_examples": 1.555592669002407, "python_documentation": 2.6764196073667614, "cs_math": 2.745716231694958, "general_education": 3.8375324225152934, "books": 3.9616053416856576}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.7461878901870935, "python_tests_examples": 1.555592669002407, "python_documentation": 2.6764196073667614, "cs_math": 2.745716231694958, "general_education": 3.8375324225152934, "books": 3.9616053416856576}, "weighted_loss": 1.9676468808025678}
+{"step": 32500, "tokens": 1922324316, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.7266952800541027, "python_tests_examples": 1.545292830337968, "python_documentation": 2.678006774398513, "cs_math": 2.81715600838983, "general_education": 3.821716686456245, "books": 3.9415612043792865}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.7266952800541027, "python_tests_examples": 1.545292830337968, "python_documentation": 2.678006774398513, "cs_math": 2.81715600838983, "general_education": 3.821716686456245, "books": 3.9415612043792865}, "weighted_loss": 1.9562870591264778}
+{"step": 33000, "tokens": 1950659059, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.759009503710785, "python_tests_examples": 1.5768835676583983, "python_documentation": 2.6973614729414646, "cs_math": 2.7902642995630775, "general_education": 3.836614855989556, "books": 3.9957197493656986}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.759009503710785, "python_tests_examples": 1.5768835676583983, "python_documentation": 2.6973614729414646, "cs_math": 2.7902642995630775, "general_education": 3.836614855989556, "books": 3.9957197493656986}, "weighted_loss": 1.984150915500071}
+{"step": 33500, "tokens": 1978433425, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.7402662197187007, "python_tests_examples": 1.5568571703293375, "python_documentation": 2.6798317536078136, "cs_math": 2.8720228350778405, "general_education": 3.801153109742646, "books": 3.9190878620001133}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.7402662197187007, "python_tests_examples": 1.5568571703293375, "python_documentation": 2.6798317536078136, "cs_math": 2.8720228350778405, "general_education": 3.801153109742646, "books": 3.9190878620001133}, "weighted_loss": 1.9687022860135113}
+{"step": 33868, "tokens": 2000044496, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.7381127430775816, "python_tests_examples": 1.5702197333558823, "python_documentation": 2.6799776620561504, "cs_math": 2.7968400148889274, "general_education": 3.8338019965578227, "books": 3.9896411339274924}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.7381127430775816, "python_tests_examples": 1.5702197333558823, "python_documentation": 2.6799776620561504, "cs_math": 2.7968400148889274, "general_education": 3.8338019965578227, "books": 3.9896411339274924}, "weighted_loss": 1.967952892529156}