python-124m-muon-5b-20261021-v1: 250051061 training targets

README.md

@@ -11,7 +11,7 @@ tags:
 # Simple LLM Python 124M / 5B
 
 Run: `python-124m-muon-5b-20261021-v1`. This is an **intermediate** base-model checkpoint, freshly initialized.
-Processed **50,065,393 nonpadding training targets** in **549 optimizer updates**.
+Processed **250,051,061 nonpadding training targets** in **3,492 optimizer updates**.
 The exact target is 5,000,000,000. This model is not instruction-tuned.
 
 124,389,632 parameters; 14 layers, width 768, 12 attention heads, RMSNorm,

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8e5e55c0beaac47b09427c261d7dc7a9b94240758b6e0f3724d419ed43d8c1f5
+oid sha256:eeb38d5c42e571001fa732d6c420bb9db632c7f0e2d3670877090eb4de9ddbec
 size 598238584

provenance.json

@@ -49,20 +49,20 @@
   "torch_version": "2.6.0+cu124",
   "gpu": "NVIDIA H100 80GB HBM3",
   "parameters": 124389632,
-  "resume_from": "/runs/python-124m-muon-5b-20261021-v1-s001/step-000000004.pt",
-  "step": 549,
-  "tokens": 50065393,
+  "resume_from": "/runs/python-124m-muon-5b-20261021-v1-s002/step-000000549.pt",
+  "step": 3492,
+  "tokens": 250051061,
   "run_id": "python-124m-muon-5b-20261021-v1",
-  "checkpoint": "/runs/python-124m-muon-5b-20261021-v1-s002/step-000000549.pt",
-  "weights_sha256": "8e5e55c0beaac47b09427c261d7dc7a9b94240758b6e0f3724d419ed43d8c1f5",
+  "checkpoint": "/runs/python-124m-muon-5b-20261021-v1-s003/step-000003492.pt",
+  "weights_sha256": "eeb38d5c42e571001fa732d6c420bb9db632c7f0e2d3670877090eb4de9ddbec",
   "weight_roundtrip_exact": true,
   "cpu_logits_roundtrip_exact": true,
   "loss_token_exposures": {
-    "python_implementation": 32533334,
-    "python_tests_examples": 7508444,
-    "python_documentation": 5009437,
-    "cs_math": 2506747,
-    "general_education": 1502082,
-    "books": 1005349
+    "python_implementation": 162530375,
+    "python_tests_examples": 37508199,
+    "python_documentation": 25004987,
+    "cs_math": 12502399,
+    "general_education": 7501644,
+    "books": 5003457
   }
 }

runtime-cost-estimate.json

@@ -1,11 +1,11 @@
 {
   "kind": "GPU compute estimate, not measured invoice",
-  "training_worker_reported_seconds": 842.245771547,
+  "training_worker_reported_seconds": 2091.3677939050003,
   "h100_usd_per_second": 0.001097,
   "excludes": [
     "GPU work outside recorded training segments",
     "CPU preparation and evaluation",
     "storage and transfer"
   ],
-  "training_gpu_usd_estimate": 7.391548891096472
+  "training_gpu_usd_estimate": 18.353843759310283
 }

validation.jsonl

@@ -5,3 +5,10 @@
 {"step": 4, "tokens": 387014, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 10.479941665000817, "python_tests_examples": 10.448600116025958, "python_documentation": 10.512487911442683, "cs_math": 10.469178979745234, "general_education": 10.556407058147432, "books": 10.516637554925667}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 10.479941665000817, "python_tests_examples": 10.448600116025958, "python_documentation": 10.512487911442683, "cs_math": 10.469178979745234, "general_education": 10.556407058147432, "books": 10.516637554925667}, "weighted_loss": 10.480984802628893}
 {"step": 500, "tokens": 45650831, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 4.162418328667418, "python_tests_examples": 3.918841802409977, "python_documentation": 5.080653659561132, "cs_math": 4.339001730203496, "general_education": 5.349691602343097, "books": 5.309149448988822}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 4.162418328667418, "python_tests_examples": 3.918841802409977, "python_documentation": 5.080653659561132, "cs_math": 4.339001730203496, "general_education": 5.349691602343097, "books": 5.309149448988822}, "weighted_loss": 4.285087373511676}
 {"step": 549, "tokens": 50065393, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 3.89459791981912, "python_tests_examples": 3.5985489956984402, "python_documentation": 4.833967628435524, "cs_math": 4.094523666119857, "general_education": 5.163108580551279, "books": 5.183435151830238}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 3.89459791981912, "python_tests_examples": 3.5985489956984402, "python_documentation": 4.833967628435524, "cs_math": 4.094523666119857, "general_education": 5.163108580551279, "books": 5.183435151830238}, "weighted_loss": 4.017955903839883}
+{"step": 549, "tokens": 50065393, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 3.89459791981912, "python_tests_examples": 3.5985489956984402, "python_documentation": 4.833967628435524, "cs_math": 4.094523666119857, "general_education": 5.163108580551279, "books": 5.183435151830238}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 3.89459791981912, "python_tests_examples": 3.5985489956984402, "python_documentation": 4.833967628435524, "cs_math": 4.094523666119857, "general_education": 5.163108580551279, "books": 5.183435151830238}, "weighted_loss": 4.017955903839883}
+{"step": 1000, "tokens": 83577218, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 2.7479512429582686, "python_tests_examples": 2.5572079607902616, "python_documentation": 4.031852252330285, "cs_math": 3.6009033007545934, "general_education": 4.664010817339089, "books": 4.6956359025866234}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 2.7479512429582686, "python_tests_examples": 2.5572079607902616, "python_documentation": 4.031852252330285, "cs_math": 3.6009033007545934, "general_education": 4.664010817339089, "books": 4.6956359025866234}, "weighted_loss": 2.986812934884077}
+{"step": 1500, "tokens": 117507250, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 2.469883143736097, "python_tests_examples": 2.2714130986945196, "python_documentation": 3.74673141802555, "cs_math": 3.380394172079909, "general_education": 4.423343837613079, "books": 4.50946524264035}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 2.469883143736097, "python_tests_examples": 2.2714130986945196, "python_documentation": 3.74673141802555, "cs_math": 3.380394172079909, "general_education": 4.423343837613079, "books": 4.50946524264035}, "weighted_loss": 2.7127184786203906}
+{"step": 2000, "tokens": 150859926, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 2.2304049274603264, "python_tests_examples": 2.0706033977710967, "python_documentation": 3.5632948293894415, "cs_math": 3.2540867296386833, "general_education": 4.292114957735012, "books": 4.391384172513383}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 2.2304049274603264, "python_tests_examples": 2.0706033977710967, "python_documentation": 3.5632948293894415, "cs_math": 3.2540867296386833, "general_education": 4.292114957735012, "books": 4.391384172513383}, "weighted_loss": 2.495978664118073}
+{"step": 2500, "tokens": 184107656, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 2.1924578516790976, "python_tests_examples": 2.022210039841007, "python_documentation": 3.43803907747578, "cs_math": 3.147088807277527, "general_education": 4.217252623405235, "books": 4.347502121851156}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 2.1924578516790976, "python_tests_examples": 2.022210039841007, "python_documentation": 3.43803907747578, "cs_math": 3.147088807277527, "general_education": 4.217252623405235, "books": 4.347502121851156}, "weighted_loss": 2.4430550788181993}
+{"step": 3000, "tokens": 217806656, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 2.068694597510086, "python_tests_examples": 1.918312337532877, "python_documentation": 3.380592492439255, "cs_math": 3.08924052137883, "general_education": 4.137618646272441, "books": 4.255268273750112}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 2.068694597510086, "python_tests_examples": 1.918312337532877, "python_documentation": 3.380592492439255, "cs_math": 3.08924052137883, "general_education": 4.137618646272441, "books": 4.255268273750112}, "weighted_loss": 2.33415353918753}
+{"step": 3492, "tokens": 250051061, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 2.0556095562875134, "python_tests_examples": 1.9033838904744125, "python_documentation": 3.356063952920466, "cs_math": 3.2277762661045117, "general_education": 4.149975508770645, "books": 4.269131061287444}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 2.0556095562875134, "python_tests_examples": 1.9033838904744125, "python_documentation": 3.356063952920466, "cs_math": 3.2277762661045117, "general_education": 4.149975508770645, "books": 4.269131061287444}, "weighted_loss": 2.328530890244186}