Qwen90M step 285000; complete resumable checkpoint

complete.json

@@ -1,18 +1,18 @@
 {
-  "step": 280248,
+  "step": 285000,
   "files": {
     "config.json": "cbc72137da3dae65085a2a8c39ad7634d16bf92d785ca00d283ee82fe32b37e7",
     "generation_config.json": "54ed0d19f6226888c91157964ede8a892bd5ae175bfad85336152b3d9b6cd7c4",
-    "model.safetensors": "ab3d534f71afc536f11eacb5cac49b41552e56c521fecc7b301c5b0f3ba211e7",
+    "model.safetensors": "8ac057f2166a1e4db0ce4a9415d89891b9b8ddcc52297129f86c08f788b2758b",
     "tokenization_byte.py": "279b9990f77b1eaedc7c1184fe13d60273a713ead856200f1fb9ae323c7b1e03",
     "tokenizer_config.json": "0ecbd4fd39e35125800d806af981ba0cacd8a7ba5f02c9e729c4ecc54af6d300",
     "special_tokens_map.json": "20d4b94d0e657c3f44d9123522c106e7315b6c1a1b6d259478f17bd2918b1263",
     "byte_vocab.json": "57a1cd2e70dd5b7cc6fd2eaa12ebfcd173c1620fa5d45c2c6282511c29b6a5f6",
-    "trainer.pt": "93080e3d08484f8e03c44bc9ba9acf6621eb9dc2cb2663dd5f9becbb62b890ee",
-    "training_config.json": "95ba9e002790d11959d1191b36ed994bfa10a6da51c22c772e143bec517a9783",
-    "performance.json": "7ab839f3b8798329051e29b39e08bd08f8678f49ed79d09e3b70b9de78a4765c",
-    "shard_scheduler.json": "1a27a42e8cdc3d5af3957968177c27079c751995153e489c49bae7322e8344e1",
-    "coverage.json": "c8401a996db8071c49afdc637c6460b684b49b8ba1e5678dd9298a5b147168b1",
+    "trainer.pt": "f77cbd7043301ba03a27d0fc3fb559b972721c8fb60167219ee0788286252c59",
+    "training_config.json": "2122ad6f8105987f9b87dd55d13b632084853908566b081c829ffe801259e2bf",
+    "performance.json": "f89f82c3f4c44a52fcd532346ea71580902d757e59e1961e9b2bed0030259dba",
+    "shard_scheduler.json": "5316864976580fabdbfc5b4749a2a377f1a44f787f1e27a21ee310ec16436feb",
+    "coverage.json": "e35c09a84496ab9947c7e21ef4a25e40c3c17be24bb510256b35b90dd258081c",
     "README.md": "9ce0f9efacbc46177a7e818b6aed274d84e7145ff944b5c7d6ecd5d33b9d4419"
   },
   "signature": "9fee47918905b0750bee87876aa424f021ccc4507ee8db22a11d01fbc6109063"

coverage.json

@@ -1,107 +1,107 @@
 {
-  "step": 280248,
+  "step": 285000,
   "stats": {
-    "climbmix/blocks": 10088928,
-    "climbmix/targets": 16865599592,
-    "climbmix/input_tokens": 16866668333,
-    "ultra_qa/blocks": 3783348,
-    "ultra_qa/targets": 6172241217,
-    "ultra_qa/input_tokens": 6172253060,
-    "cortex/blocks": 3783348,
-    "cortex/targets": 143908822,
-    "cortex/input_tokens": 6013958320,
-    "ultra_style/blocks": 3783348,
-    "ultra_style/targets": 5984022238,
-    "ultra_style/input_tokens": 5984336893,
-    "openmath/blocks": 2522232,
-    "openmath/targets": 3897349251,
-    "openmath/input_tokens": 3898648602,
-    "rewrite6/blocks": 1261116,
-    "rewrite6/targets": 2067382557,
-    "rewrite6/input_tokens": 2069649291
+    "climbmix/blocks": 10260000,
+    "climbmix/targets": 17151447815,
+    "climbmix/input_tokens": 17152534864,
+    "ultra_qa/blocks": 3847500,
+    "ultra_qa/targets": 6276806703,
+    "ultra_qa/input_tokens": 6276818734,
+    "cortex/blocks": 3847500,
+    "cortex/targets": 146351608,
+    "cortex/input_tokens": 6115948834,
+    "ultra_style/blocks": 3847500,
+    "ultra_style/targets": 6085396148,
+    "ultra_style/input_tokens": 6085715990,
+    "openmath/blocks": 2565000,
+    "openmath/targets": 3963414207,
+    "openmath/input_tokens": 3964735574,
+    "rewrite6/blocks": 1282500,
+    "rewrite6/targets": 2102512928,
+    "rewrite6/input_tokens": 2104817984
   },
   "sources": {
     "climbmix": {
-      "page": 1381,
-      "row": 2390,
+      "page": 1405,
+      "row": 588,
       "start": {
-        "file": 66,
+        "file": 68,
         "reader": {
           "format": "parquet-row-v1",
-          "row_group": 70,
+          "row_group": 1,
           "row": 1024
         }
       },
       "epoch": 0,
-      "offset": 2048,
-      "documents": 5658966
+      "offset": 0,
+      "documents": 5755468
     },
     "rewrite6": {
-      "page": 36,
-      "row": 797,
+      "page": 49,
+      "row": 2128,
       "start": {
         "file": 0,
         "reader": {
           "format": "jsonl-row-v1",
-          "row": 147456
+          "row": 200704
         }
       },
       "epoch": 15,
-      "offset": 0,
-      "documents": 3229313
+      "offset": 2048,
+      "documents": 3283892
     },
     "openmath": {
-      "page": 863,
-      "row": 3211,
+      "page": 878,
+      "row": 1736,
       "start": {
         "file": 8,
         "reader": {
           "format": "parquet-row-v1",
-          "row_group": 41,
-          "row": 648
+          "row_group": 103,
+          "row": 88
         }
       },
       "epoch": 0,
       "offset": 2048,
-      "documents": 3538059
+      "documents": 3598024
     },
     "ultra_qa": {
-      "page": 373,
-      "row": 1465,
+      "page": 379,
+      "row": 2700,
       "start": {
         "file": 2,
         "reader": {
           "format": "parquet-row-v1",
           "row_group": 7,
-          "row": 36665
+          "row": 61241
         }
       },
       "epoch": 0,
       "offset": 2048,
-      "documents": 1529273
+      "documents": 1555084
     },
     "ultra_style": {
-      "page": 570,
-      "row": 1617,
+      "page": 579,
+      "row": 4081,
       "start": {
         "file": 3,
         "reader": {
           "format": "parquet-row-v1",
           "row_group": 3,
-          "row": 22523
+          "row": 59387
         }
       },
       "epoch": 0,
       "offset": 2048,
-      "documents": 2336337
+      "documents": 2375665
     },
     "cortex": {
-      "page": 73374,
-      "row": 54,
+      "page": 74620,
+      "row": 66,
       "start": null,
       "epoch": 0,
       "offset": 0,
-      "documents": 9391926
+      "documents": 9551426
     }
   }
 }

latest.json

@@ -1 +1 @@
-{"step": 280248, "files": {"config.json": "cbc72137da3dae65085a2a8c39ad7634d16bf92d785ca00d283ee82fe32b37e7", "generation_config.json": "54ed0d19f6226888c91157964ede8a892bd5ae175bfad85336152b3d9b6cd7c4", "model.safetensors": "ab3d534f71afc536f11eacb5cac49b41552e56c521fecc7b301c5b0f3ba211e7", "tokenization_byte.py": "279b9990f77b1eaedc7c1184fe13d60273a713ead856200f1fb9ae323c7b1e03", "tokenizer_config.json": "0ecbd4fd39e35125800d806af981ba0cacd8a7ba5f02c9e729c4ecc54af6d300", "special_tokens_map.json": "20d4b94d0e657c3f44d9123522c106e7315b6c1a1b6d259478f17bd2918b1263", "byte_vocab.json": "57a1cd2e70dd5b7cc6fd2eaa12ebfcd173c1620fa5d45c2c6282511c29b6a5f6", "trainer.pt": "93080e3d08484f8e03c44bc9ba9acf6621eb9dc2cb2663dd5f9becbb62b890ee", "training_config.json": "95ba9e002790d11959d1191b36ed994bfa10a6da51c22c772e143bec517a9783", "performance.json": "7ab839f3b8798329051e29b39e08bd08f8678f49ed79d09e3b70b9de78a4765c", "shard_scheduler.json": "1a27a42e8cdc3d5af3957968177c27079c751995153e489c49bae7322e8344e1", "coverage.json": "c8401a996db8071c49afdc637c6460b684b49b8ba1e5678dd9298a5b147168b1", "README.md": "9ce0f9efacbc46177a7e818b6aed274d84e7145ff944b5c7d6ecd5d33b9d4419"}, "signature": "9fee47918905b0750bee87876aa424f021ccc4507ee8db22a11d01fbc6109063", "folder": "step-000000280248"}
+{"step": 285000, "files": {"config.json": "cbc72137da3dae65085a2a8c39ad7634d16bf92d785ca00d283ee82fe32b37e7", "generation_config.json": "54ed0d19f6226888c91157964ede8a892bd5ae175bfad85336152b3d9b6cd7c4", "model.safetensors": "8ac057f2166a1e4db0ce4a9415d89891b9b8ddcc52297129f86c08f788b2758b", "tokenization_byte.py": "279b9990f77b1eaedc7c1184fe13d60273a713ead856200f1fb9ae323c7b1e03", "tokenizer_config.json": "0ecbd4fd39e35125800d806af981ba0cacd8a7ba5f02c9e729c4ecc54af6d300", "special_tokens_map.json": "20d4b94d0e657c3f44d9123522c106e7315b6c1a1b6d259478f17bd2918b1263", "byte_vocab.json": "57a1cd2e70dd5b7cc6fd2eaa12ebfcd173c1620fa5d45c2c6282511c29b6a5f6", "trainer.pt": "f77cbd7043301ba03a27d0fc3fb559b972721c8fb60167219ee0788286252c59", "training_config.json": "2122ad6f8105987f9b87dd55d13b632084853908566b081c829ffe801259e2bf", "performance.json": "f89f82c3f4c44a52fcd532346ea71580902d757e59e1961e9b2bed0030259dba", "shard_scheduler.json": "5316864976580fabdbfc5b4749a2a377f1a44f787f1e27a21ee310ec16436feb", "coverage.json": "e35c09a84496ab9947c7e21ef4a25e40c3c17be24bb510256b35b90dd258081c", "README.md": "9ce0f9efacbc46177a7e818b6aed274d84e7145ff944b5c7d6ecd5d33b9d4419"}, "signature": "9fee47918905b0750bee87876aa424f021ccc4507ee8db22a11d01fbc6109063", "folder": "step-000000285000"}

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ab3d534f71afc536f11eacb5cac49b41552e56c521fecc7b301c5b0f3ba211e7
+oid sha256:8ac057f2166a1e4db0ce4a9415d89891b9b8ddcc52297129f86c08f788b2758b
 size 360622744

performance.json

@@ -49,11 +49,11 @@
   "selected": {
     "microbatch": 86,
     "mode": "compiled",
-    "microbatch_seconds": 0.8973423759999832,
-    "peak_gb": 85.39913082122803,
+    "microbatch_seconds": 0.8979115860001912,
+    "peak_gb": 85.39889430999756,
     "fits_budget": true,
-    "estimated_update_seconds": 0.9528963840000415,
-    "input_tokens_per_second": 196277.36827175462
+    "estimated_update_seconds": 0.953845084999557,
+    "input_tokens_per_second": 196152.9428354681
   },
   "measurements": [
     {

shard_scheduler.json

@@ -22,8 +22,8 @@
       "tokens_per_second": 1062967.6168812835,
       "note": "Bounded fresh sample; full-page latency is an extrapolation.",
       "estimated_pages_for_buffer": 1,
-      "required_token_positions_per_second": 75609.08177184862,
-      "required_pages_per_second_upper_estimate": 0.006553603578518049
+      "required_token_positions_per_second": 74771.75244308176,
+      "required_pages_per_second_upper_estimate": 0.00648102599449232
     },
     {
       "source": "rewrite6",
@@ -36,8 +36,8 @@
       "tokens_per_second": 1206672.263377472,
       "note": "Bounded fresh sample; full-page latency is an extrapolation.",
       "estimated_pages_for_buffer": 1,
-      "required_token_positions_per_second": 9451.135221481078,
-      "required_pages_per_second_upper_estimate": 0.0037701991469128284
+      "required_token_positions_per_second": 9346.46905538522,
+      "required_pages_per_second_upper_estimate": 0.0037284462483585528
     },
     {
       "source": "openmath",
@@ -50,8 +50,8 @@
       "tokens_per_second": 410042.7581134458,
       "note": "Bounded fresh sample; full-page latency is an extrapolation.",
       "estimated_pages_for_buffer": 1,
-      "required_token_positions_per_second": 18902.270442962155,
-      "required_pages_per_second_upper_estimate": 0.004372707598381542
+      "required_token_positions_per_second": 18692.93811077044,
+      "required_pages_per_second_upper_estimate": 0.004324282247452207
     },
     {
       "source": "ultra_qa",
@@ -64,8 +64,8 @@
       "tokens_per_second": 2163486.5624068812,
       "note": "Bounded fresh sample; full-page latency is an extrapolation.",
       "estimated_pages_for_buffer": 1,
-      "required_token_positions_per_second": 28353.405664443235,
-      "required_pages_per_second_upper_estimate": 0.0017265736461417594
+      "required_token_positions_per_second": 28039.40716615566,
+      "required_pages_per_second_upper_estimate": 0.001707452785018847
     },
     {
       "source": "ultra_style",
@@ -78,8 +78,8 @@
       "tokens_per_second": 1546694.2737633225,
       "note": "Bounded fresh sample; full-page latency is an extrapolation.",
       "estimated_pages_for_buffer": 1,
-      "required_token_positions_per_second": 28353.405664443235,
-      "required_pages_per_second_upper_estimate": 0.0021827236552991267
+      "required_token_positions_per_second": 28039.40716615566,
+      "required_pages_per_second_upper_estimate": 0.0021585511816974746
     },
     {
       "source": "cortex",
@@ -91,9 +91,9 @@
       "read_seconds": 0.042780299000014566,
       "tokens_per_second": 1015177.8395898147,
       "note": "Bounded fresh sample; full-page latency is an extrapolation.",
-      "estimated_pages_for_buffer": 22,
-      "required_token_positions_per_second": 28353.405664443235,
-      "required_pages_per_second_upper_estimate": 0.3520544056078974
+      "estimated_pages_for_buffer": 21,
+      "required_token_positions_per_second": 28039.40716615566,
+      "required_pages_per_second_upper_estimate": 0.34815559514453803
     }
   ],
   "gpu_measurement": "Selected forward/backward estimate; replaced by live successful update timings.",
@@ -155,59 +155,263 @@
   "shard_benchmark_scope": "Local packing/read measurements; upload capacity measured only on real data commits. Targets larger than available pages are sample-limited.",
   "estimated_fresh_generation_batch_seconds": 0.1434574083060457,
   "selected": {
-    "prefetch_batches": 62,
+    "prefetch_batches": 61,
     "prefetch_capacity_limit": 512,
     "batch_memory_estimate_bytes": 690670,
-    "producer_batch_seconds": 0.07287284228125657,
-    "producer_p95_seconds": 0.1761082670000178,
-    "gpu_update_seconds": 0.9751209546820736,
-    "producer_batches_per_second": 13.722533233168638,
-    "gpu_batches_per_second": 1.0255138044141794,
-    "producer_headroom": 13.381129706983886,
+    "producer_batch_seconds": 0.0779331768125644,
+    "producer_p95_seconds": 0.2094176940008765,
+    "gpu_update_seconds": 0.9860408187720853,
+    "producer_batches_per_second": 12.831505668055609,
+    "gpu_batches_per_second": 1.014156798544403,
+    "producer_headroom": 12.652388355008206,
     "sustainable": true,
-    "buffer_seconds": 60.457499190288566,
+    "buffer_seconds": 60.148489945097204,
     "buffer_drain_seconds": null,
     "shard_target_mib": 16,
-    "upload_pages_per_commit": 79,
+    "upload_pages_per_commit": 83,
     "max_pending_pages": 4096,
-    "generation_bytes_per_second": 382834.16564839584,
-    "upload_bytes_per_second": 4501863.534696111,
+    "generation_bytes_per_second": 313527.5392295587,
+    "upload_bytes_per_second": 5440988.382144982,
     "upload_sustainable": true,
     "logical_page_rows": 4096,
     "logical_cortex_page_rows": 128,
-    "estimated_full_shards_per_hour": 82.14730002488047,
+    "estimated_full_shards_per_hour": 67.2757113710887,
     "selection_policy": "Smallest shard within 90% of measured local peak, enlarged to amortize 30 seconds of measured page production; bounded to 256 MiB."
   },
   "upload_measurements": [
     {
-      "seconds": 0.8291290090000984,
-      "bytes": 27860,
+      "seconds": 0.8854921419988386,
+      "bytes": 43628,
       "pages": 1,
       "shards": 1
     },
     {
-      "seconds": 0.7566531879992908,
-      "bytes": 935968,
+      "seconds": 1.4819931290003296,
+      "bytes": 7791997,
+      "pages": 45,
+      "shards": 3
+    },
+    {
+      "seconds": 0.999563274999673,
+      "bytes": 6224997,
+      "pages": 32,
+      "shards": 2
+    },
+    {
+      "seconds": 1.0198160460004146,
+      "bytes": 6464312,
+      "pages": 37,
+      "shards": 3
+    },
+    {
+      "seconds": 1.407603981999273,
+      "bytes": 11506972,
+      "pages": 28,
+      "shards": 3
+    },
+    {
+      "seconds": 1.4075674270006857,
+      "bytes": 6094729,
+      "pages": 35,
+      "shards": 2
+    },
+    {
+      "seconds": 1.2288260940003966,
+      "bytes": 2371624,
+      "pages": 28,
+      "shards": 2
+    },
+    {
+      "seconds": 1.1107631630002288,
+      "bytes": 10173347,
+      "pages": 40,
+      "shards": 3
+    },
+    {
+      "seconds": 0.9085161899984087,
+      "bytes": 2572379,
+      "pages": 35,
+      "shards": 2
+    },
+    {
+      "seconds": 1.1832448179993662,
+      "bytes": 6068567,
+      "pages": 35,
+      "shards": 2
+    },
+    {
+      "seconds": 1.2676941070003522,
+      "bytes": 16626023,
+      "pages": 38,
+      "shards": 4
+    },
+    {
+      "seconds": 0.9997899369991501,
+      "bytes": 2248595,
       "pages": 27,
+      "shards": 2
+    },
+    {
+      "seconds": 0.9953625339985592,
+      "bytes": 6050082,
+      "pages": 36,
+      "shards": 2
+    },
+    {
+      "seconds": 1.2188318490007077,
+      "bytes": 7592648,
+      "pages": 38,
+      "shards": 3
+    },
+    {
+      "seconds": 0.8659798669996235,
+      "bytes": 5051955,
+      "pages": 32,
+      "shards": 2
+    },
+    {
+      "seconds": 1.0979891799997858,
+      "bytes": 6035254,
+      "pages": 36,
+      "shards": 2
+    },
+    {
+      "seconds": 1.2733040839993919,
+      "bytes": 13322526,
+      "pages": 35,
+      "shards": 4
+    },
+    {
+      "seconds": 0.8405508110008668,
+      "bytes": 1090490,
+      "pages": 33,
       "shards": 1
     },
     {
-      "seconds": 1.5188891550005792,
-      "bytes": 5974366,
-      "pages": 29,
+      "seconds": 1.1666756240010727,
+      "bytes": 10146533,
+      "pages": 27,
+      "shards": 3
+    },
+    {
+      "seconds": 1.1112560749988916,
+      "bytes": 2663601,
+      "pages": 36,
+      "shards": 3
+    },
+    {
+      "seconds": 1.182102421998934,
+      "bytes": 5962190,
+      "pages": 36,
+      "shards": 2
+    },
+    {
+      "seconds": 0.9607173089989374,
+      "bytes": 7750938,
+      "pages": 37,
+      "shards": 3
+    },
+    {
+      "seconds": 1.1050905230003991,
+      "bytes": 6187776,
+      "pages": 15,
+      "shards": 2
+    },
+    {
+      "seconds": 1.2981188600006135,
+      "bytes": 10611352,
+      "pages": 45,
+      "shards": 3
+    },
+    {
+      "seconds": 1.1970796210007393,
+      "bytes": 2312155,
+      "pages": 27,
       "shards": 2
     },
     {
-      "seconds": 1.1771950970005491,
-      "bytes": 7732243,
-      "pages": 17,
+      "seconds": 1.6818293220003397,
+      "bytes": 5841398,
+      "pages": 35,
+      "shards": 2
+    },
+    {
+      "seconds": 1.1767334710002615,
+      "bytes": 1036234,
+      "pages": 30,
+      "shards": 1
+    },
+    {
+      "seconds": 1.7388112589997036,
+      "bytes": 12464285,
+      "pages": 43,
+      "shards": 4
+    },
+    {
+      "seconds": 0.8487006530012877,
+      "bytes": 592565,
+      "pages": 19,
+      "shards": 1
+    },
+    {
+      "seconds": 1.3723258429999987,
+      "bytes": 11921886,
+      "pages": 40,
       "shards": 3
     },
     {
-      "seconds": 1.3709215139997468,
-      "bytes": 10777643,
-      "pages": 39,
+      "seconds": 1.34920726099881,
+      "bytes": 7816018,
+      "pages": 35,
       "shards": 3
+    },
+    {
+      "seconds": 1.4124847809998755,
+      "bytes": 4874448,
+      "pages": 37,
+      "shards": 2
+    },
+    {
+      "seconds": 1.2419382850002876,
+      "bytes": 7186580,
+      "pages": 29,
+      "shards": 3
+    },
+    {
+      "seconds": 1.1265525479993812,
+      "bytes": 6298017,
+      "pages": 35,
+      "shards": 2
+    },
+    {
+      "seconds": 0.9624441889991431,
+      "bytes": 1027701,
+      "pages": 30,
+      "shards": 1
+    },
+    {
+      "seconds": 1.7364580140001635,
+      "bytes": 17104134,
+      "pages": 27,
+      "shards": 5
+    },
+    {
+      "seconds": 0.9077872060006484,
+      "bytes": 916446,
+      "pages": 29,
+      "shards": 1
+    },
+    {
+      "seconds": 1.0004914799992548,
+      "bytes": 6063816,
+      "pages": 36,
+      "shards": 2
+    },
+    {
+      "seconds": 0.9483974380000291,
+      "bytes": 2643403,
+      "pages": 36,
+      "shards": 2
     }
   ],
   "cache_key": {
@@ -252,7 +456,7 @@
   },
   "cache_reused": true,
   "resume_probe_seconds": [
-    3.215140269000017,
-    0.014588057999958437
+    1.1575575899996693,
+    0.014994120000665134
   ]
 }

trainer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:93080e3d08484f8e03c44bc9ba9acf6621eb9dc2cb2663dd5f9becbb62b890ee
-size 721763455
+oid sha256:f77cbd7043301ba03a27d0fc3fb559b972721c8fb60167219ee0788286252c59
+size 721776383

training_config.json

@@ -46,8 +46,8 @@
   "cpu_threads": 4,
   "page_rows": 4096,
   "cortex_page_rows": 128,
-  "prefetch_batches": 62,
-  "prefetch_low_water": 31,
+  "prefetch_batches": 61,
+  "prefetch_low_water": 30,
   "memory_pages": 12,
   "cache_gb": 20,
   "force_rebenchmark": false,