Qwen90M step 200000; complete resumable checkpoint

complete.json

@@ -1,18 +1,18 @@
 {
-  "step": 195000,
+  "step": 200000,
   "files": {
     "config.json": "cbc72137da3dae65085a2a8c39ad7634d16bf92d785ca00d283ee82fe32b37e7",
     "generation_config.json": "54ed0d19f6226888c91157964ede8a892bd5ae175bfad85336152b3d9b6cd7c4",
-    "model.safetensors": "e7124a2db2f213964f3cd249326d01e111d4002da26c586796a207bae4728dda",
+    "model.safetensors": "9e1098c2f4131285db26132369fd315a230c2a1ed6433490bf04e1d55302bd5c",
     "tokenization_byte.py": "279b9990f77b1eaedc7c1184fe13d60273a713ead856200f1fb9ae323c7b1e03",
     "tokenizer_config.json": "0ecbd4fd39e35125800d806af981ba0cacd8a7ba5f02c9e729c4ecc54af6d300",
     "special_tokens_map.json": "20d4b94d0e657c3f44d9123522c106e7315b6c1a1b6d259478f17bd2918b1263",
     "byte_vocab.json": "57a1cd2e70dd5b7cc6fd2eaa12ebfcd173c1620fa5d45c2c6282511c29b6a5f6",
-    "trainer.pt": "ba31b41d133fff78caf1b22d96d15106f826e2e03648c9dbff5ca60049e9ff2e",
-    "training_config.json": "2122ad6f8105987f9b87dd55d13b632084853908566b081c829ffe801259e2bf",
-    "performance.json": "d8fc0ab4ba9b75a14bc27c39b9cdaf8c3a3b38cccaa4c038b8370fdf2cd49c08",
-    "shard_scheduler.json": "9382ee0711b8a32284204139fa207dcc6bdbed89a4d1e96d772bcf51dbded624",
-    "coverage.json": "a52c19a6ab0dd4e5feeaf4d905f52d95523ae03ff4b83abba914a7c129d58cfb",
+    "trainer.pt": "121ebbd424d06e2eca68e9473ccb7613c34366b9b444edaa47833ef5a5344919",
+    "training_config.json": "95ba9e002790d11959d1191b36ed994bfa10a6da51c22c772e143bec517a9783",
+    "performance.json": "7f72f0a0e868a4baec06f238f922d0c5d7e397a51386c11ac32acc776d4d3522",
+    "shard_scheduler.json": "d840cc690a9276bb980e6c1c2e254aea3ef8a4071771396fd2e073516734a949",
+    "coverage.json": "33255e32c3d706b5219a7812b83fcf6ed8998528c0f3efa9d1b474d0202a80c0",
     "README.md": "9ce0f9efacbc46177a7e818b6aed274d84e7145ff944b5c7d6ecd5d33b9d4419"
   },
   "signature": "9fee47918905b0750bee87876aa424f021ccc4507ee8db22a11d01fbc6109063"

coverage.json

@@ -1,107 +1,107 @@
 {
-  "step": 195000,
+  "step": 200000,
   "stats": {
-    "climbmix/blocks": 7020000,
-    "climbmix/targets": 11732742810,
-    "climbmix/input_tokens": 11733486473,
-    "ultra_qa/blocks": 2632500,
-    "ultra_qa/targets": 4295100300,
-    "ultra_qa/input_tokens": 4295108575,
-    "cortex/blocks": 2632500,
-    "cortex/targets": 100135715,
-    "cortex/input_tokens": 4184383493,
-    "ultra_style/blocks": 2632500,
-    "ultra_style/targets": 4163945713,
-    "ultra_style/input_tokens": 4164164467,
-    "openmath/blocks": 1755000,
-    "openmath/targets": 2711634953,
-    "openmath/input_tokens": 2712538826,
-    "rewrite6/blocks": 877500,
-    "rewrite6/targets": 1438575118,
-    "rewrite6/input_tokens": 1440152299
+    "climbmix/blocks": 7200000,
+    "climbmix/targets": 12033775305,
+    "climbmix/input_tokens": 12034537799,
+    "ultra_qa/blocks": 2700000,
+    "ultra_qa/targets": 4405129990,
+    "ultra_qa/input_tokens": 4405138485,
+    "cortex/blocks": 2700000,
+    "cortex/targets": 102703673,
+    "cortex/input_tokens": 4291696006,
+    "ultra_style/blocks": 2700000,
+    "ultra_style/targets": 4270419549,
+    "ultra_style/input_tokens": 4270644269,
+    "openmath/blocks": 1800000,
+    "openmath/targets": 2781111203,
+    "openmath/input_tokens": 2782038425,
+    "rewrite6/blocks": 900000,
+    "rewrite6/targets": 1475418140,
+    "rewrite6/input_tokens": 1477035705
   },
   "sources": {
     "climbmix": {
-      "page": 961,
-      "row": 3746,
+      "page": 986,
+      "row": 1904,
       "start": {
-        "file": 46,
+        "file": 47,
         "reader": {
           "format": "parquet-row-v1",
-          "row_group": 39,
+          "row_group": 56,
           "row": 1024
         }
       },
       "epoch": 0,
-      "offset": 2048,
-      "documents": 3940002
+      "offset": 4096,
+      "documents": 4040560
     },
     "rewrite6": {
-      "page": 47,
-      "row": 360,
+      "page": 10,
+      "row": 4037,
       "start": {
         "file": 0,
         "reader": {
           "format": "jsonl-row-v1",
-          "row": 192512
+          "row": 40960
         }
       },
-      "epoch": 10,
+      "epoch": 11,
       "offset": 0,
-      "documents": 2246912
+      "documents": 2304441
     },
     "openmath": {
-      "page": 601,
-      "row": 98,
+      "page": 616,
+      "row": 2034,
       "start": {
         "file": 5,
         "reader": {
           "format": "parquet-row-v1",
-          "row_group": 278,
-          "row": 446
+          "row_group": 339,
+          "row": 886
         }
       },
       "epoch": 0,
-      "offset": 2048,
-      "documents": 2461794
+      "offset": 0,
+      "documents": 2525170
     },
     "ultra_qa": {
-      "page": 259,
-      "row": 3323,
+      "page": 266,
+      "row": 1939,
       "start": {
         "file": 2,
         "reader": {
           "format": "parquet-row-v1",
           "row_group": 0,
-          "row": 21613
+          "row": 50285
         }
       },
       "epoch": 0,
       "offset": 2048,
-      "documents": 1064187
+      "documents": 1091475
     },
     "ultra_style": {
-      "page": 396,
-      "row": 2960,
+      "page": 407,
+      "row": 311,
       "start": {
         "file": 2,
         "reader": {
           "format": "parquet-row-v1",
-          "row_group": 2,
-          "row": 81748
+          "row_group": 3,
+          "row": 41668
         }
       },
       "epoch": 0,
       "offset": 2048,
-      "documents": 1624976
+      "documents": 1667383
     },
     "cortex": {
-      "page": 51053,
-      "row": 14,
+      "page": 52363,
+      "row": 94,
       "start": null,
       "epoch": 0,
       "offset": 0,
-      "documents": 6534798
+      "documents": 6702558
     }
   }
 }

latest.json

@@ -1 +1 @@
-{"step": 195000, "files": {"config.json": "cbc72137da3dae65085a2a8c39ad7634d16bf92d785ca00d283ee82fe32b37e7", "generation_config.json": "54ed0d19f6226888c91157964ede8a892bd5ae175bfad85336152b3d9b6cd7c4", "model.safetensors": "e7124a2db2f213964f3cd249326d01e111d4002da26c586796a207bae4728dda", "tokenization_byte.py": "279b9990f77b1eaedc7c1184fe13d60273a713ead856200f1fb9ae323c7b1e03", "tokenizer_config.json": "0ecbd4fd39e35125800d806af981ba0cacd8a7ba5f02c9e729c4ecc54af6d300", "special_tokens_map.json": "20d4b94d0e657c3f44d9123522c106e7315b6c1a1b6d259478f17bd2918b1263", "byte_vocab.json": "57a1cd2e70dd5b7cc6fd2eaa12ebfcd173c1620fa5d45c2c6282511c29b6a5f6", "trainer.pt": "ba31b41d133fff78caf1b22d96d15106f826e2e03648c9dbff5ca60049e9ff2e", "training_config.json": "2122ad6f8105987f9b87dd55d13b632084853908566b081c829ffe801259e2bf", "performance.json": "d8fc0ab4ba9b75a14bc27c39b9cdaf8c3a3b38cccaa4c038b8370fdf2cd49c08", "shard_scheduler.json": "9382ee0711b8a32284204139fa207dcc6bdbed89a4d1e96d772bcf51dbded624", "coverage.json": "a52c19a6ab0dd4e5feeaf4d905f52d95523ae03ff4b83abba914a7c129d58cfb", "README.md": "9ce0f9efacbc46177a7e818b6aed274d84e7145ff944b5c7d6ecd5d33b9d4419"}, "signature": "9fee47918905b0750bee87876aa424f021ccc4507ee8db22a11d01fbc6109063", "folder": "step-000000195000"}
+{"step": 200000, "files": {"config.json": "cbc72137da3dae65085a2a8c39ad7634d16bf92d785ca00d283ee82fe32b37e7", "generation_config.json": "54ed0d19f6226888c91157964ede8a892bd5ae175bfad85336152b3d9b6cd7c4", "model.safetensors": "9e1098c2f4131285db26132369fd315a230c2a1ed6433490bf04e1d55302bd5c", "tokenization_byte.py": "279b9990f77b1eaedc7c1184fe13d60273a713ead856200f1fb9ae323c7b1e03", "tokenizer_config.json": "0ecbd4fd39e35125800d806af981ba0cacd8a7ba5f02c9e729c4ecc54af6d300", "special_tokens_map.json": "20d4b94d0e657c3f44d9123522c106e7315b6c1a1b6d259478f17bd2918b1263", "byte_vocab.json": "57a1cd2e70dd5b7cc6fd2eaa12ebfcd173c1620fa5d45c2c6282511c29b6a5f6", "trainer.pt": "121ebbd424d06e2eca68e9473ccb7613c34366b9b444edaa47833ef5a5344919", "training_config.json": "95ba9e002790d11959d1191b36ed994bfa10a6da51c22c772e143bec517a9783", "performance.json": "7f72f0a0e868a4baec06f238f922d0c5d7e397a51386c11ac32acc776d4d3522", "shard_scheduler.json": "d840cc690a9276bb980e6c1c2e254aea3ef8a4071771396fd2e073516734a949", "coverage.json": "33255e32c3d706b5219a7812b83fcf6ed8998528c0f3efa9d1b474d0202a80c0", "README.md": "9ce0f9efacbc46177a7e818b6aed274d84e7145ff944b5c7d6ecd5d33b9d4419"}, "signature": "9fee47918905b0750bee87876aa424f021ccc4507ee8db22a11d01fbc6109063", "folder": "step-000000200000"}

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e7124a2db2f213964f3cd249326d01e111d4002da26c586796a207bae4728dda
+oid sha256:9e1098c2f4131285db26132369fd315a230c2a1ed6433490bf04e1d55302bd5c
 size 360622744

performance.json

@@ -49,11 +49,11 @@
   "selected": {
     "microbatch": 86,
     "mode": "compiled",
-    "microbatch_seconds": 0.8980338759999995,
+    "microbatch_seconds": 0.9016621779999809,
     "peak_gb": 85.39913082122803,
     "fits_budget": true,
-    "estimated_update_seconds": 0.9548798630000306,
-    "input_tokens_per_second": 196126.2316567667
+    "estimated_update_seconds": 0.9578150019999612,
+    "input_tokens_per_second": 195337.01678679453
   },
   "measurements": [
     {

shard_scheduler.json

@@ -22,8 +22,8 @@
       "tokens_per_second": 1062967.6168812835,
       "note": "Bounded fresh sample; full-page latency is an extrapolation.",
       "estimated_pages_for_buffer": 1,
-      "required_token_positions_per_second": 74454.29039832999,
-      "required_pages_per_second_upper_estimate": 0.006453509189053433
+      "required_token_positions_per_second": 75216.18075821365,
+      "required_pages_per_second_upper_estimate": 0.006519547914454685
     },
     {
       "source": "rewrite6",
@@ -36,8 +36,8 @@
       "tokens_per_second": 1206672.263377472,
       "note": "Bounded fresh sample; full-page latency is an extrapolation.",
       "estimated_pages_for_buffer": 1,
-      "required_token_positions_per_second": 9306.786299791249,
-      "required_pages_per_second_upper_estimate": 0.0037126162038420493
+      "required_token_positions_per_second": 9402.022594776707,
+      "required_pages_per_second_upper_estimate": 0.003750607385821249
     },
     {
       "source": "openmath",
@@ -50,8 +50,8 @@
       "tokens_per_second": 410042.7581134458,
       "note": "Bounded fresh sample; full-page latency is an extrapolation.",
       "estimated_pages_for_buffer": 1,
-      "required_token_positions_per_second": 18613.572599582498,
-      "required_pages_per_second_upper_estimate": 0.004305922433224167
+      "required_token_positions_per_second": 18804.045189553413,
+      "required_pages_per_second_upper_estimate": 0.0043499849147108464
     },
     {
       "source": "ultra_qa",
@@ -64,8 +64,8 @@
       "tokens_per_second": 2163486.5624068812,
       "note": "Bounded fresh sample; full-page latency is an extrapolation.",
       "estimated_pages_for_buffer": 1,
-      "required_token_positions_per_second": 27920.35889937375,
-      "required_pages_per_second_upper_estimate": 0.001700203370169813
+      "required_token_positions_per_second": 28206.06778433012,
+      "required_pages_per_second_upper_estimate": 0.0017176015422649853
     },
     {
       "source": "ultra_style",
@@ -78,8 +78,8 @@
       "tokens_per_second": 1546694.2737633225,
       "note": "Bounded fresh sample; full-page latency is an extrapolation.",
       "estimated_pages_for_buffer": 1,
-      "required_token_positions_per_second": 27920.35889937375,
-      "required_pages_per_second_upper_estimate": 0.0021493865165739087
+      "required_token_positions_per_second": 28206.06778433012,
+      "required_pages_per_second_upper_estimate": 0.002171381177430663
     },
     {
       "source": "cortex",
@@ -91,9 +91,9 @@
       "read_seconds": 0.042780299000014566,
       "tokens_per_second": 1015177.8395898147,
       "note": "Bounded fresh sample; full-page latency is an extrapolation.",
-      "estimated_pages_for_buffer": 21,
-      "required_token_positions_per_second": 27920.35889937375,
-      "required_pages_per_second_upper_estimate": 0.34667741410002545
+      "estimated_pages_for_buffer": 22,
+      "required_token_positions_per_second": 28206.06778433012,
+      "required_pages_per_second_upper_estimate": 0.35022496224505656
     }
   ],
   "gpu_measurement": "Selected forward/backward estimate; replaced by live successful update timings.",
@@ -155,413 +155,125 @@
   "shard_benchmark_scope": "Local packing/read measurements; upload capacity measured only on real data commits. Targets larger than available pages are sample-limited.",
   "estimated_fresh_generation_batch_seconds": 0.1434574083060457,
   "selected": {
-    "prefetch_batches": 61,
+    "prefetch_batches": 62,
     "prefetch_capacity_limit": 512,
     "batch_memory_estimate_bytes": 688946,
-    "producer_batch_seconds": 0.10249432328106423,
-    "producer_p95_seconds": 0.39124788300250657,
-    "gpu_update_seconds": 0.9902451504883822,
-    "producer_batches_per_second": 9.756637909182132,
-    "gpu_batches_per_second": 1.0098509439877659,
-    "producer_headroom": 9.661463374638716,
+    "producer_batch_seconds": 0.10476856211718655,
+    "producer_p95_seconds": 0.18885634199978085,
+    "gpu_update_seconds": 0.980214619471341,
+    "producer_batches_per_second": 9.544847994396184,
+    "gpu_batches_per_second": 1.0201847433568476,
+    "producer_headroom": 9.355999544738848,
     "sustainable": true,
-    "buffer_seconds": 60.40495417979132,
+    "buffer_seconds": 60.773306407223146,
     "buffer_drain_seconds": null,
     "shard_target_mib": 16,
-    "upload_pages_per_commit": 84,
+    "upload_pages_per_commit": 81,
     "max_pending_pages": 4096,
-    "generation_bytes_per_second": 328429.3339946892,
-    "upload_bytes_per_second": 4846781.735776416,
+    "generation_bytes_per_second": 364056.88282280555,
+    "upload_bytes_per_second": 5423901.113376007,
     "upload_sustainable": true,
     "logical_page_rows": 4096,
     "logical_cortex_page_rows": 128,
-    "estimated_full_shards_per_hour": 70.47328963165768,
+    "estimated_full_shards_per_hour": 78.11813224328161,
     "selection_policy": "Smallest shard within 90% of measured local peak, enlarged to amortize 30 seconds of measured page production; bounded to 256 MiB."
   },
   "upload_measurements": [
     {
-      "seconds": 1.207307631000731,
-      "bytes": 6036390,
-      "pages": 36,
-      "shards": 2
+      "seconds": 1.3317575169999145,
+      "bytes": 24816,
+      "pages": 1,
+      "shards": 1
     },
     {
-      "seconds": 1.257034783000563,
-      "bytes": 2415905,
+      "seconds": 1.540608198999962,
+      "bytes": 9815303,
       "pages": 31,
       "shards": 3
     },
     {
-      "seconds": 1.2379260390007403,
-      "bytes": 10087351,
-      "pages": 39,
-      "shards": 3
-    },
-    {
-      "seconds": 1.0402059740008553,
-      "bytes": 6293747,
-      "pages": 28,
-      "shards": 2
-    },
-    {
-      "seconds": 1.1847501439988264,
-      "bytes": 2572744,
-      "pages": 33,
-      "shards": 2
-    },
-    {
-      "seconds": 1.268939194000268,
-      "bytes": 16767391,
-      "pages": 42,
+      "seconds": 1.441364390000217,
+      "bytes": 13613764,
+      "pages": 38,
       "shards": 4
     },
     {
-      "seconds": 1.2854500899993582,
-      "bytes": 7334524,
-      "pages": 31,
-      "shards": 3
-    },
-    {
-      "seconds": 1.0401804770008312,
-      "bytes": 1254870,
+      "seconds": 0.8236200929995903,
+      "bytes": 1218836,
       "pages": 34,
       "shards": 1
     },
     {
-      "seconds": 1.428139841002121,
-      "bytes": 6304931,
-      "pages": 37,
-      "shards": 2
-    },
-    {
-      "seconds": 0.987766936996195,
-      "bytes": 2378061,
-      "pages": 29,
+      "seconds": 1.1421018979999644,
+      "bytes": 6132811,
+      "pages": 34,
       "shards": 2
     },
     {
-      "seconds": 1.0273188179999124,
-      "bytes": 9379394,
-      "pages": 33,
-      "shards": 3
-    },
-    {
-      "seconds": 1.4798621720037772,
-      "bytes": 13649971,
-      "pages": 37,
+      "seconds": 1.2374535429999014,
+      "bytes": 7166128,
+      "pages": 34,
       "shards": 4
     },
     {
-      "seconds": 0.8202600639997399,
-      "bytes": 1256174,
+      "seconds": 1.0863462409997737,
+      "bytes": 6279325,
       "pages": 35,
-      "shards": 1
-    },
-    {
-      "seconds": 1.0791088629994192,
-      "bytes": 10296860,
-      "pages": 35,
-      "shards": 3
-    },
-    {
-      "seconds": 1.2926067090011202,
-      "bytes": 2391669,
-      "pages": 32,
-      "shards": 2
-    },
-    {
-      "seconds": 1.033323986994219,
-      "bytes": 6001182,
-      "pages": 33,
-      "shards": 2
-    },
-    {
-      "seconds": 3.9172589509980753,
-      "bytes": 5890320,
-      "pages": 32,
-      "shards": 2
-    },
-    {
-      "seconds": 1.3426188599987654,
-      "bytes": 12956908,
-      "pages": 37,
-      "shards": 4
-    },
-    {
-      "seconds": 1.2346246180022717,
-      "bytes": 6315095,
-      "pages": 36,
-      "shards": 2
-    },
-    {
-      "seconds": 3.684220068003924,
-      "bytes": 2396656,
-      "pages": 29,
-      "shards": 2
-    },
-    {
-      "seconds": 1.1850955770059954,
-      "bytes": 6553717,
-      "pages": 37,
       "shards": 2
     },
     {
-      "seconds": 1.714512132006348,
-      "bytes": 6167274,
-      "pages": 33,
-      "shards": 2
-    },
-    {
-      "seconds": 1.2202410780009814,
-      "bytes": 6552255,
-      "pages": 33,
-      "shards": 3
-    },
-    {
-      "seconds": 1.1842839309974806,
-      "bytes": 12073153,
-      "pages": 39,
-      "shards": 3
-    },
-    {
-      "seconds": 0.9938246039964724,
-      "bytes": 235889,
-      "pages": 8,
-      "shards": 1
-    },
-    {
-      "seconds": 1.3448229619971244,
-      "bytes": 7303500,
-      "pages": 31,
+      "seconds": 0.9579987879997134,
+      "bytes": 7302822,
+      "pages": 28,
       "shards": 3
     },
     {
-      "seconds": 1.1307224229967687,
-      "bytes": 5856127,
-      "pages": 37,
-      "shards": 2
-    },
-    {
-      "seconds": 1.2856477119930787,
-      "bytes": 6096134,
-      "pages": 29,
-      "shards": 2
-    },
-    {
-      "seconds": 1.163583995999943,
-      "bytes": 2620539,
-      "pages": 37,
+      "seconds": 0.9735736169996017,
+      "bytes": 6894198,
+      "pages": 36,
       "shards": 2
     },
     {
-      "seconds": 1.2240531869974802,
-      "bytes": 11650354,
-      "pages": 31,
+      "seconds": 1.282220471999608,
+      "bytes": 9425496,
+      "pages": 34,
       "shards": 3
     },
     {
-      "seconds": 1.2925304770033108,
-      "bytes": 12240192,
-      "pages": 41,
-      "shards": 4
-    },
-    {
-      "seconds": 0.9206129219965078,
-      "bytes": 852268,
-      "pages": 27,
-      "shards": 1
-    },
-    {
-      "seconds": 1.2395337650013971,
-      "bytes": 6262507,
+      "seconds": 1.236358336000194,
+      "bytes": 7782594,
       "pages": 38,
       "shards": 3
     },
     {
-      "seconds": 1.0192943180009024,
-      "bytes": 2301578,
+      "seconds": 1.0319873409998763,
+      "bytes": 964782,
       "pages": 28,
-      "shards": 2
-    },
-    {
-      "seconds": 2.074865057998977,
-      "bytes": 9907182,
-      "pages": 39,
-      "shards": 3
-    },
-    {
-      "seconds": 1.498775358995772,
-      "bytes": 5765034,
-      "pages": 28,
-      "shards": 2
-    },
-    {
-      "seconds": 1.3661099450037,
-      "bytes": 8604864,
-      "pages": 41,
-      "shards": 3
+      "shards": 1
     },
     {
-      "seconds": 1.0984054020009353,
-      "bytes": 6085035,
+      "seconds": 1.3289059700000507,
+      "bytes": 6144064,
       "pages": 29,
       "shards": 2
     },
     {
-      "seconds": 1.4394773240055656,
-      "bytes": 12207803,
-      "pages": 42,
-      "shards": 4
-    },
-    {
-      "seconds": 0.8468366760062054,
-      "bytes": 871307,
-      "pages": 27,
-      "shards": 1
-    },
-    {
-      "seconds": 1.2028667299964582,
-      "bytes": 6129254,
+      "seconds": 1.1848053769999751,
+      "bytes": 6754320,
       "pages": 38,
-      "shards": 2
-    },
-    {
-      "seconds": 1.6012624290015083,
-      "bytes": 8291413,
-      "pages": 33,
       "shards": 3
     },
     {
-      "seconds": 1.3447834969992982,
-      "bytes": 10108218,
+      "seconds": 1.6214569329995356,
+      "bytes": 11854578,
       "pages": 32,
       "shards": 3
     },
     {
-      "seconds": 1.1404380909953034,
-      "bytes": 7821336,
-      "pages": 39,
-      "shards": 3
-    },
-    {
-      "seconds": 1.2159917039971333,
-      "bytes": 870322,
-      "pages": 29,
-      "shards": 1
-    },
-    {
-      "seconds": 1.047054365000804,
-      "bytes": 6014250,
-      "pages": 35,
-      "shards": 2
-    },
-    {
-      "seconds": 1.1608161260010093,
-      "bytes": 6519078,
-      "pages": 32,
-      "shards": 3
-    },
-    {
-      "seconds": 1.5200698819971876,
-      "bytes": 6082599,
-      "pages": 34,
-      "shards": 2
-    },
-    {
-      "seconds": 1.4744428199992399,
-      "bytes": 11695751,
-      "pages": 38,
-      "shards": 3
-    },
-    {
-      "seconds": 1.3584106329944916,
-      "bytes": 2483287,
+      "seconds": 0.9589299640001627,
+      "bytes": 2653813,
       "pages": 36,
       "shards": 2
-    },
-    {
-      "seconds": 1.1772192700009327,
-      "bytes": 10757429,
-      "pages": 34,
-      "shards": 3
-    },
-    {
-      "seconds": 2.1233109069944476,
-      "bytes": 2398655,
-      "pages": 29,
-      "shards": 2
-    },
-    {
-      "seconds": 1.0890797859974555,
-      "bytes": 6225655,
-      "pages": 38,
-      "shards": 2
-    },
-    {
-      "seconds": 3.324964125000406,
-      "bytes": 5816918,
-      "pages": 28,
-      "shards": 2
-    },
-    {
-      "seconds": 1.304179352002393,
-      "bytes": 11867177,
-      "pages": 37,
-      "shards": 4
-    },
-    {
-      "seconds": 1.3306422289970214,
-      "bytes": 6277468,
-      "pages": 38,
-      "shards": 2
-    },
-    {
-      "seconds": 0.9982111080025788,
-      "bytes": 877422,
-      "pages": 26,
-      "shards": 1
-    },
-    {
-      "seconds": 1.6349642249988392,
-      "bytes": 7479267,
-      "pages": 33,
-      "shards": 3
-    },
-    {
-      "seconds": 1.1783448380010668,
-      "bytes": 10185058,
-      "pages": 38,
-      "shards": 3
-    },
-    {
-      "seconds": 0.9609743260007235,
-      "bytes": 2529461,
-      "pages": 33,
-      "shards": 2
-    },
-    {
-      "seconds": 1.5562968439990073,
-      "bytes": 11835464,
-      "pages": 39,
-      "shards": 3
-    },
-    {
-      "seconds": 0.8645425069989869,
-      "bytes": 693988,
-      "pages": 20,
-      "shards": 1
-    },
-    {
-      "seconds": 1.3442884890027926,
-      "bytes": 12581350,
-      "pages": 41,
-      "shards": 4
-    },
-    {
-      "seconds": 1.4273276979947696,
-      "bytes": 6232998,
-      "pages": 37,
-      "shards": 3
     }
   ],
   "cache_key": {
@@ -606,7 +318,7 @@
   },
   "cache_reused": true,
   "resume_probe_seconds": [
-    2.7487001069999906,
-    0.015045025999995687
+    3.044028898000022,
+    0.01595121099995822
   ]
 }

trainer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ba31b41d133fff78caf1b22d96d15106f826e2e03648c9dbff5ca60049e9ff2e
-size 721763199
+oid sha256:121ebbd424d06e2eca68e9473ccb7613c34366b9b444edaa47833ef5a5344919
+size 721772159

training_config.json

@@ -46,8 +46,8 @@
   "cpu_threads": 4,
   "page_rows": 4096,
   "cortex_page_rows": 128,
-  "prefetch_batches": 61,
-  "prefetch_low_water": 30,
+  "prefetch_batches": 62,
+  "prefetch_low_water": 31,
   "memory_pages": 12,
   "cache_gb": 20,
   "force_rebenchmark": false,