Qwen90M step 335000; complete resumable checkpoint

complete.json

@@ -1,18 +1,18 @@
 {
-  "step": 330000,
+  "step": 335000,
   "files": {
     "config.json": "cbc72137da3dae65085a2a8c39ad7634d16bf92d785ca00d283ee82fe32b37e7",
     "generation_config.json": "54ed0d19f6226888c91157964ede8a892bd5ae175bfad85336152b3d9b6cd7c4",
-    "model.safetensors": "328bd745ba3eedb0c09045707f1c3645741429105fa758b46400ed4e42af0708",
+    "model.safetensors": "7438707d5944ffca46843ff585ead722c9fa7bdc8dab736da1510f4bff5c4701",
     "tokenization_byte.py": "279b9990f77b1eaedc7c1184fe13d60273a713ead856200f1fb9ae323c7b1e03",
     "tokenizer_config.json": "0ecbd4fd39e35125800d806af981ba0cacd8a7ba5f02c9e729c4ecc54af6d300",
     "special_tokens_map.json": "20d4b94d0e657c3f44d9123522c106e7315b6c1a1b6d259478f17bd2918b1263",
     "byte_vocab.json": "57a1cd2e70dd5b7cc6fd2eaa12ebfcd173c1620fa5d45c2c6282511c29b6a5f6",
-    "trainer.pt": "15e5868d1d2d9bba0011346b8b8fe6a7017e790885cbae0ddebf73fbc62c7dfe",
-    "training_config.json": "95ba9e002790d11959d1191b36ed994bfa10a6da51c22c772e143bec517a9783",
+    "trainer.pt": "5f4a15472e9622ce649a08d61cb56adda487f4944d44a36aff803668640d8309",
+    "training_config.json": "2122ad6f8105987f9b87dd55d13b632084853908566b081c829ffe801259e2bf",
     "performance.json": "90ad39d684ac792f235cbfbeef498d4c81f09977d8f89fb8c3bdd41e515f347a",
-    "shard_scheduler.json": "9ea0c54003a1a89d00999afcc33c19cd9e20fd18d66970efa7421ab898bf1d77",
-    "coverage.json": "8db65ea8720123ccbfed7d388c2266922b6f26d48d6d3b735a4714722ac5561c",
+    "shard_scheduler.json": "c1a7db519fe396e51051231707a3d26cd5328da0fb81bb9da77c36f5e680779e",
+    "coverage.json": "dddfaa0a3732c9d4616f58805f4c39815efab74bccdd566963924cd8fcbd4985",
     "README.md": "9ce0f9efacbc46177a7e818b6aed274d84e7145ff944b5c7d6ecd5d33b9d4419"
   },
   "signature": "9fee47918905b0750bee87876aa424f021ccc4507ee8db22a11d01fbc6109063"

coverage.json

@@ -1,107 +1,107 @@
 {
-  "step": 330000,
+  "step": 335000,
   "stats": {
-    "climbmix/blocks": 11880000,
-    "climbmix/targets": 19858829729,
-    "climbmix/input_tokens": 19860088763,
-    "ultra_qa/blocks": 4455000,
-    "ultra_qa/targets": 7267913693,
-    "ultra_qa/input_tokens": 7267927725,
-    "cortex/blocks": 4455000,
-    "cortex/targets": 169500703,
-    "cortex/input_tokens": 7082136293,
-    "ultra_style/blocks": 4455000,
-    "ultra_style/targets": 7046969672,
-    "ultra_style/input_tokens": 7047339282,
-    "openmath/blocks": 2970000,
-    "openmath/targets": 4589449827,
-    "openmath/input_tokens": 4590979616,
-    "rewrite6/blocks": 1485000,
-    "rewrite6/targets": 2434435841,
-    "rewrite6/input_tokens": 2437105213
+    "climbmix/blocks": 12060000,
+    "climbmix/targets": 20160288396,
+    "climbmix/input_tokens": 20161566115,
+    "ultra_qa/blocks": 4522500,
+    "ultra_qa/targets": 7378025318,
+    "ultra_qa/input_tokens": 7378039564,
+    "cortex/blocks": 4522500,
+    "cortex/targets": 172070651,
+    "cortex/input_tokens": 7189400225,
+    "ultra_style/blocks": 4522500,
+    "ultra_style/targets": 7153491246,
+    "ultra_style/input_tokens": 7153866702,
+    "openmath/blocks": 3015000,
+    "openmath/targets": 4658887005,
+    "openmath/input_tokens": 4660439825,
+    "rewrite6/blocks": 1507500,
+    "rewrite6/targets": 2471326187,
+    "rewrite6/input_tokens": 2474035818
   },
   "sources": {
     "climbmix": {
-      "page": 1627,
-      "row": 1818,
+      "page": 1651,
+      "row": 3325,
       "start": {
-        "file": 78,
+        "file": 79,
         "reader": {
           "format": "parquet-row-v1",
-          "row_group": 62,
+          "row_group": 75,
           "row": 1024
         }
       },
       "epoch": 0,
       "offset": 2048,
-      "documents": 6666010
+      "documents": 6765821
     },
     "rewrite6": {
-      "page": 25,
-      "row": 3161,
+      "page": 39,
+      "row": 3247,
       "start": {
         "file": 0,
         "reader": {
           "format": "jsonl-row-v1",
-          "row": 102400
+          "row": 159744
         }
       },
       "epoch": 18,
       "offset": 0,
-      "documents": 3802833
+      "documents": 3860263
     },
     "openmath": {
-      "page": 1016,
-      "row": 4063,
+      "page": 1032,
+      "row": 1485,
       "start": {
         "file": 9,
         "reader": {
           "format": "parquet-row-v1",
-          "row_group": 231,
-          "row": 686
+          "row_group": 297,
+          "row": 222
         }
       },
       "epoch": 0,
       "offset": 0,
-      "documents": 4165599
+      "documents": 4228557
     },
     "ultra_qa": {
-      "page": 439,
-      "row": 2714,
+      "page": 446,
+      "row": 1301,
       "start": {
         "file": 3,
         "reader": {
           "format": "parquet-row-v1",
-          "row_group": 3,
-          "row": 45392
+          "row_group": 4,
+          "row": 9563
         }
       },
       "epoch": 0,
-      "offset": 4096,
-      "documents": 1800858
+      "offset": 2048,
+      "documents": 1828117
     },
     "ultra_style": {
-      "page": 671,
-      "row": 421,
+      "page": 681,
+      "row": 1612,
       "start": {
         "file": 4,
         "reader": {
           "format": "parquet-row-v1",
           "row_group": 0,
-          "row": 8645
+          "row": 49605
         }
       },
       "epoch": 0,
-      "offset": 2048,
-      "documents": 2748837
+      "offset": 0,
+      "documents": 2790988
     },
     "cortex": {
-      "page": 86407,
-      "row": 78,
+      "page": 87716,
+      "row": 66,
       "start": null,
       "epoch": 0,
       "offset": 0,
-      "documents": 11060174
+      "documents": 11227714
     }
   }
 }

latest.json

@@ -1 +1 @@
-{"step": 330000, "files": {"config.json": "cbc72137da3dae65085a2a8c39ad7634d16bf92d785ca00d283ee82fe32b37e7", "generation_config.json": "54ed0d19f6226888c91157964ede8a892bd5ae175bfad85336152b3d9b6cd7c4", "model.safetensors": "328bd745ba3eedb0c09045707f1c3645741429105fa758b46400ed4e42af0708", "tokenization_byte.py": "279b9990f77b1eaedc7c1184fe13d60273a713ead856200f1fb9ae323c7b1e03", "tokenizer_config.json": "0ecbd4fd39e35125800d806af981ba0cacd8a7ba5f02c9e729c4ecc54af6d300", "special_tokens_map.json": "20d4b94d0e657c3f44d9123522c106e7315b6c1a1b6d259478f17bd2918b1263", "byte_vocab.json": "57a1cd2e70dd5b7cc6fd2eaa12ebfcd173c1620fa5d45c2c6282511c29b6a5f6", "trainer.pt": "15e5868d1d2d9bba0011346b8b8fe6a7017e790885cbae0ddebf73fbc62c7dfe", "training_config.json": "95ba9e002790d11959d1191b36ed994bfa10a6da51c22c772e143bec517a9783", "performance.json": "90ad39d684ac792f235cbfbeef498d4c81f09977d8f89fb8c3bdd41e515f347a", "shard_scheduler.json": "9ea0c54003a1a89d00999afcc33c19cd9e20fd18d66970efa7421ab898bf1d77", "coverage.json": "8db65ea8720123ccbfed7d388c2266922b6f26d48d6d3b735a4714722ac5561c", "README.md": "9ce0f9efacbc46177a7e818b6aed274d84e7145ff944b5c7d6ecd5d33b9d4419"}, "signature": "9fee47918905b0750bee87876aa424f021ccc4507ee8db22a11d01fbc6109063", "folder": "step-000000330000"}
+{"step": 335000, "files": {"config.json": "cbc72137da3dae65085a2a8c39ad7634d16bf92d785ca00d283ee82fe32b37e7", "generation_config.json": "54ed0d19f6226888c91157964ede8a892bd5ae175bfad85336152b3d9b6cd7c4", "model.safetensors": "7438707d5944ffca46843ff585ead722c9fa7bdc8dab736da1510f4bff5c4701", "tokenization_byte.py": "279b9990f77b1eaedc7c1184fe13d60273a713ead856200f1fb9ae323c7b1e03", "tokenizer_config.json": "0ecbd4fd39e35125800d806af981ba0cacd8a7ba5f02c9e729c4ecc54af6d300", "special_tokens_map.json": "20d4b94d0e657c3f44d9123522c106e7315b6c1a1b6d259478f17bd2918b1263", "byte_vocab.json": "57a1cd2e70dd5b7cc6fd2eaa12ebfcd173c1620fa5d45c2c6282511c29b6a5f6", "trainer.pt": "5f4a15472e9622ce649a08d61cb56adda487f4944d44a36aff803668640d8309", "training_config.json": "2122ad6f8105987f9b87dd55d13b632084853908566b081c829ffe801259e2bf", "performance.json": "90ad39d684ac792f235cbfbeef498d4c81f09977d8f89fb8c3bdd41e515f347a", "shard_scheduler.json": "c1a7db519fe396e51051231707a3d26cd5328da0fb81bb9da77c36f5e680779e", "coverage.json": "dddfaa0a3732c9d4616f58805f4c39815efab74bccdd566963924cd8fcbd4985", "README.md": "9ce0f9efacbc46177a7e818b6aed274d84e7145ff944b5c7d6ecd5d33b9d4419"}, "signature": "9fee47918905b0750bee87876aa424f021ccc4507ee8db22a11d01fbc6109063", "folder": "step-000000335000"}

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:328bd745ba3eedb0c09045707f1c3645741429105fa758b46400ed4e42af0708
+oid sha256:7438707d5944ffca46843ff585ead722c9fa7bdc8dab736da1510f4bff5c4701
 size 360622744

shard_scheduler.json

@@ -22,8 +22,8 @@
       "tokens_per_second": 1062967.6168812835,
       "note": "Bounded fresh sample; full-page latency is an extrapolation.",
       "estimated_pages_for_buffer": 1,
-      "required_token_positions_per_second": 75630.60254063262,
-      "required_pages_per_second_upper_estimate": 0.006555468944212357
+      "required_token_positions_per_second": 73863.57629496043,
+      "required_pages_per_second_upper_estimate": 0.006402307587724566
     },
     {
       "source": "rewrite6",
@@ -36,8 +36,8 @@
       "tokens_per_second": 1206672.263377472,
       "note": "Bounded fresh sample; full-page latency is an extrapolation.",
       "estimated_pages_for_buffer": 1,
-      "required_token_positions_per_second": 9453.825317579078,
-      "required_pages_per_second_upper_estimate": 0.003771272266466841
+      "required_token_positions_per_second": 9232.947036870053,
+      "required_pages_per_second_upper_estimate": 0.0036831606178674217
     },
     {
       "source": "openmath",
@@ -50,8 +50,8 @@
       "tokens_per_second": 410042.7581134458,
       "note": "Bounded fresh sample; full-page latency is an extrapolation.",
       "estimated_pages_for_buffer": 1,
-      "required_token_positions_per_second": 18907.650635158156,
-      "required_pages_per_second_upper_estimate": 0.004373952211157938
+      "required_token_positions_per_second": 18465.894073740106,
+      "required_pages_per_second_upper_estimate": 0.004271759605323816
     },
     {
       "source": "ultra_qa",
@@ -64,8 +64,8 @@
       "tokens_per_second": 2163486.5624068812,
       "note": "Bounded fresh sample; full-page latency is an extrapolation.",
       "estimated_pages_for_buffer": 1,
-      "required_token_positions_per_second": 28361.475952737233,
-      "required_pages_per_second_upper_estimate": 0.0017270650843573335
+      "required_token_positions_per_second": 27698.84111061016,
+      "required_pages_per_second_upper_estimate": 0.0016867140990481273
     },
     {
       "source": "ultra_style",
@@ -78,8 +78,8 @@
       "tokens_per_second": 1546694.2737633225,
       "note": "Bounded fresh sample; full-page latency is an extrapolation.",
       "estimated_pages_for_buffer": 1,
-      "required_token_positions_per_second": 28361.475952737233,
-      "required_pages_per_second_upper_estimate": 0.002183344928431987
+      "required_token_positions_per_second": 27698.84111061016,
+      "required_pages_per_second_upper_estimate": 0.0021323334639944016
     },
     {
       "source": "cortex",
@@ -91,9 +91,9 @@
       "read_seconds": 0.042780299000014566,
       "tokens_per_second": 1015177.8395898147,
       "note": "Bounded fresh sample; full-page latency is an extrapolation.",
-      "estimated_pages_for_buffer": 22,
-      "required_token_positions_per_second": 28361.475952737233,
-      "required_pages_per_second_upper_estimate": 0.3521546115789914
+      "estimated_pages_for_buffer": 21,
+      "required_token_positions_per_second": 27698.84111061016,
+      "required_pages_per_second_upper_estimate": 0.34392690453592956
     }
   ],
   "gpu_measurement": "Selected forward/backward estimate; replaced by live successful update timings.",
@@ -155,413 +155,413 @@
   "shard_benchmark_scope": "Local packing/read measurements; upload capacity measured only on real data commits. Targets larger than available pages are sample-limited.",
   "estimated_fresh_generation_batch_seconds": 0.1434574083060457,
   "selected": {
-    "prefetch_batches": 62,
+    "prefetch_batches": 61,
     "prefetch_capacity_limit": 512,
     "batch_memory_estimate_bytes": 690670,
-    "producer_batch_seconds": 0.0792378665312583,
-    "producer_p95_seconds": 0.2171078139999736,
-    "gpu_update_seconds": 0.9748434829722473,
-    "producer_batches_per_second": 12.620228733764721,
-    "gpu_batches_per_second": 1.0258056985220354,
-    "producer_headroom": 12.302747734729635,
+    "producer_batch_seconds": 0.11786020207830461,
+    "producer_p95_seconds": 0.29016289100036374,
+    "gpu_update_seconds": 0.9981645040524572,
+    "producer_batches_per_second": 8.48462824911512,
+    "gpu_batches_per_second": 1.0018388711881567,
+    "producer_headroom": 8.469054748347462,
     "sustainable": true,
-    "buffer_seconds": 60.440295944279335,
+    "buffer_seconds": 60.88803474719989,
     "buffer_drain_seconds": null,
     "shard_target_mib": 16,
-    "upload_pages_per_commit": 83,
+    "upload_pages_per_commit": 79,
     "max_pending_pages": 4096,
-    "generation_bytes_per_second": 523269.04561499885,
-    "upload_bytes_per_second": 4017522.9884006456,
+    "generation_bytes_per_second": 418892.7533661353,
+    "upload_bytes_per_second": 5731055.085245443,
     "upload_sustainable": true,
     "logical_page_rows": 4096,
     "logical_cortex_page_rows": 128,
-    "estimated_full_shards_per_hour": 112.28135610902285,
+    "estimated_full_shards_per_hour": 89.88463354814571,
     "selection_policy": "Smallest shard within 90% of measured local peak, enlarged to amortize 30 seconds of measured page production; bounded to 256 MiB."
   },
   "upload_measurements": [
     {
-      "seconds": 1.6009747379994224,
-      "bytes": 7396060,
+      "seconds": 1.0918037890005508,
+      "bytes": 11651730,
       "pages": 37,
       "shards": 3
     },
     {
-      "seconds": 0.9060076790001403,
-      "bytes": 1026778,
-      "pages": 33,
-      "shards": 1
+      "seconds": 1.0898228620008013,
+      "bytes": 2320314,
+      "pages": 28,
+      "shards": 2
     },
     {
-      "seconds": 1.30779888699999,
-      "bytes": 10925777,
-      "pages": 35,
+      "seconds": 0.9809586890005448,
+      "bytes": 9894024,
+      "pages": 31,
       "shards": 3
     },
     {
-      "seconds": 0.9117075719987042,
-      "bytes": 2359249,
-      "pages": 28,
+      "seconds": 1.0132985079999344,
+      "bytes": 6275693,
+      "pages": 36,
       "shards": 2
     },
     {
-      "seconds": 0.9851995050012192,
-      "bytes": 5930447,
-      "pages": 34,
+      "seconds": 0.9494441940005345,
+      "bytes": 2648505,
+      "pages": 35,
       "shards": 2
     },
     {
-      "seconds": 1.2710061699999642,
-      "bytes": 13577169,
-      "pages": 42,
-      "shards": 4
-    },
-    {
-      "seconds": 0.8450371549988631,
-      "bytes": 895470,
-      "pages": 25,
-      "shards": 1
+      "seconds": 1.0081078959992738,
+      "bytes": 5874669,
+      "pages": 36,
+      "shards": 2
     },
     {
-      "seconds": 32.774606008000774,
-      "bytes": 10411787,
-      "pages": 35,
-      "shards": 3
+      "seconds": 1.0266657669999404,
+      "bytes": 4655742,
+      "pages": 12,
+      "shards": 2
     },
     {
-      "seconds": 1.2439632269997674,
-      "bytes": 7901436,
-      "pages": 46,
-      "shards": 3
+      "seconds": 1.4716465549990971,
+      "bytes": 13803834,
+      "pages": 48,
+      "shards": 4
     },
     {
-      "seconds": 0.8556265790011821,
-      "bytes": 1138175,
-      "pages": 33,
-      "shards": 1
+      "seconds": 0.940834745999382,
+      "bytes": 6400703,
+      "pages": 35,
+      "shards": 2
     },
     {
-      "seconds": 1.1632403840012557,
-      "bytes": 11705717,
-      "pages": 30,
-      "shards": 3
+      "seconds": 1.1989260670015938,
+      "bytes": 2439937,
+      "pages": 28,
+      "shards": 2
     },
     {
-      "seconds": 1.197896118999779,
-      "bytes": 12261236,
-      "pages": 40,
+      "seconds": 1.2518158220009354,
+      "bytes": 10474054,
+      "pages": 38,
       "shards": 4
     },
     {
-      "seconds": 0.8173304430001735,
-      "bytes": 1136030,
-      "pages": 33,
+      "seconds": 1.1628889589992468,
+      "bytes": 1077101,
+      "pages": 34,
       "shards": 1
     },
     {
-      "seconds": 1.1651587139986077,
-      "bytes": 5801842,
-      "pages": 20,
-      "shards": 2
+      "seconds": 1.117781933000515,
+      "bytes": 7831651,
+      "pages": 22,
+      "shards": 3
     },
     {
-      "seconds": 1.0791701340003783,
-      "bytes": 2256184,
-      "pages": 26,
+      "seconds": 1.0544361239990394,
+      "bytes": 7130727,
+      "pages": 44,
       "shards": 2
     },
     {
-      "seconds": 1.5194503539987636,
-      "bytes": 9946744,
+      "seconds": 1.1150904570004059,
+      "bytes": 10599538,
+      "pages": 29,
+      "shards": 3
+    },
+    {
+      "seconds": 1.0015506020008615,
+      "bytes": 7818449,
       "pages": 38,
       "shards": 3
     },
     {
-      "seconds": 1.188283532999776,
-      "bytes": 2401103,
-      "pages": 29,
-      "shards": 2
+      "seconds": 0.8705007850003312,
+      "bytes": 1070987,
+      "pages": 34,
+      "shards": 1
     },
     {
-      "seconds": 1.182281362998765,
-      "bytes": 11896636,
-      "pages": 38,
+      "seconds": 0.9655942159988626,
+      "bytes": 7298211,
+      "pages": 29,
       "shards": 3
     },
     {
-      "seconds": 0.8353424750002887,
-      "bytes": 975468,
-      "pages": 29,
+      "seconds": 0.8577407669999957,
+      "bytes": 1078743,
+      "pages": 30,
       "shards": 1
     },
     {
-      "seconds": 1.1824829600009252,
-      "bytes": 12495213,
-      "pages": 37,
+      "seconds": 1.3106396379989746,
+      "bytes": 15458880,
+      "pages": 39,
       "shards": 4
     },
     {
-      "seconds": 1.254413848000695,
-      "bytes": 6063292,
-      "pages": 35,
+      "seconds": 0.9607973950005544,
+      "bytes": 7512102,
+      "pages": 38,
       "shards": 3
     },
     {
-      "seconds": 0.9384880159996101,
-      "bytes": 1115458,
-      "pages": 34,
+      "seconds": 2.3653303009996307,
+      "bytes": 1160564,
+      "pages": 33,
       "shards": 1
     },
     {
-      "seconds": 1.1859958829991228,
-      "bytes": 7375397,
-      "pages": 29,
-      "shards": 3
-    },
-    {
-      "seconds": 0.9071520359993883,
-      "bytes": 6539230,
-      "pages": 25,
-      "shards": 2
+      "seconds": 1.3003170710017002,
+      "bytes": 11354768,
+      "pages": 32,
+      "shards": 4
     },
     {
-      "seconds": 1.2508507479997206,
-      "bytes": 11653692,
-      "pages": 45,
-      "shards": 4
+      "seconds": 0.953995298999871,
+      "bytes": 345798,
+      "pages": 9,
+      "shards": 1
     },
     {
-      "seconds": 1.2195578610007942,
-      "bytes": 5944204,
-      "pages": 27,
+      "seconds": 1.1301041130009253,
+      "bytes": 5919931,
+      "pages": 35,
       "shards": 2
     },
     {
-      "seconds": 0.9418964010001218,
-      "bytes": 1018742,
-      "pages": 33,
+      "seconds": 0.7547573850006302,
+      "bytes": 662301,
+      "pages": 19,
       "shards": 1
     },
     {
-      "seconds": 1.3071213839994016,
-      "bytes": 12190710,
-      "pages": 40,
+      "seconds": 1.1137079630007065,
+      "bytes": 13478878,
+      "pages": 47,
       "shards": 4
     },
     {
-      "seconds": 0.9488432640009705,
-      "bytes": 921326,
-      "pages": 29,
-      "shards": 1
+      "seconds": 1.1504248440014635,
+      "bytes": 10949848,
+      "pages": 32,
+      "shards": 3
     },
     {
-      "seconds": 1.0550517599986051,
-      "bytes": 5962735,
-      "pages": 26,
+      "seconds": 0.9190237900002103,
+      "bytes": 2687904,
+      "pages": 36,
       "shards": 2
     },
     {
-      "seconds": 1.1898453599987988,
-      "bytes": 13419202,
-      "pages": 43,
-      "shards": 4
+      "seconds": 0.9390472680024686,
+      "bytes": 6039556,
+      "pages": 27,
+      "shards": 2
     },
     {
-      "seconds": 0.9201347599992005,
-      "bytes": 838883,
-      "pages": 26,
+      "seconds": 1.1019230559977586,
+      "bytes": 1147220,
+      "pages": 34,
       "shards": 1
     },
     {
-      "seconds": 1.1256958809990465,
-      "bytes": 12103363,
-      "pages": 38,
+      "seconds": 1.2733968520005874,
+      "bytes": 11422716,
+      "pages": 32,
       "shards": 4
     },
     {
-      "seconds": 1.0281697059999715,
-      "bytes": 1191797,
+      "seconds": 1.048131932002434,
+      "bytes": 6792716,
       "pages": 34,
-      "shards": 1
+      "shards": 2
     },
     {
-      "seconds": 0.9877725879996433,
-      "bytes": 6235947,
+      "seconds": 0.9567362410016358,
+      "bytes": 6277518,
       "pages": 36,
       "shards": 2
     },
     {
-      "seconds": 1.2409586350004247,
-      "bytes": 7163170,
-      "pages": 28,
+      "seconds": 1.2580463369995414,
+      "bytes": 7425150,
+      "pages": 37,
       "shards": 3
     },
     {
-      "seconds": 1.270394237999426,
-      "bytes": 10514630,
-      "pages": 29,
-      "shards": 3
+      "seconds": 1.0858683609985746,
+      "bytes": 5956157,
+      "pages": 32,
+      "shards": 2
     },
     {
-      "seconds": 1.509260009999707,
-      "bytes": 6276691,
+      "seconds": 1.2466707620005764,
+      "bytes": 7844213,
       "pages": 36,
+      "shards": 3
+    },
+    {
+      "seconds": 1.0721679589987616,
+      "bytes": 5937253,
+      "pages": 29,
       "shards": 2
     },
     {
-      "seconds": 1.1396728859999712,
-      "bytes": 2416731,
-      "pages": 27,
+      "seconds": 0.9170985109994945,
+      "bytes": 6599086,
+      "pages": 28,
       "shards": 2
     },
     {
-      "seconds": 0.9309691420003219,
-      "bytes": 6123756,
-      "pages": 33,
+      "seconds": 1.249970442000631,
+      "bytes": 7464066,
+      "pages": 32,
+      "shards": 3
+    },
+    {
+      "seconds": 1.004503175001446,
+      "bytes": 5154959,
+      "pages": 38,
       "shards": 2
     },
     {
-      "seconds": 1.1473543519987288,
-      "bytes": 5736316,
-      "pages": 34,
+      "seconds": 1.2185796390003816,
+      "bytes": 6487750,
+      "pages": 35,
       "shards": 2
     },
     {
-      "seconds": 1.0078334160007216,
-      "bytes": 7473663,
-      "pages": 37,
+      "seconds": 1.3924951020017033,
+      "bytes": 2562340,
+      "pages": 31,
       "shards": 3
     },
     {
-      "seconds": 1.0918037890005508,
-      "bytes": 11651730,
-      "pages": 37,
+      "seconds": 1.429218111999944,
+      "bytes": 10705946,
+      "pages": 39,
       "shards": 3
     },
     {
-      "seconds": 1.0898228620008013,
-      "bytes": 2320314,
-      "pages": 28,
+      "seconds": 1.0223479320011393,
+      "bytes": 6489223,
+      "pages": 24,
       "shards": 2
     },
     {
-      "seconds": 0.9809586890005448,
-      "bytes": 9894024,
-      "pages": 31,
+      "seconds": 1.180141814002127,
+      "bytes": 7534736,
+      "pages": 35,
       "shards": 3
     },
     {
-      "seconds": 1.0132985079999344,
-      "bytes": 6275693,
-      "pages": 36,
+      "seconds": 1.0304640750000544,
+      "bytes": 6585841,
+      "pages": 35,
       "shards": 2
     },
     {
-      "seconds": 0.9494441940005345,
-      "bytes": 2648505,
+      "seconds": 0.9793735439998272,
+      "bytes": 7348514,
       "pages": 35,
-      "shards": 2
+      "shards": 3
     },
     {
-      "seconds": 1.0081078959992738,
-      "bytes": 5874669,
-      "pages": 36,
+      "seconds": 1.1231975430018792,
+      "bytes": 5800198,
+      "pages": 27,
       "shards": 2
     },
     {
-      "seconds": 1.0266657669999404,
-      "bytes": 4655742,
-      "pages": 12,
-      "shards": 2
+      "seconds": 0.8728474010022182,
+      "bytes": 1156879,
+      "pages": 34,
+      "shards": 1
     },
     {
-      "seconds": 1.4716465549990971,
-      "bytes": 13803834,
-      "pages": 48,
+      "seconds": 1.1430372309987433,
+      "bytes": 13489384,
+      "pages": 40,
       "shards": 4
     },
     {
-      "seconds": 0.940834745999382,
-      "bytes": 6400703,
-      "pages": 35,
-      "shards": 2
-    },
-    {
-      "seconds": 1.1989260670015938,
-      "bytes": 2439937,
+      "seconds": 1.410483870000462,
+      "bytes": 10063648,
       "pages": 28,
-      "shards": 2
-    },
-    {
-      "seconds": 1.2518158220009354,
-      "bytes": 10474054,
-      "pages": 38,
-      "shards": 4
+      "shards": 3
     },
     {
-      "seconds": 1.1628889589992468,
-      "bytes": 1077101,
+      "seconds": 1.0145048790000146,
+      "bytes": 1134517,
       "pages": 34,
       "shards": 1
     },
     {
-      "seconds": 1.117781933000515,
-      "bytes": 7831651,
-      "pages": 22,
+      "seconds": 3.9685035570000764,
+      "bytes": 7473998,
+      "pages": 38,
       "shards": 3
     },
     {
-      "seconds": 1.0544361239990394,
-      "bytes": 7130727,
-      "pages": 44,
+      "seconds": 1.2958799169973645,
+      "bytes": 6217387,
+      "pages": 35,
       "shards": 2
     },
     {
-      "seconds": 1.1150904570004059,
-      "bytes": 10599538,
-      "pages": 29,
-      "shards": 3
+      "seconds": 0.9571121369990578,
+      "bytes": 4079115,
+      "pages": 20,
+      "shards": 2
     },
     {
-      "seconds": 1.0015506020008615,
-      "bytes": 7818449,
-      "pages": 38,
-      "shards": 3
+      "seconds": 1.3280163199997332,
+      "bytes": 13612044,
+      "pages": 47,
+      "shards": 4
     },
     {
-      "seconds": 0.8705007850003312,
-      "bytes": 1070987,
+      "seconds": 0.827956348999578,
+      "bytes": 1165763,
       "pages": 34,
       "shards": 1
     },
     {
-      "seconds": 0.9655942159988626,
-      "bytes": 7298211,
+      "seconds": 1.215623644999141,
+      "bytes": 7281440,
       "pages": 29,
       "shards": 3
     },
     {
-      "seconds": 0.8577407669999957,
-      "bytes": 1078743,
+      "seconds": 1.1079499599982228,
+      "bytes": 10281570,
+      "pages": 37,
+      "shards": 3
+    },
+    {
+      "seconds": 0.9046422840001469,
+      "bytes": 1080144,
       "pages": 30,
       "shards": 1
     },
     {
-      "seconds": 1.3106396379989746,
-      "bytes": 15458880,
-      "pages": 39,
-      "shards": 4
+      "seconds": 1.0989511400002812,
+      "bytes": 7379943,
+      "pages": 33,
+      "shards": 3
     },
     {
-      "seconds": 0.9607973950005544,
-      "bytes": 7512102,
-      "pages": 38,
-      "shards": 3
+      "seconds": 1.0861735590005992,
+      "bytes": 6754175,
+      "pages": 31,
+      "shards": 2
     },
     {
-      "seconds": 2.3653303009996307,
-      "bytes": 1160564,
-      "pages": 33,
-      "shards": 1
+      "seconds": 1.2741842929972336,
+      "bytes": 12412930,
+      "pages": 39,
+      "shards": 4
     }
   ],
   "cache_key": {

trainer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:15e5868d1d2d9bba0011346b8b8fe6a7017e790885cbae0ddebf73fbc62c7dfe
-size 721778495
+oid sha256:5f4a15472e9622ce649a08d61cb56adda487f4944d44a36aff803668640d8309
+size 721759039

training_config.json

@@ -46,8 +46,8 @@
   "cpu_threads": 4,
   "page_rows": 4096,
   "cortex_page_rows": 128,
-  "prefetch_batches": 62,
-  "prefetch_low_water": 31,
+  "prefetch_batches": 61,
+  "prefetch_low_water": 30,
   "memory_pages": 12,
   "cache_gb": 20,
   "force_rebenchmark": false,