Qwen90M step 320000; complete resumable checkpoint

complete.json

@@ -1,18 +1,18 @@
 {
-  "step": 315000,
+  "step": 320000,
   "files": {
     "config.json": "cbc72137da3dae65085a2a8c39ad7634d16bf92d785ca00d283ee82fe32b37e7",
     "generation_config.json": "54ed0d19f6226888c91157964ede8a892bd5ae175bfad85336152b3d9b6cd7c4",
-    "model.safetensors": "7bad7905e71cbf5282827daa6e33886fd079a2b0c210ba5a1021c2c4c8ef5b70",
+    "model.safetensors": "7af0a447c456f6760d4f6e84edc350c5e69416f8c3a7a61824547f910318f805",
     "tokenization_byte.py": "279b9990f77b1eaedc7c1184fe13d60273a713ead856200f1fb9ae323c7b1e03",
     "tokenizer_config.json": "0ecbd4fd39e35125800d806af981ba0cacd8a7ba5f02c9e729c4ecc54af6d300",
     "special_tokens_map.json": "20d4b94d0e657c3f44d9123522c106e7315b6c1a1b6d259478f17bd2918b1263",
     "byte_vocab.json": "57a1cd2e70dd5b7cc6fd2eaa12ebfcd173c1620fa5d45c2c6282511c29b6a5f6",
-    "trainer.pt": "0becabd5ddaf3503de95d1c46e0a2cab7a3ad153cc954b1a5c27591cd4e7fba4",
-    "training_config.json": "65390fe09f12d0188f44175607f7a9d1665082a136656274cb5d054eb998e80b",
-    "performance.json": "f89f82c3f4c44a52fcd532346ea71580902d757e59e1961e9b2bed0030259dba",
-    "shard_scheduler.json": "2059ce5f880963ea177af3f92919853c35636a63fc6fe06b4f84a8f420e442a2",
-    "coverage.json": "1a679a788576b42e3b6606d45a64495b0d61f4444204e639605bcbbd2ffce4ee",
+    "trainer.pt": "38c2f886ccf49b55cd34e215a6a6c1aa8127d46138bf70445d4ca70032231d6c",
+    "training_config.json": "95ba9e002790d11959d1191b36ed994bfa10a6da51c22c772e143bec517a9783",
+    "performance.json": "90ad39d684ac792f235cbfbeef498d4c81f09977d8f89fb8c3bdd41e515f347a",
+    "shard_scheduler.json": "d5343011e97e001a66563c4b9f78ab277480b08424f02858b0f8f0948b4c217f",
+    "coverage.json": "1f8b82802b7d23b4ac70b19b7e71ccea4fd98cb1ced9c8e431325fe81b59768d",
     "README.md": "9ce0f9efacbc46177a7e818b6aed274d84e7145ff944b5c7d6ecd5d33b9d4419"
   },
   "signature": "9fee47918905b0750bee87876aa424f021ccc4507ee8db22a11d01fbc6109063"

coverage.json

@@ -1,107 +1,107 @@
 {
-  "step": 315000,
+  "step": 320000,
   "stats": {
-    "climbmix/blocks": 11340000,
-    "climbmix/targets": 18956161933,
-    "climbmix/input_tokens": 18957363575,
-    "ultra_qa/blocks": 4252500,
-    "ultra_qa/targets": 6937463581,
-    "ultra_qa/input_tokens": 6937476970,
-    "cortex/blocks": 4252500,
-    "cortex/targets": 161788999,
-    "cortex/input_tokens": 6760036876,
-    "ultra_style/blocks": 4252500,
-    "ultra_style/targets": 6726387227,
-    "ultra_style/input_tokens": 6726740294,
-    "openmath/blocks": 2835000,
-    "openmath/targets": 4380942289,
-    "openmath/input_tokens": 4382402714,
-    "rewrite6/blocks": 1417500,
-    "rewrite6/targets": 2323732855,
-    "rewrite6/input_tokens": 2326280728
+    "climbmix/blocks": 11520000,
+    "climbmix/targets": 19257096294,
+    "climbmix/input_tokens": 19258316982,
+    "ultra_qa/blocks": 4320000,
+    "ultra_qa/targets": 7047544993,
+    "ultra_qa/input_tokens": 7047558585,
+    "cortex/blocks": 4320000,
+    "cortex/targets": 164372097,
+    "cortex/input_tokens": 6867485637,
+    "ultra_style/blocks": 4320000,
+    "ultra_style/targets": 6833023501,
+    "ultra_style/input_tokens": 6833381920,
+    "openmath/blocks": 2880000,
+    "openmath/targets": 4450371997,
+    "openmath/input_tokens": 4451855399,
+    "rewrite6/blocks": 1440000,
+    "rewrite6/targets": 2360725625,
+    "rewrite6/input_tokens": 2363313839
   },
   "sources": {
     "climbmix": {
-      "page": 1553,
-      "row": 1737,
+      "page": 1578,
+      "row": 267,
       "start": {
-        "file": 75,
+        "file": 76,
         "reader": {
           "format": "parquet-row-v1",
-          "row_group": 13,
+          "row_group": 31,
           "row": 1024
         }
       },
       "epoch": 0,
-      "offset": 18432,
-      "documents": 6362825
+      "offset": 2048,
+      "documents": 6463755
     },
     "rewrite6": {
-      "page": 33,
-      "row": 2804,
+      "page": 47,
+      "row": 2882,
       "start": {
         "file": 0,
         "reader": {
           "format": "jsonl-row-v1",
-          "row": 135168
+          "row": 192512
         }
       },
       "epoch": 17,
-      "offset": 0,
-      "documents": 3629840
+      "offset": 2048,
+      "documents": 3687262
     },
     "openmath": {
-      "page": 970,
-      "row": 3337,
+      "page": 986,
+      "row": 621,
       "start": {
         "file": 9,
         "reader": {
           "format": "parquet-row-v1",
-          "row_group": 43,
-          "row": 270
+          "row_group": 108,
+          "row": 806
         }
       },
       "epoch": 0,
-      "offset": 2048,
-      "documents": 3976457
+      "offset": 0,
+      "documents": 4039277
     },
     "ultra_qa": {
-      "page": 419,
-      "row": 2676,
+      "page": 426,
+      "row": 1436,
       "start": {
         "file": 3,
         "reader": {
           "format": "parquet-row-v1",
           "row_group": 2,
-          "row": 28083
+          "row": 56755
         }
       },
       "epoch": 0,
-      "offset": 2048,
-      "documents": 1718900
+      "offset": 4096,
+      "documents": 1746332
     },
     "ultra_style": {
-      "page": 640,
-      "row": 3189,
+      "page": 650,
+      "row": 3626,
       "start": {
         "file": 3,
         "reader": {
           "format": "parquet-row-v1",
-          "row_group": 6,
-          "row": 56507
+          "row_group": 7,
+          "row": 10908
         }
       },
       "epoch": 0,
-      "offset": 0,
-      "documents": 2624629
+      "offset": 2048,
+      "documents": 2666026
     },
     "cortex": {
-      "page": 82476,
-      "row": 126,
+      "page": 83788,
+      "row": 117,
       "start": null,
       "epoch": 0,
       "offset": 0,
-      "documents": 10557054
+      "documents": 10724981
     }
   }
 }

latest.json

@@ -1 +1 @@
-{"step": 315000, "files": {"config.json": "cbc72137da3dae65085a2a8c39ad7634d16bf92d785ca00d283ee82fe32b37e7", "generation_config.json": "54ed0d19f6226888c91157964ede8a892bd5ae175bfad85336152b3d9b6cd7c4", "model.safetensors": "7bad7905e71cbf5282827daa6e33886fd079a2b0c210ba5a1021c2c4c8ef5b70", "tokenization_byte.py": "279b9990f77b1eaedc7c1184fe13d60273a713ead856200f1fb9ae323c7b1e03", "tokenizer_config.json": "0ecbd4fd39e35125800d806af981ba0cacd8a7ba5f02c9e729c4ecc54af6d300", "special_tokens_map.json": "20d4b94d0e657c3f44d9123522c106e7315b6c1a1b6d259478f17bd2918b1263", "byte_vocab.json": "57a1cd2e70dd5b7cc6fd2eaa12ebfcd173c1620fa5d45c2c6282511c29b6a5f6", "trainer.pt": "0becabd5ddaf3503de95d1c46e0a2cab7a3ad153cc954b1a5c27591cd4e7fba4", "training_config.json": "65390fe09f12d0188f44175607f7a9d1665082a136656274cb5d054eb998e80b", "performance.json": "f89f82c3f4c44a52fcd532346ea71580902d757e59e1961e9b2bed0030259dba", "shard_scheduler.json": "2059ce5f880963ea177af3f92919853c35636a63fc6fe06b4f84a8f420e442a2", "coverage.json": "1a679a788576b42e3b6606d45a64495b0d61f4444204e639605bcbbd2ffce4ee", "README.md": "9ce0f9efacbc46177a7e818b6aed274d84e7145ff944b5c7d6ecd5d33b9d4419"}, "signature": "9fee47918905b0750bee87876aa424f021ccc4507ee8db22a11d01fbc6109063", "folder": "step-000000315000"}
+{"step": 320000, "files": {"config.json": "cbc72137da3dae65085a2a8c39ad7634d16bf92d785ca00d283ee82fe32b37e7", "generation_config.json": "54ed0d19f6226888c91157964ede8a892bd5ae175bfad85336152b3d9b6cd7c4", "model.safetensors": "7af0a447c456f6760d4f6e84edc350c5e69416f8c3a7a61824547f910318f805", "tokenization_byte.py": "279b9990f77b1eaedc7c1184fe13d60273a713ead856200f1fb9ae323c7b1e03", "tokenizer_config.json": "0ecbd4fd39e35125800d806af981ba0cacd8a7ba5f02c9e729c4ecc54af6d300", "special_tokens_map.json": "20d4b94d0e657c3f44d9123522c106e7315b6c1a1b6d259478f17bd2918b1263", "byte_vocab.json": "57a1cd2e70dd5b7cc6fd2eaa12ebfcd173c1620fa5d45c2c6282511c29b6a5f6", "trainer.pt": "38c2f886ccf49b55cd34e215a6a6c1aa8127d46138bf70445d4ca70032231d6c", "training_config.json": "95ba9e002790d11959d1191b36ed994bfa10a6da51c22c772e143bec517a9783", "performance.json": "90ad39d684ac792f235cbfbeef498d4c81f09977d8f89fb8c3bdd41e515f347a", "shard_scheduler.json": "d5343011e97e001a66563c4b9f78ab277480b08424f02858b0f8f0948b4c217f", "coverage.json": "1f8b82802b7d23b4ac70b19b7e71ccea4fd98cb1ced9c8e431325fe81b59768d", "README.md": "9ce0f9efacbc46177a7e818b6aed274d84e7145ff944b5c7d6ecd5d33b9d4419"}, "signature": "9fee47918905b0750bee87876aa424f021ccc4507ee8db22a11d01fbc6109063", "folder": "step-000000320000"}

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7bad7905e71cbf5282827daa6e33886fd079a2b0c210ba5a1021c2c4c8ef5b70
+oid sha256:7af0a447c456f6760d4f6e84edc350c5e69416f8c3a7a61824547f910318f805
 size 360622744

performance.json

@@ -49,11 +49,11 @@
   "selected": {
     "microbatch": 86,
     "mode": "compiled",
-    "microbatch_seconds": 0.8979115860001912,
-    "peak_gb": 85.39889430999756,
+    "microbatch_seconds": 0.9004576890000067,
+    "peak_gb": 85.39913082122803,
     "fits_budget": true,
-    "estimated_update_seconds": 0.953845084999557,
-    "input_tokens_per_second": 196152.9428354681
+    "estimated_update_seconds": 0.9568634569999972,
+    "input_tokens_per_second": 195598.30756245417
   },
   "measurements": [
     {

shard_scheduler.json

@@ -22,8 +22,8 @@
       "tokens_per_second": 1062967.6168812835,
       "note": "Bounded fresh sample; full-page latency is an extrapolation.",
       "estimated_pages_for_buffer": 1,
-      "required_token_positions_per_second": 76228.00974719698,
-      "required_pages_per_second_upper_estimate": 0.006607250686762633
+      "required_token_positions_per_second": 75458.65267281262,
+      "required_pages_per_second_upper_estimate": 0.006540564765472674
     },
     {
       "source": "rewrite6",
@@ -36,8 +36,8 @@
       "tokens_per_second": 1206672.263377472,
       "note": "Bounded fresh sample; full-page latency is an extrapolation.",
       "estimated_pages_for_buffer": 1,
-      "required_token_positions_per_second": 9528.501218399622,
-      "required_pages_per_second_upper_estimate": 0.00380106159980837
+      "required_token_positions_per_second": 9432.331584101577,
+      "required_pages_per_second_upper_estimate": 0.003762698094822713
     },
     {
       "source": "openmath",
@@ -50,8 +50,8 @@
       "tokens_per_second": 410042.7581134458,
       "note": "Bounded fresh sample; full-page latency is an extrapolation.",
       "estimated_pages_for_buffer": 1,
-      "required_token_positions_per_second": 19057.002436799245,
-      "required_pages_per_second_upper_estimate": 0.004408502121965669
+      "required_token_positions_per_second": 18864.663168203155,
+      "required_pages_per_second_upper_estimate": 0.004364007817231477
     },
     {
       "source": "ultra_qa",
@@ -64,8 +64,8 @@
       "tokens_per_second": 2163486.5624068812,
       "note": "Bounded fresh sample; full-page latency is an extrapolation.",
       "estimated_pages_for_buffer": 1,
-      "required_token_positions_per_second": 28585.503655198867,
-      "required_pages_per_second_upper_estimate": 0.0017407071960547304
+      "required_token_positions_per_second": 28296.994752304734,
+      "required_pages_per_second_upper_estimate": 0.0017231385175577071
     },
     {
       "source": "ultra_style",
@@ -78,8 +78,8 @@
       "tokens_per_second": 1546694.2737633225,
       "note": "Bounded fresh sample; full-page latency is an extrapolation.",
       "estimated_pages_for_buffer": 1,
-      "required_token_positions_per_second": 28585.503655198867,
-      "required_pages_per_second_upper_estimate": 0.002200591201115855
+      "required_token_positions_per_second": 28296.994752304734,
+      "required_pages_per_second_upper_estimate": 0.002178380987127306
     },
     {
       "source": "cortex",
@@ -92,8 +92,8 @@
       "tokens_per_second": 1015177.8395898147,
       "note": "Bounded fresh sample; full-page latency is an extrapolation.",
       "estimated_pages_for_buffer": 22,
-      "required_token_positions_per_second": 28585.503655198867,
-      "required_pages_per_second_upper_estimate": 0.35493628587107623
+      "required_token_positions_per_second": 28296.994752304734,
+      "required_pages_per_second_upper_estimate": 0.35135397087431536
     }
   ],
   "gpu_measurement": "Selected forward/backward estimate; replaced by live successful update timings.",
@@ -155,413 +155,269 @@
   "shard_benchmark_scope": "Local packing/read measurements; upload capacity measured only on real data commits. Targets larger than available pages are sample-limited.",
   "estimated_fresh_generation_batch_seconds": 0.1434574083060457,
   "selected": {
-    "prefetch_batches": 63,
+    "prefetch_batches": 62,
     "prefetch_capacity_limit": 512,
     "batch_memory_estimate_bytes": 690670,
-    "producer_batch_seconds": 0.11422511410142988,
-    "producer_p95_seconds": 0.3618045629991684,
-    "gpu_update_seconds": 0.9672035285259575,
-    "producer_batches_per_second": 8.754642163123757,
-    "gpu_batches_per_second": 1.033908552343709,
-    "producer_headroom": 8.467520791155419,
+    "producer_batch_seconds": 0.0511901649687303,
+    "producer_p95_seconds": 0.18669305799994618,
+    "gpu_update_seconds": 0.9770648877032472,
+    "producer_batches_per_second": 19.535002487506215,
+    "gpu_batches_per_second": 1.0234734791776885,
+    "producer_headroom": 19.086965011737917,
     "sustainable": true,
-    "buffer_seconds": 60.93382229713532,
+    "buffer_seconds": 60.57802303760133,
     "buffer_drain_seconds": null,
     "shard_target_mib": 16,
-    "upload_pages_per_commit": 85,
+    "upload_pages_per_commit": 83,
     "max_pending_pages": 4096,
-    "generation_bytes_per_second": 354783.7727048669,
-    "upload_bytes_per_second": 5691532.768220302,
+    "generation_bytes_per_second": 395163.83161593793,
+    "upload_bytes_per_second": 5372382.916989888,
     "upload_sustainable": true,
     "logical_page_rows": 4096,
     "logical_cortex_page_rows": 128,
-    "estimated_full_shards_per_hour": 76.12833867892746,
+    "estimated_full_shards_per_hour": 84.79295932158092,
     "selection_policy": "Smallest shard within 90% of measured local peak, enlarged to amortize 30 seconds of measured page production; bounded to 256 MiB."
   },
   "upload_measurements": [
     {
-      "seconds": 1.11581381400174,
-      "bytes": 5950902,
-      "pages": 37,
-      "shards": 2
+      "seconds": 1.0731404800000064,
+      "bytes": 44464,
+      "pages": 1,
+      "shards": 1
     },
     {
-      "seconds": 0.9459429260023171,
-      "bytes": 2335478,
-      "pages": 29,
-      "shards": 2
+      "seconds": 1.131805757000052,
+      "bytes": 882364,
+      "pages": 25,
+      "shards": 1
     },
     {
-      "seconds": 1.2669872590049636,
-      "bytes": 5719604,
-      "pages": 21,
-      "shards": 2
+      "seconds": 1.1303505340000584,
+      "bytes": 11613053,
+      "pages": 29,
+      "shards": 3
     },
     {
-      "seconds": 1.032783564995043,
-      "bytes": 5149031,
-      "pages": 45,
+      "seconds": 0.9671371199999612,
+      "bytes": 2609325,
+      "pages": 34,
       "shards": 2
     },
     {
-      "seconds": 1.4498847849972663,
-      "bytes": 7603712,
-      "pages": 36,
-      "shards": 3
-    },
-    {
-      "seconds": 1.3791402839997318,
-      "bytes": 11555501,
-      "pages": 30,
-      "shards": 3
+      "seconds": 0.9158913300000222,
+      "bytes": 6129878,
+      "pages": 32,
+      "shards": 2
     },
     {
-      "seconds": 1.030656152004667,
-      "bytes": 6775920,
-      "pages": 30,
+      "seconds": 1.5951903079999283,
+      "bytes": 6553135,
+      "pages": 34,
       "shards": 3
     },
     {
-      "seconds": 1.0988766239970573,
-      "bytes": 6218344,
-      "pages": 37,
+      "seconds": 0.9309580040001038,
+      "bytes": 6008731,
+      "pages": 35,
       "shards": 2
     },
     {
-      "seconds": 1.1409971860048245,
-      "bytes": 830736,
-      "pages": 27,
-      "shards": 1
+      "seconds": 1.3192515309999635,
+      "bytes": 6183122,
+      "pages": 35,
+      "shards": 2
     },
     {
-      "seconds": 1.1426053430041065,
-      "bytes": 7670085,
-      "pages": 37,
+      "seconds": 1.2959069240000645,
+      "bytes": 8294236,
+      "pages": 30,
       "shards": 3
     },
     {
-      "seconds": 1.1845013959973585,
-      "bytes": 10539391,
-      "pages": 31,
+      "seconds": 1.3743943120000495,
+      "bytes": 10663078,
+      "pages": 35,
       "shards": 3
     },
     {
-      "seconds": 0.8672450950034545,
-      "bytes": 606235,
-      "pages": 21,
+      "seconds": 0.8003151779998916,
+      "bytes": 1078242,
+      "pages": 33,
       "shards": 1
     },
     {
-      "seconds": 1.3488649450009689,
-      "bytes": 13749450,
-      "pages": 49,
-      "shards": 4
+      "seconds": 1.0970220180001888,
+      "bytes": 7456296,
+      "pages": 36,
+      "shards": 3
     },
     {
-      "seconds": 0.8790644170003361,
-      "bytes": 1116619,
+      "seconds": 1.0635986559996127,
+      "bytes": 6186061,
       "pages": 35,
-      "shards": 1
-    },
-    {
-      "seconds": 1.4015686020065914,
-      "bytes": 11114007,
-      "pages": 33,
-      "shards": 4
-    },
-    {
-      "seconds": 1.0096355339992442,
-      "bytes": 5988067,
-      "pages": 29,
       "shards": 2
     },
     {
-      "seconds": 1.234346370001731,
-      "bytes": 974463,
-      "pages": 34,
-      "shards": 1
-    },
-    {
-      "seconds": 1.1329207589951693,
-      "bytes": 7046164,
-      "pages": 25,
+      "seconds": 1.1765199270002995,
+      "bytes": 6893561,
+      "pages": 28,
       "shards": 3
     },
     {
-      "seconds": 1.0467858490010258,
-      "bytes": 11558244,
-      "pages": 41,
-      "shards": 4
-    },
-    {
-      "seconds": 1.1082996040058788,
-      "bytes": 6095964,
-      "pages": 36,
-      "shards": 2
-    },
-    {
-      "seconds": 1.0661074119998375,
-      "bytes": 7367457,
-      "pages": 30,
+      "seconds": 3.6869929439999396,
+      "bytes": 11547756,
+      "pages": 29,
       "shards": 3
     },
     {
-      "seconds": 0.7073749000046519,
-      "bytes": 1147032,
-      "pages": 34,
+      "seconds": 0.8728801329998532,
+      "bytes": 1038671,
+      "pages": 33,
       "shards": 1
     },
     {
-      "seconds": 2.1967378000044846,
-      "bytes": 12230231,
-      "pages": 35,
+      "seconds": 1.359243317000164,
+      "bytes": 7794752,
+      "pages": 37,
       "shards": 4
     },
     {
-      "seconds": 1.34678302899556,
-      "bytes": 6766636,
-      "pages": 30,
-      "shards": 2
-    },
-    {
-      "seconds": 0.8585810450022109,
-      "bytes": 306974,
-      "pages": 9,
+      "seconds": 0.911286476999976,
+      "bytes": 856023,
+      "pages": 24,
       "shards": 1
     },
     {
-      "seconds": 1.175691934004135,
-      "bytes": 6239046,
-      "pages": 37,
-      "shards": 2
-    },
-    {
-      "seconds": 1.251433448996977,
-      "bytes": 7613150,
-      "pages": 31,
+      "seconds": 1.8453115359998264,
+      "bytes": 9879354,
+      "pages": 35,
       "shards": 3
     },
     {
-      "seconds": 0.8646319059989764,
-      "bytes": 4940868,
-      "pages": 32,
-      "shards": 2
-    },
-    {
-      "seconds": 0.9655562699990696,
-      "bytes": 7648817,
-      "pages": 39,
+      "seconds": 1.0517817119998654,
+      "bytes": 7714712,
+      "pages": 37,
       "shards": 3
     },
     {
-      "seconds": 1.2715992000012193,
-      "bytes": 6184683,
-      "pages": 36,
-      "shards": 2
-    },
-    {
-      "seconds": 0.9518272349960171,
-      "bytes": 6809793,
-      "pages": 31,
-      "shards": 2
-    },
-    {
-      "seconds": 1.123792185993807,
-      "bytes": 11983615,
+      "seconds": 0.9509194960000968,
+      "bytes": 6981450,
       "pages": 37,
-      "shards": 4
-    },
-    {
-      "seconds": 1.1926247609953862,
-      "bytes": 798107,
-      "pages": 26,
-      "shards": 1
-    },
-    {
-      "seconds": 0.9836393130026408,
-      "bytes": 5980946,
-      "pages": 38,
       "shards": 2
     },
     {
-      "seconds": 1.1501939880035934,
-      "bytes": 7275318,
+      "seconds": 1.187183233000269,
+      "bytes": 9995831,
       "pages": 33,
       "shards": 3
     },
     {
-      "seconds": 1.0238104009986273,
-      "bytes": 4591810,
-      "pages": 32,
-      "shards": 2
-    },
-    {
-      "seconds": 2.234572448003746,
-      "bytes": 13274402,
-      "pages": 35,
-      "shards": 4
-    },
-    {
-      "seconds": 1.07868888899975,
-      "bytes": 1300513,
-      "pages": 36,
-      "shards": 1
-    },
-    {
-      "seconds": 1.4324448540064623,
-      "bytes": 9935435,
-      "pages": 35,
+      "seconds": 1.3044787690000703,
+      "bytes": 7311631,
+      "pages": 28,
       "shards": 3
     },
     {
-      "seconds": 1.2297556650009938,
-      "bytes": 7626325,
+      "seconds": 0.7353289520001454,
+      "bytes": 1217005,
       "pages": 34,
-      "shards": 3
-    },
-    {
-      "seconds": 0.8381233459949726,
-      "bytes": 1148268,
-      "pages": 32,
       "shards": 1
     },
     {
-      "seconds": 0.9499172950017964,
-      "bytes": 7609450,
-      "pages": 39,
+      "seconds": 1.1706305770003382,
+      "bytes": 7901550,
+      "pages": 37,
       "shards": 3
     },
     {
-      "seconds": 0.9056497909987229,
-      "bytes": 4987206,
-      "pages": 25,
+      "seconds": 1.067005848000008,
+      "bytes": 5080800,
+      "pages": 32,
       "shards": 2
     },
     {
-      "seconds": 1.4441368190018693,
-      "bytes": 11962393,
-      "pages": 36,
-      "shards": 3
-    },
-    {
-      "seconds": 1.1221481789980317,
-      "bytes": 7553276,
-      "pages": 38,
-      "shards": 3
-    },
-    {
-      "seconds": 0.8207161469981656,
-      "bytes": 1164563,
-      "pages": 36,
-      "shards": 1
-    },
-    {
-      "seconds": 1.0845683700026711,
-      "bytes": 10426904,
-      "pages": 28,
-      "shards": 3
+      "seconds": 0.9038640889998533,
+      "bytes": 6072537,
+      "pages": 27,
+      "shards": 2
     },
     {
-      "seconds": 1.0798400059939013,
-      "bytes": 7684665,
+      "seconds": 1.318457908000255,
+      "bytes": 13323724,
       "pages": 39,
-      "shards": 3
+      "shards": 4
     },
     {
-      "seconds": 1.0031564969976898,
-      "bytes": 997756,
-      "pages": 28,
+      "seconds": 0.8108393100001194,
+      "bytes": 1203088,
+      "pages": 33,
       "shards": 1
     },
     {
-      "seconds": 1.3090939850007999,
-      "bytes": 13205063,
-      "pages": 36,
-      "shards": 4
-    },
-    {
-      "seconds": 1.0676715709996643,
-      "bytes": 4977396,
-      "pages": 37,
+      "seconds": 1.1118799440000657,
+      "bytes": 10497270,
+      "pages": 30,
       "shards": 3
     },
     {
-      "seconds": 0.9372590880011558,
-      "bytes": 6217543,
-      "pages": 37,
+      "seconds": 0.8437159310005882,
+      "bytes": 2537590,
+      "pages": 35,
       "shards": 2
     },
     {
-      "seconds": 1.0400478770025074,
-      "bytes": 7313545,
-      "pages": 31,
-      "shards": 3
-    },
-    {
-      "seconds": 0.8696056030021282,
-      "bytes": 1079259,
-      "pages": 34,
-      "shards": 1
+      "seconds": 1.248541894000482,
+      "bytes": 6395911,
+      "pages": 35,
+      "shards": 2
     },
     {
-      "seconds": 1.1580835760032642,
-      "bytes": 10392878,
-      "pages": 30,
+      "seconds": 1.171668709000187,
+      "bytes": 7290583,
+      "pages": 33,
       "shards": 3
     },
     {
-      "seconds": 1.3803016649981146,
-      "bytes": 13473573,
-      "pages": 39,
-      "shards": 4
-    },
-    {
-      "seconds": 0.9864481259937747,
-      "bytes": 1047459,
-      "pages": 35,
-      "shards": 1
+      "seconds": 1.2945431420002933,
+      "bytes": 10577919,
+      "pages": 37,
+      "shards": 3
     },
     {
-      "seconds": 1.009063662000699,
-      "bytes": 7464078,
-      "pages": 31,
-      "shards": 3
+      "seconds": 1.2328179739997722,
+      "bytes": 5713232,
+      "pages": 27,
+      "shards": 2
     },
     {
-      "seconds": 1.1531412479962455,
-      "bytes": 5169545,
-      "pages": 35,
+      "seconds": 0.9817969460000313,
+      "bytes": 2633705,
+      "pages": 36,
       "shards": 2
     },
     {
-      "seconds": 1.040382787999988,
-      "bytes": 5992429,
-      "pages": 33,
+      "seconds": 1.1478959790001682,
+      "bytes": 6105703,
+      "pages": 35,
       "shards": 2
     },
     {
-      "seconds": 1.0942238300049212,
-      "bytes": 7544195,
-      "pages": 32,
+      "seconds": 1.3559997279999152,
+      "bytes": 10442134,
+      "pages": 30,
       "shards": 3
     },
     {
-      "seconds": 1.183839971999987,
-      "bytes": 6786466,
-      "pages": 32,
+      "seconds": 0.8416516239994962,
+      "bytes": 2569575,
+      "pages": 36,
       "shards": 2
     },
     {
-      "seconds": 1.5759701440038043,
-      "bytes": 11667943,
-      "pages": 39,
-      "shards": 4
-    },
-    {
-      "seconds": 1.0282209380020504,
-      "bytes": 1021880,
+      "seconds": 1.165861663999749,
+      "bytes": 11609605,
       "pages": 32,
-      "shards": 1
+      "shards": 3
     }
   ],
   "cache_key": {
@@ -606,7 +462,7 @@
   },
   "cache_reused": true,
   "resume_probe_seconds": [
-    1.1575575899996693,
-    0.014994120000665134
+    2.358020696999972,
+    0.013266629999975521
   ]
 }

trainer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0becabd5ddaf3503de95d1c46e0a2cab7a3ad153cc954b1a5c27591cd4e7fba4
-size 721773887
+oid sha256:38c2f886ccf49b55cd34e215a6a6c1aa8127d46138bf70445d4ca70032231d6c
+size 721783871

training_config.json

@@ -46,7 +46,7 @@
   "cpu_threads": 4,
   "page_rows": 4096,
   "cortex_page_rows": 128,
-  "prefetch_batches": 63,
+  "prefetch_batches": 62,
   "prefetch_low_water": 31,
   "memory_pages": 12,
   "cache_gb": 20,