hyb1: results of w2

results/hyb1/w2/done.txt

@@ -0,0 +1 @@
+done

results/hyb1/w2/logs/auto.log

@@ -0,0 +1,10 @@
+15:58 START w2
+15:58 stockfish ok
+16:00 DATA_DONE
+16:09 PREP FASTPREP_DONE 392086372 537210114
+16:09 RUN hyb1 TOTAL=60000 H=1000 LR=1e-3 OLR=0.7 BS=4096 CK=5 WAIT=300 TAKEOVER=900
+16:35 CRASH hyb1 1 RuntimeError: commit hyb1: r40 w2 failed for good
+16:35 PREP_FIXED castling FASTPREP_DONE 392086372 567331261
+16:35 CRASH hyb1 2 FileNotFoundError: [Errno 2] No such file or directory: 'prepA/part_00721.npz'
+18:08 TRAINED hyb1
+18:14 MATCH hyb1_sf3190 

results/hyb1/w2/logs/diloco_hyb1.log

@@ -0,0 +1,90 @@
+16:09:56 w2 387,134,719 positions from 922 parts; GPU memory 16.6 GB
+16:10:11 w2 checkpoint at round 35
+16:10:16 w2 replayed up to round 37
+16:16:09 w2 joining at round 38 leader 0
+16:21:06 w2 round 39 step 39000/60000 inner 261s (261 ms/step) sync 33s used [0, 1, 2, 3] lr 2.88e-04 loss [1.2724, 0.5411]
+16:24:38 w2 round 40 step 40000/60000 inner 203s (203 ms/step) sync 7s used [0, 1, 2, 3] lr 2.65e-04 loss [1.2753, 0.543]
+16:28:44 w2 commit hyb1: r40 w2 failed (BadRequestError: (Request ID: Root=1-6aef51bc-1d526b8248b03816432b6ed8;8f8a0841-14de-419d-8682-c07aaaf28901)
+
+Bad request for commit endpoint:
+Private storage limit re), retry 1
+16:28:47 w2 commit hyb1: r40 w2 failed (BadRequestError: (Request ID: Root=1-6aef51bf-108143af2f9be0cf7ea04bae;5f32f495-c0e3-4095-9a10-427907405175)
+
+Bad request for commit endpoint:
+Private storage limit re), retry 2
+16:28:54 w2 commit hyb1: r40 w2 failed (BadRequestError: (Request ID: Root=1-6aef51c6-27541b3d4ce4c6cb22aed425;028318be-4e24-4161-8629-18942deb9bc4)
+
+Bad request for commit endpoint:
+Private storage limit re), retry 3
+16:29:06 w2 commit hyb1: r40 w2 failed (BadRequestError: (Request ID: Root=1-6aef51d2-3ba4d4bb1cd0ba802d84cead;1915ca48-aa36-4ec9-bb27-6aa63d833269)
+
+Bad request for commit endpoint:
+Private storage limit re), retry 4
+16:29:31 w2 commit hyb1: r40 w2 failed (BadRequestError: (Request ID: Root=1-6aef51eb-607e2dc32bd796d7268ae04d;3be6011d-7d84-4745-9540-1bf19fa629cb)
+
+Bad request for commit endpoint:
+Private storage limit re), retry 5
+16:30:20 w2 commit hyb1: r40 w2 failed (BadRequestError: (Request ID: Root=1-6aef521b-797e04d878eaac78376b0394;8f0d4110-c3a2-4d60-9454-0090fdac7715)
+
+Bad request for commit endpoint:
+Private storage limit re), retry 6
+16:31:20 w2 commit hyb1: r40 w2 failed (BadRequestError: (Request ID: Root=1-6aef5258-36b4463608829ec570a6bc8d;60725246-17e1-41d4-98f7-f5f1e1bfba62)
+
+Bad request for commit endpoint:
+Private storage limit re), retry 7
+16:32:20 w2 commit hyb1: r40 w2 failed (BadRequestError: (Request ID: Root=1-6aef5294-43e4d0aa7e19c1036e6aadad;9e08c79d-250f-473c-b1ec-6ee5efb155b1)
+
+Bad request for commit endpoint:
+Private storage limit re), retry 8
+16:33:21 w2 commit hyb1: r40 w2 failed (BadRequestError: (Request ID: Root=1-6aef52d0-5c46877c589323f13ed99883;b39e15e3-1db4-44cf-9fab-5646842cac03)
+
+Bad request for commit endpoint:
+Private storage limit re), retry 9
+16:34:21 w2 commit hyb1: r40 w2 failed (BadRequestError: (Request ID: Root=1-6aef530d-5fc0af8d540660ff443c41c2;dec31d5c-4c91-4535-ae64-432555e9fea6)
+
+Bad request for commit endpoint:
+Private storage limit re), retry 10
+Traceback (most recent call last):
+  File "/marimo/chess/code/diloco2.py", line 293, in <module>
+    put([add(f"{ROOT}/r{r:05d}/w{WID}.pt", blob(own))], f"{run}: r{r} w{WID}")
+    ~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/marimo/chess/code/diloco2.py", line 72, in put
+    retry(lambda: api.create_commit(REPO, operations=ops, commit_message=msg), "commit " + msg)
+    ~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/marimo/chess/code/diloco2.py", line 56, in retry
+    raise RuntimeError(what + " failed for good")
+RuntimeError: commit hyb1: r40 w2 failed for good
+Traceback (most recent call last):
+  File "/marimo/chess/code/diloco2.py", line 137, in <module>
+    D, sizes, nparts = load(groups)
+                       ~~~~^^^^^^^^
+  File "/marimo/chess/code/diloco2.py", line 102, in load
+    z = np.load(f)                             # first pass: sizes (+ the PV-ply filter) to preallocate
+  File "/usr/local/lib/python3.13/site-packages/numpy/lib/_npyio_impl.py", line 454, in load
+    fid = stack.enter_context(open(os.fspath(file), "rb"))
+                              ~~~~^^^^^^^^^^^^^^^^^^^^^^^
+FileNotFoundError: [Errno 2] No such file or directory: 'prepA/part_00721.npz'
+16:36:43 w2 392,036,107 positions from 922 parts; GPU memory 16.8 GB
+16:36:47 w2 checkpoint at round 40
+16:36:47 w2 replayed up to round 40
+16:51:36 w2 joining at round 41 leader 0
+16:55:58 w2 round 42 step 42000/60000 inner 208s (208 ms/step) sync 52s used [0, 1, 2, 3] lr 2.22e-04 loss [1.2827, 0.5365]
+16:59:41 w2 round 43 step 43000/60000 inner 203s (203 ms/step) sync 19s used [0, 1, 2, 3] lr 2.02e-04 loss [1.2787, 0.5395]
+17:03:40 w2 round 44 step 44000/60000 inner 203s (203 ms/step) sync 35s used [0, 1, 2, 3] lr 1.82e-04 loss [1.2752, 0.5444]
+17:07:22 w2 round 45 step 45000/60000 inner 203s (203 ms/step) sync 17s used [0, 1, 2, 3] lr 1.64e-04 loss [1.27, 0.5465]
+17:11:00 w2 round 46 step 46000/60000 inner 203s (203 ms/step) sync 13s used [0, 1, 2, 3] lr 1.46e-04 loss [1.2978, 0.5403]
+17:16:14 w2 round 47 step 47000/60000 inner 203s (203 ms/step) sync 110s used [0, 1, 2, 3] lr 1.29e-04 loss [1.2528, 0.5457]
+17:19:51 w2 round 48 step 48000/60000 inner 203s (203 ms/step) sync 12s used [0, 1, 2, 3] lr 1.14e-04 loss [1.2358, 0.5391]
+17:23:59 w2 round 49 step 49000/60000 inner 203s (203 ms/step) sync 43s used [0, 1, 2, 3] lr 9.91e-05 loss [1.2309, 0.5417]
+17:28:29 w2 round 50 step 50000/60000 inner 203s (203 ms/step) sync 65s used [0, 1, 2, 3] lr 8.57e-05 loss [1.2389, 0.5363]
+17:32:40 w2 round 51 step 51000/60000 inner 203s (203 ms/step) sync 46s used [0, 1, 2, 3] lr 7.34e-05 loss [1.2831, 0.5447]
+17:36:32 w2 round 52 step 52000/60000 inner 203s (203 ms/step) sync 28s used [0, 1, 2, 3] lr 6.24e-05 loss [1.2318, 0.5406]
+17:40:36 w2 round 53 step 53000/60000 inner 203s (203 ms/step) sync 39s used [0, 1, 2, 3] lr 5.26e-05 loss [1.2417, 0.5449]
+17:45:06 w2 round 54 step 54000/60000 inner 203s (203 ms/step) sync 65s used [0, 1, 2, 3] lr 4.40e-05 loss [1.2784, 0.5398]
+17:48:38 w2 round 55 step 55000/60000 inner 203s (203 ms/step) sync 7s used [0, 1, 2, 3] lr 3.67e-05 loss [1.2501, 0.5421]
+17:52:21 w2 round 56 step 56000/60000 inner 203s (203 ms/step) sync 18s used [0, 1, 2, 3] lr 3.07e-05 loss [1.2323, 0.5375]
+17:56:32 w2 round 57 step 57000/60000 inner 203s (203 ms/step) sync 46s used [0, 1, 2, 3] lr 2.60e-05 loss [1.2518, 0.5364]
+18:01:05 w2 round 58 step 58000/60000 inner 203s (203 ms/step) sync 69s used [0, 1, 2, 3] lr 2.27e-05 loss [1.2496, 0.5374]
+18:04:54 w2 round 59 step 59000/60000 inner 203s (203 ms/step) sync 23s used [0, 1, 2, 3] lr 2.07e-05 loss [1.25, 0.5412]
+18:08:34 w2 round 60 step 60000/60000 inner 203s (203 ms/step) sync 16s used [0, 1, 2, 3] lr 2.00e-05 loss [1.2507, 0.5362]
+18:08:34 w2 DILOCO_DONE

results/hyb1/w2/logs/fastprep.log

@@ -0,0 +1,20 @@
+1/922 row groups, 177,532 roots, 0 PV positions, 8s
+51/922 row groups, 13,744,059 roots, 18,053,545 PV positions, 26s
+101/922 row groups, 29,537,708 roots, 38,333,025 PV positions, 45s
+151/922 row groups, 46,620,892 roots, 61,906,632 PV positions, 64s
+201/922 row groups, 63,820,785 roots, 84,461,546 PV positions, 84s
+251/922 row groups, 81,922,064 roots, 109,479,260 PV positions, 105s
+301/922 row groups, 100,488,623 roots, 132,702,549 PV positions, 125s
+351/922 row groups, 118,971,326 roots, 156,935,434 PV positions, 146s
+401/922 row groups, 138,519,685 roots, 183,828,582 PV positions, 170s
+451/922 row groups, 169,398,548 roots, 227,613,909 PV positions, 211s
+501/922 row groups, 192,730,796 roots, 259,201,439 PV positions, 248s
+551/922 row groups, 215,576,641 roots, 292,656,626 PV positions, 288s
+601/922 row groups, 238,322,925 roots, 323,326,449 PV positions, 326s
+651/922 row groups, 260,430,465 roots, 356,174,885 PV positions, 363s
+701/922 row groups, 282,766,659 roots, 386,916,181 PV positions, 402s
+751/922 row groups, 305,763,381 roots, 420,325,215 PV positions, 441s
+801/922 row groups, 330,004,960 roots, 452,564,760 PV positions, 467s
+851/922 row groups, 354,498,498 roots, 487,430,635 PV positions, 480s
+901/922 row groups, 380,851,882 roots, 522,525,782 PV positions, 496s
+FASTPREP_DONE 392086372 537210114

results/hyb1/w2/logs/hyb1_sf3190.log

@@ -0,0 +1,71 @@
+game 1/32 sf3190 - gm2b16 1/2-1/2 (89 plies) score gm2b16 0.5/1
+game 2/32 sf3190 - gm2b16 1/2-1/2 (110 plies) score gm2b16 1.0/2
+game 3/32 gm2b16 - sf3190 1-0 (121 plies) score gm2b16 2.0/3
+game 4/32 gm2b16 - sf3190 1/2-1/2 (229 plies) score gm2b16 2.5/4
+game 5/32 sf3190 - gm2b16 1/2-1/2 (134 plies) score gm2b16 3.0/5
+game 6/32 gm2b16 - sf3190 1/2-1/2 (164 plies) score gm2b16 3.5/6
+game 7/32 gm2b16 - sf3190 1/2-1/2 (210 plies) score gm2b16 4.0/7
+game 8/32 sf3190 - gm2b16 1/2-1/2 (108 plies) score gm2b16 4.5/8
+game 9/32 gm2b16 - sf3190 1/2-1/2 (122 plies) score gm2b16 5.0/9
+multiprocessing.pool.RemoteTraceback: 
+"""
+Traceback (most recent call last):
+  File "/tmp/uv-venv/lib/python3.13/site-packages/chess/engine.py", line 1900, in _parse_uci_bestmove
+    move = board.push_uci(tokens[0].lower())
+  File "/tmp/uv-venv/lib/python3.13/site-packages/chess/__init__.py", line 3284, in push_uci
+    move = self.parse_uci(uci)
+  File "/tmp/uv-venv/lib/python3.13/site-packages/chess/__init__.py", line 3267, in parse_uci
+    raise IllegalMoveError(f"illegal uci: {uci!r} in {self.fen()}")
+chess.IllegalMoveError: illegal uci: 'b8f8' in 1r3N2/7k/6q1/6Pp/8/2QR4/1r6/1K3R2 w - - 0 49
+
+During handling of the above exception, another exception occurred:
+
+Traceback (most recent call last):
+  File "/usr/local/lib/python3.13/multiprocessing/pool.py", line 125, in worker
+    result = (True, func(*args, **kwds))
+                    ~~~~^^^^^^^^^^^^^^^
+  File "/marimo/chess/code/match.py", line 104, in game
+    m, info = play_move(name, b)
+              ~~~~~~~~~^^^^^^^^^
+  File "/marimo/chess/code/match.py", line 39, in play_move
+    return _sf[name].play(board, chess.engine.Limit(time=SPM)).move, {}
+           ~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/tmp/uv-venv/lib/python3.13/site-packages/chess/engine.py", line 2980, in play
+    return future.result()
+           ~~~~~~~~~~~~~^^
+  File "/usr/local/lib/python3.13/concurrent/futures/_base.py", line 456, in result
+    return self.__get_result()
+           ~~~~~~~~~~~~~~~~~^^
+  File "/usr/local/lib/python3.13/concurrent/futures/_base.py", line 401, in __get_result
+    raise self._exception
+  File "/usr/local/lib/python3.13/asyncio/tasks.py", line 507, in wait_for
+    return await fut
+           ^^^^^^^^^
+  File "/tmp/uv-venv/lib/python3.13/site-packages/chess/engine.py", line 1703, in play
+    return await self.communicate(UciPlayCommand)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/tmp/uv-venv/lib/python3.13/site-packages/chess/engine.py", line 1021, in communicate
+    return await command.result
+           ^^^^^^^^^^^^^^^^^^^^
+  File "/tmp/uv-venv/lib/python3.13/site-packages/chess/engine.py", line 1294, in _line_received
+    self.line_received(line)
+    ~~~~~~~~~~~~~~~~~~^^^^^^
+  File "/tmp/uv-venv/lib/python3.13/site-packages/chess/engine.py", line 1638, in line_received
+    self._bestmove(remaining)
+    ~~~~~~~~~~~~~~^^^^^^^^^^^
+  File "/tmp/uv-venv/lib/python3.13/site-packages/chess/engine.py", line 1657, in _bestmove
+    best = _parse_uci_bestmove(self.engine.board, arg)
+  File "/tmp/uv-venv/lib/python3.13/site-packages/chess/engine.py", line 1902, in _parse_uci_bestmove
+    raise EngineError(err)
+chess.engine.EngineError: illegal uci: 'b8f8' in 1r3N2/7k/6q1/6Pp/8/2QR4/1r6/1K3R2 w - - 0 49
+"""
+
+The above exception was the direct cause of the following exception:
+
+Traceback (most recent call last):
+  File "/marimo/chess/code/match.py", line 127, in <module>
+    for g in it:
+             ^^
+  File "/usr/local/lib/python3.13/multiprocessing/pool.py", line 873, in next
+    raise value
+chess.engine.EngineError: illegal uci: 'b8f8' in 1r3N2/7k/6q1/6Pp/8/2QR4/1r6/1K3R2 w - - 0 49