tchauffi commited on
Commit
3656ccf
·
verified ·
1 Parent(s): c516647

Switch to the Docker demo: Rust engine + Next.js frontend

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. .dockerignore +3 -0
  2. .gitattributes +1 -1
  3. Dockerfile +39 -0
  4. README.md +15 -11
  5. app.py +0 -243
  6. chesstransformer/__init__.py +0 -1
  7. chesstransformer/benchmark/__init__.py +0 -3
  8. chesstransformer/benchmark/bot_benchmark.py +0 -399
  9. chesstransformer/bots/__init__.py +0 -5
  10. chesstransformer/bots/legacy_position2move_bot.py +0 -222
  11. chesstransformer/bots/pos2move_v2_bot.py +0 -499
  12. chesstransformer/bots/pos2move_v2_mcts_bot.py +0 -318
  13. chesstransformer/bots/position2move_bot.py +0 -88
  14. chesstransformer/bots/random_bot.py +0 -33
  15. chesstransformer/datasets/__init__.py +0 -10
  16. chesstransformer/datasets/dataset_h5_convertor.py +0 -404
  17. chesstransformer/datasets/flat_shard_dataset.py +0 -231
  18. chesstransformer/datasets/h5_lichess_dataset.py +0 -295
  19. chesstransformer/datasets/lichess.py +0 -223
  20. chesstransformer/datasets/lichess_simple_uci.py +0 -136
  21. chesstransformer/datasets/puzzle_dataset.py +0 -259
  22. chesstransformer/datasets/shard_format.py +0 -129
  23. chesstransformer/distributed_muon.py +0 -222
  24. chesstransformer/introspection/__init__.py +0 -5
  25. chesstransformer/introspection/analyze.py +0 -450
  26. chesstransformer/introspection/viewer.html +0 -860
  27. chesstransformer/models/__init__.py +0 -1
  28. chesstransformer/models/tokenizer/__init__.py +0 -93
  29. chesstransformer/models/tokenizer/alphazero_move_encoder.py +0 -127
  30. chesstransformer/models/tokenizer/move_tokenizer.py +0 -38
  31. chesstransformer/models/tokenizer/position_tokenizer.py +0 -88
  32. chesstransformer/models/tokenizer/train_bpe_tokenizer.py +0 -94
  33. chesstransformer/models/transformer/__init__.py +0 -0
  34. chesstransformer/models/transformer/attention.py +0 -228
  35. chesstransformer/models/transformer/gpt.py +0 -79
  36. chesstransformer/models/transformer/pos2move_v2.py +0 -311
  37. chesstransformer/models/transformer/position2move.py +0 -329
  38. chesstransformer/models/transformer/rope.py +0 -104
  39. chesstransformer/models/transformer/utils.py +0 -46
  40. chesstransformer/optimizer.py +0 -206
  41. chesstransformer/trainers/__init__.py +0 -11
  42. chesstransformer/trainers/moveseq2moveseq_trainer.py +0 -342
  43. chesstransformer/trainers/pos2move_v2_trainer.py +0 -1128
  44. chesstransformer/trainers/position2move_trainer.py +0 -682
  45. chesstransformer/utils/__init__.py +0 -3
  46. chesstransformer/utils/chess_utils.py +0 -123
  47. chesstransformer/utils/evaluate_puzzles.py +0 -426
  48. frontend/.dockerignore +36 -0
  49. frontend/.gitignore +41 -0
  50. frontend/Dockerfile +50 -0
.dockerignore ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ **/node_modules
2
+ **/.next
3
+ **/target
.gitattributes CHANGED
@@ -1 +1 @@
1
- *.safetensors filter=lfs diff=lfs merge=lfs -text
 
1
+ *.onnx filter=lfs diff=lfs merge=lfs -text
Dockerfile ADDED
@@ -0,0 +1,39 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Hugging Face Docker Space: one process, one port. ct-bot serves the JSON API
2
+ # and the static Next.js build together on 7860.
3
+ # Build context is the bundle prepare.sh assembles (rust/, frontend/, the model).
4
+
5
+ # --- frontend: static export ---------------------------------------------------
6
+ FROM node:20-slim AS web
7
+ WORKDIR /web
8
+ COPY frontend/package.json frontend/package-lock.json ./
9
+ RUN npm ci
10
+ COPY frontend/ ./
11
+ # Empty API URL = same origin: the page calls /api/* on the server that served it.
12
+ ENV NEXT_OUTPUT=export NEXT_PUBLIC_API_URL=""
13
+ RUN npx next build
14
+
15
+ # --- engine --------------------------------------------------------------------
16
+ FROM rust:1.92-slim-trixie AS engine
17
+ # ort's build script downloads the prebuilt ONNX Runtime over TLS; g++ provides
18
+ # the libstdc++ the statically linked runtime needs at link time.
19
+ RUN apt-get update && apt-get install -y --no-install-recommends \
20
+ pkg-config ca-certificates libssl-dev g++ && rm -rf /var/lib/apt/lists/*
21
+ WORKDIR /build
22
+ COPY rust/ ./rust/
23
+ RUN cargo build --release --manifest-path rust/Cargo.toml -p ct-bot
24
+
25
+ # --- runtime -------------------------------------------------------------------
26
+ FROM debian:trixie-slim
27
+ RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates \
28
+ && rm -rf /var/lib/apt/lists/* && useradd -m -u 1000 user
29
+ WORKDIR /app
30
+ COPY --from=engine /build/rust/target/release/ct-bot /usr/local/bin/ct-bot
31
+ COPY --from=web /web/out ./web
32
+ COPY model.int8.onnx ./model.int8.onnx
33
+ USER user
34
+
35
+ # MCTS_SIMS can be overridden from the Space settings (Variables) without a
36
+ # rebuild. On the free 2-vCPU tier 800 sims is ~2 s a move; 2000 is ~5 s.
37
+ ENV PORT=7860 MCTS_SIMS=800 ORT_THREADS=2 STATIC_DIR=/app/web RUST_LOG=info
38
+ EXPOSE 7860
39
+ CMD ["sh", "-c", "exec ct-bot serve --model /app/model.int8.onnx --threads \"$ORT_THREADS\""]
README.md CHANGED
@@ -1,11 +1,10 @@
1
  ---
2
  title: ChessTransformer
3
  emoji: ♟️
4
- colorFrom: indigo
5
  colorTo: gray
6
- sdk: gradio
7
- sdk_version: 5.49.0
8
- app_file: app.py
9
  pinned: false
10
  license: mit
11
  short_description: Play a chess transformer trained only on human games
@@ -14,15 +13,20 @@ short_description: Play a chess transformer trained only on human games
14
  # ♟️ ChessTransformer
15
 
16
  Play an **11.7M-parameter transformer trained only on human games** (no self-play,
17
- no reinforcement learning), playing via AlphaZero-style MCTS. At full strength it
18
- reaches **~2100 Elo** against Stockfish; this Space runs on CPU at a lower
19
- simulation count so moves come back in ~1–2s.
20
 
21
- You play **White** — drag a piece and the bot replies automatically. Use the
22
- slider to trade strength for speed.
 
 
 
 
 
 
23
 
24
  Code, weights, and the training pipeline: **https://github.com/tchauffi/ChessTransformer**
25
 
26
  > This `README.md` is the Space config. The deployable Space is assembled by
27
- > `prepare.sh` (see `DEPLOY.md`) — it bundles `app.py`, this file, the slim
28
- > `requirements.txt`, the `chesstransformer` package source, and the model weights.
 
1
  ---
2
  title: ChessTransformer
3
  emoji: ♟️
4
+ colorFrom: green
5
  colorTo: gray
6
+ sdk: docker
7
+ app_port: 7860
 
8
  pinned: false
9
  license: mit
10
  short_description: Play a chess transformer trained only on human games
 
13
  # ♟️ ChessTransformer
14
 
15
  Play an **11.7M-parameter transformer trained only on human games** (no self-play,
16
+ no reinforcement learning), searching with AlphaZero-style MCTS. At full strength it
17
+ reaches **~2100 Elo** against Stockfish.
 
18
 
19
+ - Pick a color and a clock, then play: drag or click to move, premove while it
20
+ thinks, right-click to draw arrows and mark squares.
21
+ - **Watch it play itself**, or press **Autoplay** mid-game to hand it your side.
22
+ - The **Engine search** tab shows the branches it explored for each move; hover one
23
+ to replay that line on the board.
24
+
25
+ The engine is the Rust `ct-bot` (ONNX Runtime, int8, CPU). On this free 2-vCPU
26
+ Space it searches 800 positions a move, about 2 seconds.
27
 
28
  Code, weights, and the training pipeline: **https://github.com/tchauffi/ChessTransformer**
29
 
30
  > This `README.md` is the Space config. The deployable Space is assembled by
31
+ > `prepare.sh` (see `DEPLOY.md`): this file, the Dockerfile, the `rust/` and
32
+ > `frontend/` sources, and the int8 model.
app.py DELETED
@@ -1,243 +0,0 @@
1
- """ChessTransformer — Hugging Face Space (Gradio).
2
-
3
- Play the transformer chess engine in the browser. Runs on CPU: the model is
4
- only 11.7M params, so MCTS at a modest sim count returns a move in ~1-2s.
5
-
6
- Weights are resolved in this order:
7
- 1. ``MODEL_PATH`` env var (a checkpoint dir with model.safetensors + config)
8
- 2. a ``model/`` dir next to this file (how ``prepare.sh`` ships the Space)
9
- 3. the repo's ``data/models/pos2move_v2.1`` (for local dev runs)
10
- 4. download from GitHub (LFS media) as a last resort
11
- """
12
-
13
- from __future__ import annotations
14
-
15
- import itertools
16
- import os
17
- import sys
18
- from pathlib import Path
19
-
20
- import chess
21
- import gradio as gr
22
- from gradio_chessboard import Chessboard
23
-
24
- # ── make `chesstransformer` importable ──────────────────────────────────────
25
- # On the Space the package is vendored next to this file (cwd is on sys.path).
26
- # For local dev runs from the repo, add the repo's src/ to the path.
27
- _repo_src = Path(__file__).resolve().parents[1] / "src"
28
- if (_repo_src / "chesstransformer").exists():
29
- sys.path.insert(0, str(_repo_src))
30
-
31
- from chesstransformer.bots import Pos2MoveV2MctsBot # noqa: E402
32
-
33
- START_FEN = chess.STARTING_FEN
34
- DEFAULT_SIMS = int(os.environ.get("MCTS_SIMS", "400"))
35
-
36
- # GitHub locations for the last-resort weight download.
37
- _GH = "tchauffi/ChessTransformer"
38
- _BRANCH = os.environ.get("MODEL_BRANCH", "main")
39
- _RAW = f"https://raw.githubusercontent.com/{_GH}/{_BRANCH}/data/models/pos2move_v2.1"
40
- _LFS = f"https://media.githubusercontent.com/media/{_GH}/{_BRANCH}/data/models/pos2move_v2.1"
41
-
42
-
43
- def _download_weights() -> str:
44
- import urllib.request
45
-
46
- dest = Path("/tmp/pos2move_v2.1")
47
- dest.mkdir(parents=True, exist_ok=True)
48
- for fname, url in [("model_config.json", f"{_RAW}/model_config.json"),
49
- ("model.safetensors", f"{_LFS}/model.safetensors")]:
50
- out = dest / fname
51
- if not out.exists():
52
- print(f"Downloading {fname} …")
53
- urllib.request.urlretrieve(url, out)
54
- return str(dest)
55
-
56
-
57
- def resolve_model_dir() -> str:
58
- env = os.environ.get("MODEL_PATH")
59
- if env and (Path(env) / "model.safetensors").exists():
60
- return env
61
- for cand in (
62
- Path(__file__).resolve().parent / "model",
63
- Path("model"),
64
- Path(__file__).resolve().parents[1] / "data" / "models" / "pos2move_v2.1",
65
- ):
66
- if (cand / "model.safetensors").exists():
67
- return str(cand)
68
- return _download_weights()
69
-
70
-
71
- # ── load the bot once (CPU, no compile) ─────────────────────────────────────
72
- MODEL_DIR = resolve_model_dir()
73
- bot = Pos2MoveV2MctsBot(
74
- model_dir=MODEL_DIR,
75
- num_simulations=DEFAULT_SIMS,
76
- device="cpu",
77
- compile=False,
78
- time_limit=0.0,
79
- # Sample the opening moves (∝ visit count) so games aren't identical every
80
- # time; switches to deterministic best-move play after the opening.
81
- move_temp=1.0,
82
- move_temp_plies=12,
83
- )
84
-
85
-
86
- def _human_color(human_color: str) -> bool:
87
- return chess.BLACK if human_color == "black" else chess.WHITE
88
-
89
-
90
- def _white_pov_value(board: chess.Board) -> float:
91
- """Value-head eval in [-1, 1] from White's perspective (+ = White better)."""
92
- if board.is_game_over():
93
- if board.is_checkmate():
94
- return -1.0 if board.turn == chess.WHITE else 1.0
95
- return 0.0 # stalemate / draw
96
- v = bot.get_value(board)
97
- if v is None:
98
- return 0.0
99
- return v if board.turn == chess.WHITE else -v
100
-
101
-
102
- def eval_bar_html(board: chess.Board, orientation: str) -> str:
103
- """A vertical eval bar (White share fills from the bottom of White's side).
104
- Flipped to match the board orientation so the viewer's side is at the bottom."""
105
- wv = _white_pov_value(board)
106
- white_pct = round(max(0.0, min(1.0, (wv + 1) / 2)) * 100, 1)
107
- white_div = f'<div style="height:{white_pct}%;background:#f5f5f5;"></div>'
108
- black_div = f'<div style="height:{round(100 - white_pct, 1)}%;background:#3a3a3a;"></div>'
109
- # orientation "white": Black on top, White on bottom; flipped when "black".
110
- segments = black_div + white_div if orientation == "white" else white_div + black_div
111
- return (
112
- f'<div title="Model eval (White POV): {wv:+.2f}" '
113
- 'style="display:flex;flex-direction:column;width:26px;height:460px;'
114
- 'border:1px solid #999;border-radius:4px;overflow:hidden;">'
115
- f'{segments}</div>'
116
- )
117
-
118
-
119
- def _result_text(board: chess.Board, human_color: str) -> str:
120
- outcome = board.outcome()
121
- if outcome is None:
122
- return ""
123
- if outcome.winner is None:
124
- return f"Draw ({outcome.termination.name.lower().replace('_', ' ')}). New game?"
125
- who = "You win! 🎉" if outcome.winner == _human_color(human_color) else "ChessTransformer wins."
126
- return f"Checkmate — {who} New game?"
127
-
128
-
129
- def _bot_move(board: chess.Board, sims: int):
130
- """Play the side to move with the bot; return its SAN and value."""
131
- bot.num_simulations = int(sims)
132
- uci, value = bot.predict(board)
133
- san = board.san(chess.Move.from_uci(uci))
134
- board.push_uci(uci)
135
- return san, value
136
-
137
-
138
- def on_move(fen: str, sims: int, human_color: str):
139
- """Human just moved; reply with the bot's move (whichever side is to move).
140
- The board orientation matches the human's color, so the eval bar does too.
141
-
142
- A generator: first yields a "thinking" status (so the player sees the bot is
143
- working during the ~1-3s CPU search), then yields the bot's reply."""
144
- board = chess.Board(fen)
145
- if board.is_game_over():
146
- yield board.fen(), _result_text(board, human_color), eval_bar_html(board, human_color)
147
- return
148
-
149
- # Immediate feedback while the search runs (board/eval unchanged for now).
150
- yield gr.update(), "⏳ ChessTransformer is thinking…", gr.update()
151
-
152
- san, value = _bot_move(board, sims)
153
-
154
- if board.is_game_over():
155
- status = f"Bot played **{san}**. {_result_text(board, human_color)}"
156
- else:
157
- eval_str = f" (eval {value:+.2f})" if value is not None else ""
158
- status = f"Bot played **{san}**{eval_str}. Your move."
159
- yield board.fen(), status, eval_bar_html(board, human_color)
160
-
161
-
162
- # Monotonic id so every New game yields a *distinct* setup dict — otherwise a
163
- # same-colour replay returns an identical dict and the gr.render block, seeing no
164
- # change, wouldn't rebuild (and reset) the board.
165
- _game_counter = itertools.count(1)
166
-
167
-
168
- def new_game(play_as: str, sims: int):
169
- """Start a fresh game. If the human plays Black, the bot (White) opens.
170
-
171
- Returns (setup, eval_bar). The setup dict drives a gr.render block — changing
172
- it rebuilds the board so its orientation (chessboard.js reads it only at
173
- construction) actually flips to the human's side."""
174
- human_color = "black" if play_as == "Black" else "white"
175
- board = chess.Board()
176
- if human_color == "black":
177
- san, _ = _bot_move(board, sims)
178
- status = f"New game — you are **Black**. Bot opened with **{san}**. Your move."
179
- else:
180
- status = "New game — you are **White**. Make your move."
181
- setup = {"fen": board.fen(), "orientation": human_color, "color": human_color,
182
- "status": status, "nonce": next(_game_counter)}
183
- return setup, eval_bar_html(board, human_color)
184
-
185
-
186
- with gr.Blocks(title="ChessTransformer", theme=gr.themes.Soft()) as demo:
187
- gr.Markdown(
188
- "# ♟️ ChessTransformer\n"
189
- "An **11.7M-parameter transformer trained only on human games** "
190
- "(no self-play, no RL), playing via AlphaZero-style MCTS. "
191
- "It reaches **~2100 Elo** vs Stockfish at full strength; this CPU demo "
192
- "runs at a lower sim count so moves come back quickly.\n\n"
193
- "**Pick your color and hit New game.** Drag a piece to move — the bot "
194
- "replies automatically. "
195
- "[GitHub repo →](https://github.com/tchauffi/ChessTransformer)"
196
- )
197
- setup = gr.State({"fen": START_FEN, "orientation": "white", "color": "white",
198
- "status": "You are **White**. Make your move.", "nonce": 0})
199
-
200
- with gr.Row():
201
- # Solid min_widths so the board container always has a size when the
202
- # gr.render block (re)mounts it — chessboard.js reloads the whole page if
203
- # it inits at 0 width, which is what caused the mobile flicker/reload loop.
204
- eval_col = gr.Column(scale=0, min_width=44)
205
- board_col = gr.Column(scale=3, min_width=340)
206
- ctrl_col = gr.Column(scale=1, min_width=200)
207
-
208
- with eval_col:
209
- eval_bar = gr.HTML(eval_bar_html(chess.Board(), "white"), visible=False)
210
-
211
- with ctrl_col:
212
- play_as = gr.Radio(["White", "Black"], value="White", label="Play as")
213
- sims = gr.Slider(32, 1200, value=DEFAULT_SIMS, step=8,
214
- label="Engine strength (MCTS sims/move)",
215
- info="Higher = stronger but slower on CPU")
216
- show_eval = gr.Checkbox(False, label="Show evaluation bar")
217
- new_btn = gr.Button("New game", variant="primary")
218
-
219
- # The board lives in a gr.render so switching color rebuilds it with the new
220
- # orientation (chessboard.js reads orientation only at construction). The
221
- # min_width above keeps the container sized so the remount can't init at 0.
222
- with board_col:
223
- @gr.render(inputs=setup)
224
- def render_board(cfg):
225
- board = Chessboard(value=cfg["fen"], game_mode=True, min_width=320,
226
- orientation=cfg["orientation"], label="Drag to move")
227
- status = gr.Markdown(cfg["status"])
228
- color = cfg["color"]
229
-
230
- def handle(fen, s): # generator → streams the "thinking" update first
231
- yield from on_move(fen, s, color)
232
-
233
- # show_progress="hidden": our "⏳ thinking…" status is the signal, so
234
- # Gradio's default overlay doesn't sit over the board.
235
- board.move(handle, inputs=[board, sims], outputs=[board, status, eval_bar],
236
- show_progress="hidden")
237
-
238
- show_eval.change(lambda show: gr.update(visible=show), inputs=show_eval, outputs=eval_bar)
239
- new_btn.click(new_game, inputs=[play_as, sims], outputs=[setup, eval_bar])
240
-
241
-
242
- if __name__ == "__main__":
243
- demo.launch()
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
chesstransformer/__init__.py DELETED
@@ -1 +0,0 @@
1
- """This is the main package for the ChessTransformer project."""
 
 
chesstransformer/benchmark/__init__.py DELETED
@@ -1,3 +0,0 @@
1
- from .bot_benchmark import BotBenchmark
2
-
3
- __all__ = ["BotBenchmark"]
 
 
 
 
chesstransformer/benchmark/bot_benchmark.py DELETED
@@ -1,399 +0,0 @@
1
- """
2
- Bot benchmarking system for comparing chess bot performance.
3
-
4
- Usage:
5
- python -m chesstransformer.benchmark.bot_benchmark \\
6
- --bot1-type position2move --bot1-checkpoint path/to/checkpoint \\
7
- --bot2-type random \\
8
- --num-games 100 \\
9
- --output results.json
10
- """
11
-
12
- import chess
13
- import time
14
- import json
15
- from pathlib import Path
16
- from typing import List, Tuple, Optional
17
- from dataclasses import dataclass, asdict
18
- from tqdm import tqdm
19
-
20
- from chesstransformer.bots import Position2MoveBot, LegacyPosition2MoveBot, RandomBot
21
-
22
-
23
- @dataclass
24
- class GameResult:
25
- """Single game result statistics."""
26
-
27
- game_id: int
28
- white_bot: str
29
- black_bot: str
30
- result: str # "1-0", "0-1", "1/2-1/2"
31
- num_moves: int
32
- white_time: float
33
- black_time: float
34
- termination: str # "checkmate", "stalemate", "insufficient_material", "max_moves"
35
-
36
- def to_dict(self):
37
- return asdict(self)
38
-
39
-
40
- @dataclass
41
- class BenchmarkStats:
42
- """Aggregate statistics for a bot."""
43
-
44
- bot_name: str
45
- total_games: int
46
- wins: int
47
- losses: int
48
- draws: int
49
- avg_moves_per_game: float
50
- avg_time_per_move: float
51
- total_time: float
52
- win_rate: float
53
- win_as_white: int
54
- win_as_black: int
55
-
56
- def to_dict(self):
57
- return asdict(self)
58
-
59
-
60
- class BotBenchmark:
61
- """Benchmark system for comparing chess bots."""
62
-
63
- def __init__(self, max_moves: int = 500, time_limit: Optional[float] = None):
64
- """
65
- Initialize benchmark.
66
-
67
- Args:
68
- max_moves: Maximum moves per game before declaring draw
69
- time_limit: Optional time limit per move in seconds
70
- """
71
- self.max_moves = max_moves
72
- self.time_limit = time_limit
73
- self.game_results: List[GameResult] = []
74
- self.bot_counter = 0 # Counter for unique bot IDs
75
-
76
- def create_bot(
77
- self, bot_type: str, checkpoint_path: Optional[str] = None, bot_name: Optional[str] = None, **kwargs
78
- ):
79
- """
80
- Factory method to create bots.
81
-
82
- Args:
83
- bot_type: Type of bot ("position2move", "legacy_position2move", "random")
84
- checkpoint_path: Path to model checkpoint (for Position2MoveBot)
85
- bot_name: Optional custom name for the bot (for tracking multiple instances)
86
- **kwargs: Additional bot-specific arguments
87
- """
88
- if bot_type.lower() == "position2move":
89
- if checkpoint_path:
90
- bot = Position2MoveBot(model_path=checkpoint_path)
91
- else:
92
- bot = Position2MoveBot()
93
- elif bot_type.lower() == "legacy_position2move":
94
- if checkpoint_path:
95
- bot = LegacyPosition2MoveBot(model_path=checkpoint_path)
96
- else:
97
- bot = LegacyPosition2MoveBot()
98
- elif bot_type.lower() == "random":
99
- bot = RandomBot()
100
- else:
101
- raise ValueError(f"Unknown bot type: {bot_type}")
102
-
103
- # Assign unique identifier
104
- if bot_name:
105
- bot._benchmark_id = bot_name
106
- else:
107
- self.bot_counter += 1
108
- bot._benchmark_id = f"{bot.__class__.__name__}_{self.bot_counter}"
109
-
110
- return bot
111
-
112
- def play_game(self, white_bot, black_bot, game_id: int, verbose: bool = False) -> GameResult:
113
- """
114
- Play a single game between two bots.
115
-
116
- Args:
117
- white_bot: Bot playing white
118
- black_bot: Bot playing black
119
- game_id: Unique game identifier
120
- verbose: Print move-by-move progress
121
-
122
- Returns:
123
- GameResult with statistics
124
- """
125
- board = chess.Board()
126
- white_time = 0.0
127
- black_time = 0.0
128
- move_count = 0
129
-
130
- while not board.is_game_over() and move_count < self.max_moves:
131
- current_bot = white_bot if board.turn == chess.WHITE else black_bot
132
-
133
- start_time = time.time()
134
-
135
- move, _ = current_bot.predict(board)
136
-
137
- elapsed = time.time() - start_time
138
-
139
- if board.turn == chess.WHITE:
140
- white_time += elapsed
141
- else:
142
- black_time += elapsed
143
-
144
- board.push(chess.Move.from_uci(move))
145
- move_count += 1
146
-
147
- if verbose and move_count % 10 == 0:
148
- print(f"Move {move_count}: {move}")
149
-
150
- # Determine result and termination reason
151
- if board.is_checkmate():
152
- result = "0-1" if board.turn == chess.WHITE else "1-0"
153
- termination = "checkmate"
154
- elif board.is_stalemate():
155
- result = "1/2-1/2"
156
- termination = "stalemate"
157
- elif board.is_insufficient_material():
158
- result = "1/2-1/2"
159
- termination = "insufficient_material"
160
- elif board.is_seventyfive_moves():
161
- result = "1/2-1/2"
162
- termination = "75_move_rule"
163
- elif board.is_fivefold_repetition():
164
- result = "1/2-1/2"
165
- termination = "repetition"
166
- elif move_count >= self.max_moves:
167
- result = "1/2-1/2"
168
- termination = "max_moves"
169
- else:
170
- # Should not reach here if game ended normally
171
- result = "1/2-1/2"
172
- termination = "unknown"
173
-
174
- return GameResult(
175
- game_id=game_id,
176
- white_bot=white_bot._benchmark_id,
177
- black_bot=black_bot._benchmark_id,
178
- result=result,
179
- num_moves=move_count,
180
- white_time=white_time,
181
- black_time=black_time,
182
- termination=termination,
183
- )
184
-
185
- def run_benchmark(
186
- self, bot1, bot2, num_games: int, alternate_colors: bool = True, verbose: bool = False
187
- ) -> Tuple[BenchmarkStats, BenchmarkStats]:
188
- """
189
- Run full benchmark between two bots.
190
-
191
- Args:
192
- bot1: First bot
193
- bot2: Second bot
194
- num_games: Number of games to play
195
- alternate_colors: Alternate colors each game
196
- verbose: Print game-by-game results
197
-
198
- Returns:
199
- Tuple of (bot1_stats, bot2_stats)
200
- """
201
- self.game_results = []
202
-
203
- pbar = tqdm(range(num_games), desc="Playing games")
204
- for game_id in pbar:
205
- if alternate_colors and game_id % 2 == 1:
206
- # Swap colors
207
- white_bot, black_bot = bot2, bot1
208
- else:
209
- white_bot, black_bot = bot1, bot2
210
-
211
- result = self.play_game(white_bot, black_bot, game_id, verbose=verbose)
212
- self.game_results.append(result)
213
-
214
- if verbose:
215
- print(f"Game {game_id + 1}: {result.white_bot} vs {result.black_bot} = {result.result}")
216
-
217
- # Update progress bar with current stats
218
- bot1_wins = sum(1 for r in self.game_results if self._bot_won(r, bot1._benchmark_id))
219
- bot2_wins = sum(1 for r in self.game_results if self._bot_won(r, bot2._benchmark_id))
220
- pbar.set_postfix({bot1._benchmark_id: bot1_wins, bot2._benchmark_id: bot2_wins})
221
-
222
- # Calculate statistics
223
- bot1_stats = self._calculate_stats(bot1._benchmark_id)
224
- bot2_stats = self._calculate_stats(bot2._benchmark_id)
225
-
226
- return bot1_stats, bot2_stats
227
-
228
- def _bot_won(self, result: GameResult, bot_name: str) -> bool:
229
- """Check if bot won the game."""
230
- if bot_name == result.white_bot and result.result == "1-0":
231
- return True
232
- if bot_name == result.black_bot and result.result == "0-1":
233
- return True
234
- return False
235
-
236
- def _calculate_stats(self, bot_name: str) -> BenchmarkStats:
237
- """Calculate aggregate statistics for a bot."""
238
- bot_results = []
239
- wins = 0
240
- losses = 0
241
- draws = 0
242
- win_as_white = 0
243
- win_as_black = 0
244
- total_time = 0.0
245
- total_moves = 0
246
-
247
- for result in self.game_results:
248
- is_white = result.white_bot == bot_name
249
- is_black = result.black_bot == bot_name
250
-
251
- if not (is_white or is_black):
252
- continue
253
-
254
- bot_results.append(result)
255
- total_moves += result.num_moves
256
-
257
- if is_white:
258
- total_time += result.white_time
259
- if result.result == "1-0":
260
- wins += 1
261
- win_as_white += 1
262
- elif result.result == "0-1":
263
- losses += 1
264
- else:
265
- draws += 1
266
- else:
267
- total_time += result.black_time
268
- if result.result == "0-1":
269
- wins += 1
270
- win_as_black += 1
271
- elif result.result == "1-0":
272
- losses += 1
273
- else:
274
- draws += 1
275
-
276
- total_games = len(bot_results)
277
- avg_moves = total_moves / total_games if total_games > 0 else 0
278
- avg_time_per_move = total_time / total_moves if total_moves > 0 else 0
279
- win_rate = wins / total_games if total_games > 0 else 0
280
-
281
- return BenchmarkStats(
282
- bot_name=bot_name,
283
- total_games=total_games,
284
- wins=wins,
285
- losses=losses,
286
- draws=draws,
287
- avg_moves_per_game=avg_moves,
288
- avg_time_per_move=avg_time_per_move,
289
- total_time=total_time,
290
- win_rate=win_rate,
291
- win_as_white=win_as_white,
292
- win_as_black=win_as_black,
293
- )
294
-
295
- def save_results(self, output_path: str):
296
- """Save benchmark results to JSON file."""
297
- output = {
298
- "config": {"max_moves": self.max_moves, "time_limit": self.time_limit, "num_games": len(self.game_results)},
299
- "games": [r.to_dict() for r in self.game_results],
300
- "stats": {
301
- bot_name: self._calculate_stats(bot_name).to_dict()
302
- for bot_name in set([r.white_bot for r in self.game_results] + [r.black_bot for r in self.game_results])
303
- },
304
- }
305
-
306
- output_file = Path(output_path)
307
- output_file.parent.mkdir(parents=True, exist_ok=True)
308
-
309
- with output_file.open("w", encoding="utf-8") as f:
310
- json.dump(output, f, indent=2)
311
-
312
- print(f"Results saved to: {output_path}")
313
-
314
- def print_summary(self, bot1_stats: BenchmarkStats, bot2_stats: BenchmarkStats):
315
- """Print formatted summary of benchmark results."""
316
- print("\n" + "=" * 80)
317
- print("BENCHMARK RESULTS")
318
- print("=" * 80)
319
-
320
- for stats in [bot1_stats, bot2_stats]:
321
- print(f"\n{stats.bot_name}:")
322
- print(f" Total Games: {stats.total_games}")
323
- print(f" Wins: {stats.wins} ({stats.win_rate:.1%})")
324
- print(f" - As White: {stats.win_as_white}")
325
- print(f" - As Black: {stats.win_as_black}")
326
- print(f" Losses: {stats.losses}")
327
- print(f" Draws: {stats.draws}")
328
- print(f" Avg Moves/Game: {stats.avg_moves_per_game:.1f}")
329
- print(f" Avg Time/Move: {stats.avg_time_per_move:.3f}s")
330
- print(f" Total Time: {stats.total_time:.1f}s")
331
-
332
- print("\n" + "=" * 80)
333
-
334
-
335
- def main():
336
- """CLI entry point for bot benchmarking."""
337
- import argparse
338
-
339
- parser = argparse.ArgumentParser(description="Benchmark chess bots")
340
- parser.add_argument(
341
- "--bot1-type",
342
- type=str,
343
- required=True,
344
- choices=["position2move", "legacy_position2move", "random"],
345
- help="Type of first bot",
346
- )
347
- parser.add_argument("--bot1-checkpoint", type=str, default=None, help="Checkpoint path for bot1 (if applicable)")
348
- parser.add_argument(
349
- "--bot1-name", type=str, default=None, help="Custom name for bot1 (useful when comparing same bot types)"
350
- )
351
- parser.add_argument(
352
- "--bot2-type",
353
- type=str,
354
- required=True,
355
- choices=["position2move", "legacy_position2move", "random"],
356
- help="Type of second bot",
357
- )
358
- parser.add_argument("--bot2-checkpoint", type=str, default=None, help="Checkpoint path for bot2 (if applicable)")
359
- parser.add_argument(
360
- "--bot2-name", type=str, default=None, help="Custom name for bot2 (useful when comparing same bot types)"
361
- )
362
- parser.add_argument("--num-games", type=int, default=100, help="Number of games to play")
363
- parser.add_argument("--max-moves", type=int, default=500, help="Maximum moves per game")
364
- parser.add_argument("--time-limit", type=float, default=None, help="Time limit per move in seconds")
365
- parser.add_argument("--output", type=str, default="benchmark_results.json", help="Output file for results")
366
- parser.add_argument("--no-alternate", action="store_true", help="Don't alternate colors between games")
367
- parser.add_argument("--verbose", action="store_true", help="Print detailed game information")
368
-
369
- args = parser.parse_args()
370
-
371
- # Create benchmark
372
- benchmark = BotBenchmark(max_moves=args.max_moves, time_limit=args.time_limit)
373
-
374
- # Create bots
375
- print(f"Creating {args.bot1_type} bot...")
376
- bot1 = benchmark.create_bot(args.bot1_type, checkpoint_path=args.bot1_checkpoint, bot_name=args.bot1_name)
377
-
378
- print(f"Creating {args.bot2_type} bot...")
379
- bot2 = benchmark.create_bot(args.bot2_type, checkpoint_path=args.bot2_checkpoint, bot_name=args.bot2_name)
380
-
381
- # Run benchmark
382
- print(f"\nRunning benchmark: {args.num_games} games")
383
- print(f"Max moves per game: {args.max_moves}")
384
- if args.time_limit:
385
- print(f"Time limit per move: {args.time_limit}s")
386
-
387
- bot1_stats, bot2_stats = benchmark.run_benchmark(
388
- bot1, bot2, num_games=args.num_games, alternate_colors=not args.no_alternate, verbose=args.verbose
389
- )
390
-
391
- # Print summary
392
- benchmark.print_summary(bot1_stats, bot2_stats)
393
-
394
- # Save results
395
- benchmark.save_results(args.output)
396
-
397
-
398
- if __name__ == "__main__":
399
- main()
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
chesstransformer/bots/__init__.py DELETED
@@ -1,5 +0,0 @@
1
- from .pos2move_v2_bot import Pos2MoveV2Bot
2
- from .pos2move_v2_mcts_bot import Pos2MoveV2MctsBot
3
- from .random_bot import RandomBot
4
-
5
- __all__ = ["Pos2MoveV2Bot", "Pos2MoveV2MctsBot", "RandomBot"]
 
 
 
 
 
 
chesstransformer/bots/legacy_position2move_bot.py DELETED
@@ -1,222 +0,0 @@
1
- """Legacy Position2Move bot for V2 model compatibility.
2
-
3
- The V2 model was trained with an older architecture that uses:
4
- - MultiHeadAttention instead of GroupedQueryAttention
5
- - LayerNorm with gamma/beta instead of RMSNorm with weight
6
- - Standard GELU FeedForward instead of SwiGLU
7
- - No CLS token, castling, en passant, or halfmove embeddings
8
- """
9
-
10
- from pathlib import Path
11
- import json
12
- import torch
13
- import torch.nn as nn
14
- import chess
15
- from safetensors import safe_open
16
- from chesstransformer.models.transformer.attention import MultiHeadAttention
17
- from chesstransformer.models.tokenizer import PostionTokenizer, MoveTokenizer
18
-
19
-
20
- class LayerNorm(nn.Module):
21
- """Legacy LayerNorm with gamma/beta naming."""
22
-
23
- def __init__(self, embed_dim):
24
- super().__init__()
25
- self.gamma = nn.Parameter(torch.ones(embed_dim))
26
- self.beta = nn.Parameter(torch.zeros(embed_dim))
27
- self.eps = 1e-6
28
-
29
- def forward(self, x):
30
- mean = x.mean(dim=-1, keepdim=True)
31
- var = x.var(dim=-1, unbiased=False, keepdim=True)
32
- return self.gamma * (x - mean) / torch.sqrt(var + self.eps) + self.beta
33
-
34
-
35
- class FeedForward(nn.Module):
36
- """Legacy FeedForward network."""
37
-
38
- def __init__(self, embed_dim):
39
- super().__init__()
40
- self.layers = nn.Sequential(
41
- nn.Linear(embed_dim, 4 * embed_dim),
42
- nn.GELU(approximate="tanh"),
43
- nn.Linear(4 * embed_dim, embed_dim),
44
- )
45
-
46
- def forward(self, x):
47
- return self.layers(x)
48
-
49
-
50
- class LegacyTransformerBlock(nn.Module):
51
- """Legacy transformer block for V2 model."""
52
-
53
- def __init__(
54
- self,
55
- embed_dim,
56
- num_heads,
57
- context_length,
58
- dropout,
59
- qkv_bias,
60
- mask_future=False,
61
- ):
62
- super().__init__()
63
- self.att = MultiHeadAttention(
64
- d_in=embed_dim,
65
- d_out=embed_dim,
66
- num_heads=num_heads,
67
- context_length=context_length,
68
- dropout=dropout,
69
- qkv_bias=qkv_bias,
70
- mask_future=mask_future,
71
- )
72
- self.feed_forward = FeedForward(embed_dim)
73
- self.norm1 = LayerNorm(embed_dim)
74
- self.norm2 = LayerNorm(embed_dim)
75
- self.dropout_shortcut = nn.Dropout(dropout)
76
-
77
- def forward(self, x):
78
- shortcut = x
79
- x = self.norm1(x)
80
- x = self.att(x)
81
- x = self.dropout_shortcut(x)
82
- x = x + shortcut
83
-
84
- shortcut = x
85
- x = self.norm2(x)
86
- x = self.feed_forward(x)
87
- x = self.dropout_shortcut(x)
88
- x = x + shortcut
89
-
90
- return x
91
-
92
-
93
- class LegacyPosition2MoveModel(nn.Module):
94
- """Legacy Position2Move model architecture for V2 compatibility."""
95
-
96
- def __init__(
97
- self,
98
- vocab_size: int = 13,
99
- move_vocab_size: int = 1968,
100
- embed_dim: int = 512,
101
- nb_transformer_layers: int = 8,
102
- num_heads: int = 8,
103
- dropout: float = 0.1,
104
- kvq_bias: bool = False,
105
- mask_future: bool = False,
106
- ):
107
- super().__init__()
108
-
109
- # Board piece embeddings
110
- self.token_embedding = nn.Embedding(vocab_size, embed_dim)
111
- self.position_embedding = nn.Embedding(64, embed_dim) # 64 squares
112
- self.player_embedding = nn.Embedding(2, embed_dim) # 2 players
113
-
114
- self.dropout = nn.Dropout(dropout)
115
- self.transformer_blocks = nn.Sequential(
116
- *[
117
- LegacyTransformerBlock(
118
- embed_dim=embed_dim,
119
- num_heads=num_heads,
120
- context_length=64,
121
- dropout=dropout,
122
- qkv_bias=kvq_bias,
123
- mask_future=mask_future,
124
- )
125
- for _ in range(nb_transformer_layers)
126
- ]
127
- )
128
-
129
- self.norm = LayerNorm(embed_dim)
130
- self.lm_head = nn.Linear(embed_dim, move_vocab_size, bias=False)
131
-
132
- def forward(self, x, is_white):
133
- batch_size, seq_len = x.shape
134
-
135
- # Get embeddings
136
- token_emb = self.token_embedding(x)
137
- pos_indices = torch.arange(seq_len, device=x.device)
138
- pos_emb = self.position_embedding(pos_indices)
139
- player_emb = self.player_embedding(is_white.long())
140
-
141
- # Combine embeddings
142
- x = token_emb + pos_emb.unsqueeze(0) + player_emb.unsqueeze(1)
143
- x = self.dropout(x)
144
-
145
- # Transformer blocks
146
- x = self.transformer_blocks(x)
147
- x = self.norm(x)
148
-
149
- # Mean pooling
150
- x = x.mean(dim=1)
151
-
152
- # Output
153
- logits = self.lm_head(x)
154
- return logits
155
-
156
-
157
- data_folder = Path(__file__).parents[1] / "data"
158
-
159
-
160
- class LegacyPosition2MoveBot:
161
- """Bot using the legacy V2 model architecture."""
162
-
163
- def __init__(
164
- self,
165
- model_path: str = str((data_folder / "models/position2moveV2.0/model.safetensors").resolve()),
166
- device: str = "cpu",
167
- ):
168
- self.device = device
169
- self.position_tokenizer = PostionTokenizer()
170
- self.move_tokenizer = MoveTokenizer()
171
-
172
- config_path = Path(model_path).parent / "config.json"
173
-
174
- with open(config_path, "r") as f:
175
- self.config = json.load(f)
176
-
177
- # Load legacy model
178
- self.model = LegacyPosition2MoveModel(**self.config).to(device)
179
-
180
- with safe_open(model_path, framework="pt", device=device) as f:
181
- state_dict = {k: f.get_tensor(k) for k in f.keys()}
182
- self.model.load_state_dict(state_dict)
183
-
184
- self.model.eval()
185
-
186
- @torch.no_grad()
187
- def predict(self, board: chess.Board):
188
- tokens_ids = self.position_tokenizer.encode(board)
189
- torch_input = torch.tensor(tokens_ids).unsqueeze(0).long().to(self.device)
190
-
191
- is_white = board.turn
192
- is_white = torch.tensor([is_white]).bool().to(self.device)
193
-
194
- logits = self.model(torch_input, is_white)
195
-
196
- legal_moves = [m.uci() for m in board.legal_moves]
197
-
198
- mask = torch.full((logits.size(-1),), float("-inf")).to(self.device)
199
- for move in legal_moves:
200
- move_id = self.move_tokenizer.encode(move)
201
- mask[move_id] = 0.0
202
-
203
- masked_logits = logits[0] + mask
204
- probs = torch.softmax(masked_logits, dim=-1)
205
-
206
- # Sample from distribution with temperature
207
- predicted_move_id = torch.multinomial(probs * 0.2, num_samples=1).item()
208
- proba = probs[predicted_move_id].item()
209
-
210
- predicted_move = self.move_tokenizer.decode(predicted_move_id)
211
- return predicted_move, proba
212
-
213
-
214
- if __name__ == "__main__":
215
- bot = LegacyPosition2MoveBot()
216
-
217
- board = chess.Board()
218
- print("Initial board:")
219
- print(board)
220
-
221
- move, proba = bot.predict(board)
222
- print(f"Bot suggests move: {move} with probability {proba:.4f}")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
chesstransformer/bots/pos2move_v2_bot.py DELETED
@@ -1,499 +0,0 @@
1
- """Bot for Pos2MoveV2 with alpha-beta search + BF16."""
2
-
3
- from __future__ import annotations
4
-
5
- import json
6
- import time as _time
7
- from pathlib import Path
8
-
9
- import chess
10
- import chess.polyglot
11
- import numpy as np
12
- import torch
13
- from safetensors import safe_open
14
-
15
- from chesstransformer.models.tokenizer import PostionTokenizer
16
- from chesstransformer.models.tokenizer.alphazero_move_encoder import move_to_action_plane
17
- from chesstransformer.models.transformer.pos2move_v2 import Pos2MoveV2
18
-
19
- _DEFAULT_RUN = Path(__file__).parents[3] / "data" / "models" / "pos2move_v2.1"
20
-
21
- # Fixed batch-size buckets. Batched evals are zero-padded up to the next bucket
22
- # so the (optionally compiled) model only ever sees a handful of static shapes.
23
- _BATCH_BUCKETS = (4, 8, 16, 32)
24
-
25
-
26
- class _SearchTimeout(Exception):
27
- pass
28
-
29
-
30
- class Pos2MoveV2Bot:
31
- def __init__(
32
- self,
33
- model_dir: str | Path = _DEFAULT_RUN,
34
- depth: int = 3,
35
- top_p: float = 0.90,
36
- time_limit: float = 30.0,
37
- tt_size: int = 500_000,
38
- device: str = "cuda" if torch.cuda.is_available() else "cpu",
39
- use_ema: bool = False,
40
- compile: bool = True,
41
- quiescence_depth: int = 4,
42
- ):
43
- self.depth = depth
44
- self.top_p = top_p
45
- self.time_limit = time_limit
46
- self.quiescence_depth = quiescence_depth
47
- self.nodes_searched = 0
48
- self.tt_hits = 0
49
- self.completed_depth = 0
50
- self.device = device
51
- self.position_tokenizer = PostionTokenizer()
52
-
53
- model_dir = Path(model_dir)
54
-
55
- config_path = model_dir / "model_config.json"
56
- if not config_path.exists():
57
- config_path = model_dir.parent.parent / "model_config.json"
58
- if not config_path.exists():
59
- raise FileNotFoundError(f"No model_config.json found for {model_dir}")
60
-
61
- with open(config_path) as f:
62
- config = json.load(f)
63
-
64
- model = Pos2MoveV2(**config)
65
-
66
- safetensors_path = model_dir / "model.safetensors"
67
- if not safetensors_path.exists():
68
- raise FileNotFoundError(f"No model.safetensors in {model_dir}")
69
- with safe_open(str(safetensors_path), framework="pt", device="cpu") as f:
70
- state_dict = {k: f.get_tensor(k) for k in f.keys()}
71
-
72
- if any(k.startswith("_orig_mod.") for k in state_dict):
73
- state_dict = {k.replace("_orig_mod.", ""): v for k, v in state_dict.items()}
74
-
75
- model.load_state_dict(state_dict)
76
-
77
- if use_ema:
78
- ema_path = model_dir / "ema_state.pt"
79
- if not ema_path.exists():
80
- raise FileNotFoundError(f"No ema_state.pt in {model_dir}")
81
- ema_state = torch.load(str(ema_path), map_location="cpu", weights_only=True)
82
- ema_state = {k.replace("_orig_mod.", ""): v for k, v in ema_state.items()}
83
- missing, unexpected = model.load_state_dict(ema_state, strict=False)
84
- if unexpected:
85
- print(f"WARNING: EMA contains unexpected keys: {unexpected}")
86
- print(f"EMA weights loaded ({len(ema_state)} tensors; {len(missing)} buffers kept).")
87
-
88
- model.eval()
89
- self.model = model.bfloat16().to(device)
90
-
91
- self._buf_board = torch.zeros(1, 64, dtype=torch.long, device=device)
92
- self._buf_player = torch.zeros(1, dtype=torch.long, device=device)
93
- self._buf_castling = torch.zeros(1, dtype=torch.long, device=device)
94
- self._buf_ep = torch.zeros(1, dtype=torch.long, device=device)
95
- self._batch_bufs: dict[int, tuple] = {}
96
-
97
- self._tt_size = tt_size
98
- self._tt: dict[int, tuple[np.ndarray, float]] = {}
99
- self._deadline: float = 0.0
100
-
101
- # CUDA graphs collapse the Python/kernel-launch overhead that dominates
102
- # this tiny model at small batch sizes. Bucketed batches keep the number
103
- # of captured graphs small. Falls back to eager if compilation fails.
104
- self._compiled = False
105
- if compile and device == "cuda":
106
- self._eager_model = self.model
107
- try:
108
- self.model = torch.compile(self.model, mode="reduce-overhead")
109
- self._compiled = True
110
- except Exception as e: # pragma: no cover - environment dependent
111
- print(f"WARNING: torch.compile unavailable ({e}); using eager.")
112
- self.model = self._eager_model
113
-
114
- self._warmup()
115
-
116
- print(
117
- f"Pos2MoveV2Bot loaded (device={device}, depth={depth}, "
118
- f"top_p={top_p:.0%}, params={sum(p.numel() for p in self.model.parameters()) / 1e6:.1f}M)"
119
- )
120
-
121
- @torch.no_grad()
122
- def _warmup(self):
123
- try:
124
- # Batch-1 path (single-position _evaluate).
125
- for _ in range(3):
126
- self.model(self._buf_board, self._buf_player, self._buf_castling, self._buf_ep)
127
- # Each bucket shape used by _batch_run, so compilation / CUDA-graph
128
- # capture happens up front rather than mid-search.
129
- for bucket in _BATCH_BUCKETS:
130
- b, p, c, e = self._batch_buffers(bucket)
131
- for _ in range(3):
132
- self.model(b, p, c, e)
133
- except Exception as e: # pragma: no cover - environment dependent
134
- if self._compiled:
135
- print(f"WARNING: compiled warmup failed ({e}); falling back to eager.")
136
- self.model = self._eager_model
137
- self._compiled = False
138
- self.model(self._buf_board, self._buf_player, self._buf_castling, self._buf_ep)
139
- else:
140
- raise
141
- if self.device == "cuda":
142
- torch.cuda.synchronize()
143
-
144
- def _adaptive_time_limit(self, board: chess.Board) -> float:
145
- if self.time_limit <= 0:
146
- return 0.0
147
- ply = len(board.move_stack)
148
- move_number = ply // 2 + 1
149
- if move_number <= 10:
150
- scale = 0.10
151
- elif move_number <= 20:
152
- scale = 0.40
153
- elif move_number <= 40:
154
- scale = 1.0
155
- else:
156
- scale = 0.60
157
- return self.time_limit * scale
158
-
159
- def predict(self, board: chess.Board) -> tuple[str, float | None]:
160
- self.nodes_searched = 0
161
- self.tt_hits = 0
162
- self.completed_depth = 0
163
-
164
- effective_limit = self._adaptive_time_limit(board)
165
- self._deadline = _time.monotonic() + effective_limit if effective_limit > 0 else float("inf")
166
-
167
- best_move = None
168
- best_value = 0.0
169
-
170
- for d in range(1, self.depth + 1):
171
- stack_len = len(board.move_stack)
172
- try:
173
- move, value = self._search_root(board, d)
174
- best_move = move
175
- best_value = value
176
- self.completed_depth = d
177
- except _SearchTimeout:
178
- while len(board.move_stack) > stack_len:
179
- board.pop()
180
- break
181
-
182
- if best_move is None:
183
- best_move = next(iter(board.legal_moves))
184
-
185
- return best_move.uci(), best_value
186
-
187
- def get_value(self, board: chess.Board) -> float:
188
- _, value = self._evaluate(board)
189
- return value
190
-
191
- def _check_time(self):
192
- if _time.monotonic() >= self._deadline:
193
- raise _SearchTimeout()
194
-
195
- @torch.no_grad()
196
- def _evaluate(self, board: chess.Board) -> tuple[np.ndarray, float]:
197
- key = chess.polyglot.zobrist_hash(board)
198
- cached = self._tt.get(key)
199
- if cached is not None:
200
- self.tt_hits += 1
201
- return cached
202
-
203
- position = self.position_tokenizer.encode(board)
204
- self._buf_board[0] = torch.tensor(position, dtype=torch.long, device=self.device)
205
- self._buf_player[0] = int(board.turn)
206
-
207
- castling = 0
208
- if board.has_kingside_castling_rights(chess.WHITE):
209
- castling |= 1
210
- if board.has_queenside_castling_rights(chess.WHITE):
211
- castling |= 2
212
- if board.has_kingside_castling_rights(chess.BLACK):
213
- castling |= 4
214
- if board.has_queenside_castling_rights(chess.BLACK):
215
- castling |= 8
216
- self._buf_castling[0] = castling
217
-
218
- ep_file = chess.square_file(board.ep_square) if board.has_legal_en_passant() else 8
219
- self._buf_ep[0] = ep_file
220
-
221
- move_logits, value_t = self.model(
222
- self._buf_board, self._buf_player, self._buf_castling, self._buf_ep
223
- )
224
-
225
- logits = move_logits[0].float().cpu().numpy()
226
- value = float(value_t.item())
227
-
228
- if self._tt_size > 0 and len(self._tt) >= self._tt_size:
229
- del self._tt[next(iter(self._tt))]
230
- self._tt[key] = (logits, value)
231
-
232
- return logits, value
233
-
234
- def _batch_buffers(self, bucket: int):
235
- """Persistent zero-padded device buffers per bucket size.
236
-
237
- Using a small fixed set of batch shapes keeps torch.compile / CUDA
238
- graphs from recompiling (and re-capturing) on every distinct batch size.
239
- """
240
- bufs = self._batch_bufs.get(bucket)
241
- if bufs is None:
242
- bufs = (
243
- torch.zeros(bucket, 64, dtype=torch.long, device=self.device),
244
- torch.zeros(bucket, dtype=torch.long, device=self.device),
245
- torch.zeros(bucket, dtype=torch.long, device=self.device),
246
- torch.zeros(bucket, dtype=torch.long, device=self.device),
247
- )
248
- self._batch_bufs[bucket] = bufs
249
- return bufs
250
-
251
- @staticmethod
252
- def _bucket_for(n: int) -> int:
253
- for b in _BATCH_BUCKETS:
254
- if n <= b:
255
- return b
256
- return _BATCH_BUCKETS[-1]
257
-
258
- @torch.no_grad()
259
- def _batch_run(self, items: list[tuple[np.ndarray, int, int, int, int]]):
260
- n = len(items)
261
- bucket = self._bucket_for(n)
262
- board_arr, player_arr, castling_arr, ep_arr = self._batch_buffers(bucket)
263
- # Zero the padding rows (token/index 0 is a valid embedding index).
264
- board_arr.zero_()
265
- player_arr.zero_()
266
- castling_arr.zero_()
267
- ep_arr.zero_()
268
-
269
- for i, (pos, player, castling, ep, _) in enumerate(items):
270
- board_arr[i] = torch.tensor(pos, dtype=torch.long)
271
- player_arr[i] = player
272
- castling_arr[i] = castling
273
- ep_arr[i] = ep
274
-
275
- move_logits, value_t = self.model(board_arr, player_arr, castling_arr, ep_arr)
276
- # Slice off padding before the host copy.
277
- logits_np = move_logits[:n].float().cpu().numpy()
278
- values_np = value_t[:n].float().cpu().numpy()
279
-
280
- for i, (_, _, _, _, key) in enumerate(items):
281
- if self._tt_size > 0 and len(self._tt) >= self._tt_size:
282
- del self._tt[next(iter(self._tt))]
283
- self._tt[key] = (logits_np[i], float(values_np[i].item()))
284
-
285
- def _encode_position(self, board: chess.Board) -> tuple[np.ndarray, int, int, int]:
286
- position = self.position_tokenizer.encode(board)
287
- player = int(board.turn)
288
- castling = 0
289
- if board.has_kingside_castling_rights(chess.WHITE):
290
- castling |= 1
291
- if board.has_queenside_castling_rights(chess.WHITE):
292
- castling |= 2
293
- if board.has_kingside_castling_rights(chess.BLACK):
294
- castling |= 4
295
- if board.has_queenside_castling_rights(chess.BLACK):
296
- castling |= 8
297
- ep_file = chess.square_file(board.ep_square) if board.has_legal_en_passant() else 8
298
- return position, player, castling, ep_file
299
-
300
- def _prefetch_children(self, board: chess.Board, moves: list[chess.Move]):
301
- uncached = []
302
- for move in moves:
303
- board.push(move)
304
- key = chess.polyglot.zobrist_hash(board)
305
- # Cheap terminal proxy: positions with no legal moves (checkmate /
306
- # stalemate) are scored directly by _alpha_beta, so skip a wasted
307
- # model eval on them. Claimable-draw positions (threefold / 50-move)
308
- # still have legal moves; caching their eval is harmless because
309
- # _alpha_beta's authoritative is_game_over check overrides it.
310
- if key not in self._tt and any(board.legal_moves):
311
- pos, player, castling, ep_file = self._encode_position(board)
312
- uncached.append((pos, player, castling, ep_file, key))
313
- board.pop()
314
- if len(uncached) >= 2:
315
- self._batch_run(uncached)
316
-
317
- def _top_p_for_depth(self, depth: int, max_depth: int) -> float:
318
- plies_from_root = max_depth - depth
319
- if plies_from_root == 0:
320
- return self.top_p
321
- elif plies_from_root <= 2:
322
- return self.top_p * 0.7
323
- else:
324
- return self.top_p * 0.5
325
-
326
- def _search_root(self, board: chess.Board, depth: int) -> tuple[chess.Move, float]:
327
- ordered_moves = self._order_moves(board, self.top_p)
328
- if not ordered_moves:
329
- return chess.Move.null(), 0.0
330
- self._prefetch_children(board, [m for m, _ in ordered_moves])
331
- best_move = ordered_moves[0][0]
332
- alpha = float("-inf")
333
- for move, _ in ordered_moves:
334
- self._check_time()
335
- board.push(move)
336
- value = -self._alpha_beta(board, depth - 1, -float("inf"), -alpha, depth)
337
- board.pop()
338
- if value > alpha:
339
- alpha = value
340
- best_move = move
341
- return best_move, alpha
342
-
343
- def _terminal_value(self, board: chess.Board) -> float | None:
344
- """Negamax terminal value (side-to-move POV), or None if not terminal.
345
-
346
- Uses cheap checks instead of ``is_game_over(claim_draw=True)`` (whose
347
- ``can_claim_threefold_repetition`` dominated the profile): no-legal-move
348
- positions are mate/stalemate, and draws are detected via material,
349
- 3-fold repetition and the 50-move clock — without the expensive
350
- "a move *could* force a repetition" probe.
351
- """
352
- if not any(board.legal_moves):
353
- return -1.0 if board.is_check() else 0.0
354
- if (
355
- board.is_insufficient_material()
356
- or board.is_repetition(3)
357
- or board.halfmove_clock >= 100
358
- ):
359
- return 0.0
360
- return None
361
-
362
- @torch.no_grad()
363
- def _quiescence(self, board: chess.Board, alpha: float, beta: float, qdepth: int) -> float:
364
- """Resolve tactical noise before trusting the value head.
365
-
366
- Stand-pat on the value head, then search only forcing moves (captures
367
- and promotions) so the leaf isn't evaluated in the middle of an
368
- unresolved exchange (the horizon effect).
369
- """
370
- self.nodes_searched += 1
371
- self._check_time()
372
-
373
- term = self._terminal_value(board)
374
- if term is not None:
375
- return term
376
-
377
- logits, stand_pat = self._evaluate(board)
378
- if stand_pat >= beta:
379
- return beta
380
- if stand_pat > alpha:
381
- alpha = stand_pat
382
- if qdepth <= 0:
383
- return alpha
384
-
385
- noisy = []
386
- for move in board.legal_moves:
387
- if board.is_capture(move) or move.promotion:
388
- plane = move_to_action_plane(move.from_square, move.to_square, move.promotion)
389
- noisy.append((float(logits[move.from_square, plane]), move))
390
- if not noisy:
391
- return alpha
392
- noisy.sort(key=lambda x: -x[0])
393
-
394
- for _, move in noisy:
395
- board.push(move)
396
- score = -self._quiescence(board, -beta, -alpha, qdepth - 1)
397
- board.pop()
398
- if score >= beta:
399
- return beta
400
- if score > alpha:
401
- alpha = score
402
- return alpha
403
-
404
- def _alpha_beta(self, board: chess.Board, depth: int, alpha: float, beta: float, max_depth: int) -> float:
405
- self.nodes_searched += 1
406
- self._check_time()
407
-
408
- term = self._terminal_value(board)
409
- if term is not None:
410
- return term
411
-
412
- if depth <= 0:
413
- if self.quiescence_depth > 0:
414
- return self._quiescence(board, alpha, beta, self.quiescence_depth)
415
- _, value = self._evaluate(board)
416
- return value
417
-
418
- p = self._top_p_for_depth(depth, max_depth)
419
- ordered_moves = self._order_moves(board, p)
420
- self._prefetch_children(board, [m for m, _ in ordered_moves])
421
-
422
- for move, _ in ordered_moves:
423
- board.push(move)
424
- score = -self._alpha_beta(board, depth - 1, -beta, -alpha, max_depth)
425
- board.pop()
426
- if score >= beta:
427
- return beta
428
- if score > alpha:
429
- alpha = score
430
- return alpha
431
-
432
- def _order_moves(self, board: chess.Board, p: float, min_moves: int = 2, max_moves: int = 30):
433
- logits, _ = self._evaluate(board)
434
- moves = []
435
- raw_scores = []
436
- for move in board.legal_moves:
437
- plane = move_to_action_plane(move.from_square, move.to_square, move.promotion)
438
- raw_scores.append(float(logits[move.from_square, plane]))
439
- moves.append(move)
440
- if not moves:
441
- return []
442
-
443
- scores = np.array(raw_scores, dtype=np.float64)
444
- scores -= scores.max()
445
- exp_scores = np.exp(scores)
446
- probs = exp_scores / exp_scores.sum()
447
-
448
- order = np.argsort(-probs)
449
- cumsum = 0.0
450
- selected = []
451
- for idx in order:
452
- selected.append((moves[idx], float(probs[idx])))
453
- cumsum += probs[idx]
454
- if cumsum >= p and len(selected) >= min_moves:
455
- break
456
- if len(selected) >= max_moves:
457
- break
458
-
459
- while len(selected) < min_moves and len(selected) < len(moves):
460
- selected.append((moves[order[len(selected)]], float(probs[order[len(selected)]])))
461
-
462
- return selected
463
-
464
-
465
- if __name__ == "__main__":
466
- import argparse
467
- import time
468
-
469
- parser = argparse.ArgumentParser()
470
- parser.add_argument("model_dir", nargs="?", default=str(_DEFAULT_RUN))
471
- parser.add_argument("depth", nargs="?", type=int, default=3)
472
- parser.add_argument("top_p", nargs="?", type=float, default=0.90)
473
- parser.add_argument("--ema", action="store_true")
474
- args = parser.parse_args()
475
-
476
- bot = Pos2MoveV2Bot(
477
- model_dir=args.model_dir, depth=args.depth, top_p=args.top_p,
478
- time_limit=30.0, use_ema=args.ema,
479
- )
480
-
481
- board = chess.Board()
482
- print(f"\nStarting position:\n{board}\n")
483
-
484
- for i in range(10):
485
- t0 = time.time()
486
- move_uci, value = bot.predict(board)
487
- elapsed = time.time() - t0
488
- print(
489
- f"{'White' if board.turn else 'Black'} plays {move_uci} "
490
- f"(depth={bot.completed_depth}/{args.depth}, value={value:+.3f}, "
491
- f"nodes={bot.nodes_searched}, tt_hits={bot.tt_hits}, time={elapsed:.2f}s)"
492
- )
493
- board.push_uci(move_uci)
494
- print(board)
495
- print()
496
-
497
- if board.is_game_over():
498
- print(f"Game over: {board.result()}")
499
- break
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
chesstransformer/bots/pos2move_v2_mcts_bot.py DELETED
@@ -1,318 +0,0 @@
1
- """AlphaZero-style PUCT MCTS engine for Pos2MoveV2.
2
-
3
- Subclasses :class:`Pos2MoveV2Bot` to reuse its (torch.compiled, transposition-
4
- cached) ``_evaluate`` — the policy head supplies move priors and the value head
5
- scores leaves. Move selection is by visit count (the standard AlphaZero choice).
6
-
7
- Node representation is **edge-centric**: a node stores its children's visit
8
- counts / values / priors as numpy arrays, and child ``_Node`` objects are created
9
- lazily only when a child is actually descended into. This avoids allocating a
10
- node per legal move at every expansion (which dominated latency) and lets PUCT
11
- selection run vectorized over the arrays. Leaves are collected in batches with
12
- virtual loss so the network sees one batched forward per wave.
13
- """
14
-
15
- from __future__ import annotations
16
-
17
- import math
18
-
19
- import chess
20
- import numpy as np
21
-
22
- from chesstransformer.bots.pos2move_v2_bot import _BATCH_BUCKETS, _DEFAULT_RUN, Pos2MoveV2Bot
23
- from chesstransformer.models.tokenizer.alphazero_move_encoder import move_to_action_plane
24
-
25
-
26
- class _Node:
27
- """A search node. Child visit stats live in arrays here (edge-centric); child
28
- ``_Node`` objects are created lazily on first descent. A node is *expanded*
29
- once ``moves`` is set."""
30
-
31
- __slots__ = ("moves", "priors", "child_N", "child_W", "children")
32
-
33
- def __init__(self):
34
- self.moves: list[chess.Move] | None = None
35
- self.priors: np.ndarray | None = None
36
- self.child_N: np.ndarray | None = None # int64, per child
37
- self.child_W: np.ndarray | None = None # float64, child-POV total value
38
- self.children: list[_Node | None] | None = None
39
-
40
- @property
41
- def expanded(self) -> bool:
42
- return self.moves is not None
43
-
44
-
45
- class Pos2MoveV2MctsBot(Pos2MoveV2Bot):
46
- def __init__(
47
- self,
48
- *args,
49
- num_simulations: int = 800,
50
- c_puct: float = 1.0,
51
- prior_temp: float = 1.0,
52
- fpu: float | None = 0.2,
53
- sim_batch: int = 16,
54
- tree_reuse: bool = True,
55
- move_temp: float = 0.0,
56
- move_temp_plies: int = 0,
57
- **kwargs,
58
- ):
59
- super().__init__(*args, **kwargs)
60
- self.num_simulations = num_simulations
61
- self.c_puct = c_puct
62
- # Move selection. By default the move is argmax(visit count) — fully
63
- # deterministic (what the gauntlet / API rely on). For variety (e.g. so
64
- # the opening isn't identical every game), set move_temp > 0 to *sample*
65
- # the move ∝ visit_count**(1/move_temp) for the first move_temp_plies
66
- # half-moves, then fall back to argmax. move_temp=1 samples proportional
67
- # to visits; higher is more random.
68
- self.move_temp = move_temp
69
- self.move_temp_plies = move_temp_plies
70
- # Tree reuse: keep the searched subtree under the moves actually played
71
- # and re-root it next move, so prior visits carry over (effectively
72
- # deeper search for free). Requires the caller to keep board.move_stack
73
- # across calls; falls back to a fresh tree otherwise (e.g. boards rebuilt
74
- # from FEN with no history).
75
- self.tree_reuse = tree_reuse
76
- self._reuse_root: _Node | None = None
77
- self._reuse_moves: list | None = None
78
- # prior_temp > 1 flattens the policy priors so good-but-low-prior moves
79
- # still get explored (the value head can then promote them). fpu (first-
80
- # play-urgency): value unvisited children as parent_Q - fpu; None keeps
81
- # the legacy behavior (unvisited contribute 0 to selection).
82
- self.prior_temp = prior_temp
83
- self.fpu = fpu
84
- # Leaves collected per network call. >1 amortizes the GPU->CPU sync that
85
- # otherwise dominates (one stall per simulation). 1 = legacy single-leaf.
86
- # Capped at the largest batch bucket so _batch_run's preallocated buffers
87
- # (and the captured CUDA graphs) are never overrun.
88
- self.sim_batch = max(1, min(sim_batch, _BATCH_BUCKETS[-1]))
89
-
90
- # ── expansion ────────────────────────────────────────────────────────
91
- def _compute_priors(self, moves: list[chess.Move], logits: np.ndarray) -> np.ndarray:
92
- """Softmax policy priors over legal moves, with temperature flattening."""
93
- scores = np.array(
94
- [
95
- logits[m.from_square, move_to_action_plane(m.from_square, m.to_square, m.promotion)]
96
- for m in moves
97
- ],
98
- dtype=np.float64,
99
- )
100
- scores /= self.prior_temp
101
- scores -= scores.max()
102
- exp = np.exp(scores)
103
- return exp / exp.sum()
104
-
105
- def _expand_node(self, node: _Node, moves: list[chess.Move], priors: np.ndarray):
106
- self.nodes_searched += 1
107
- n = len(moves)
108
- node.moves = moves
109
- node.priors = priors
110
- node.child_N = np.zeros(n, dtype=np.int64)
111
- node.child_W = np.zeros(n, dtype=np.float64)
112
- node.children = [None] * n
113
-
114
- def _expand_leaf(self, node: _Node, board: chess.Board) -> float:
115
- """Evaluate ``board`` (policy + value) and expand ``node``; return value."""
116
- logits, value = self._evaluate(board)
117
- moves = list(board.legal_moves)
118
- self._expand_node(node, moves, self._compute_priors(moves, logits))
119
- return value
120
-
121
- # ── selection / descent / backup ─────────────────────────────────────
122
- def _select_idx(self, node: _Node) -> int:
123
- """Vectorized PUCT: pick the child index maximizing -Q + U."""
124
- N = node.child_N
125
- total = int(N.sum())
126
- sqrt_total = math.sqrt(1.0 + total) # +1 ≈ the node's own eval visit
127
- # value-to-parent = -child_Q (child stores value from its own POV).
128
- q_parent = np.zeros_like(node.child_W)
129
- np.divide(-node.child_W, N, out=q_parent, where=N > 0)
130
- if self.fpu is not None:
131
- parent_q = (-node.child_W.sum() / total) if total > 0 else 0.0
132
- q_parent[N == 0] = parent_q - self.fpu
133
- u = (self.c_puct * sqrt_total) * node.priors / (1.0 + N)
134
- return int(np.argmax(q_parent + u))
135
-
136
- def _descend(self, node: _Node, board: chess.Board, apply_vl: bool):
137
- """Walk to an unexpanded leaf, pushing moves. Returns (leaf_node, path),
138
- where path is the list of (node, child_idx) edges traversed."""
139
- path: list[tuple[_Node, int]] = []
140
- while node.expanded:
141
- idx = self._select_idx(node)
142
- if apply_vl:
143
- # Virtual loss: a virtual *win* in the child's POV looks like a
144
- # loss to this node under -Q selection, so concurrent descents in
145
- # the batch avoid the path.
146
- node.child_N[idx] += 1
147
- node.child_W[idx] += 1.0
148
- path.append((node, idx))
149
- board.push(node.moves[idx])
150
- child = node.children[idx]
151
- if child is None:
152
- child = _Node()
153
- node.children[idx] = child
154
- node = child
155
- return node, path
156
-
157
- @staticmethod
158
- def _backup(path: list[tuple[_Node, int]], value: float):
159
- """Negamax backup over edges (no virtual loss)."""
160
- v = value
161
- for node, idx in reversed(path):
162
- node.child_N[idx] += 1
163
- node.child_W[idx] += v
164
- v = -v
165
-
166
- @staticmethod
167
- def _backup_vl(path: list[tuple[_Node, int]], value: float):
168
- """Backup that undoes the per-edge virtual loss (+1 added during descent)
169
- and applies the real, sign-alternating value. N already counted."""
170
- v = value
171
- for node, idx in reversed(path):
172
- node.child_W[idx] += v - 1.0
173
- v = -v
174
-
175
- # ── search drivers ───────────────────────────────────────────────────
176
- def _simulate(self, root: _Node, board: chess.Board):
177
- node, path = self._descend(root, board, apply_vl=False)
178
- pushed = len(path)
179
- term = self._terminal_value(board)
180
- value = term if term is not None else self._expand_leaf(node, board)
181
- self._backup(path, value)
182
- for _ in range(pushed):
183
- board.pop()
184
-
185
- def _run_batch(self, root: _Node, board: chess.Board, batch_n: int):
186
- """Collect up to ``batch_n`` leaves (virtual loss during descent),
187
- evaluate the unique uncached non-terminal ones in one batched forward,
188
- then expand and back them up."""
189
- leaves = [] # (path, leaf_node, key, moves, cached eval or None)
190
- items_by_key = {} # key -> (pos, player, castling, ep, key)
191
-
192
- for _ in range(batch_n):
193
- node, path = self._descend(root, board, apply_vl=True)
194
- term = self._terminal_value(board)
195
- if term is not None:
196
- self._backup_vl(path, term)
197
- else:
198
- key = chess.polyglot.zobrist_hash(board)
199
- moves = list(board.legal_moves)
200
- # Reuse cached policy+value (transpositions / reused subtree /
201
- # prior moves); only batch positions we haven't evaluated yet.
202
- # Capture the cached entry *now*: once the TT is at capacity,
203
- # _batch_run's inserts evict oldest entries, so a hit found here
204
- # may no longer be in the TT after the batch runs.
205
- cached = self._tt.get(key)
206
- if cached is not None:
207
- self.tt_hits += 1
208
- elif key not in items_by_key:
209
- pos, player, castling, ep = self._encode_position(board)
210
- items_by_key[key] = (pos, player, castling, ep, key)
211
- leaves.append((path, node, key, moves, cached))
212
- for _ in range(len(path)):
213
- board.pop()
214
-
215
- if items_by_key:
216
- self._batch_run(list(items_by_key.values())) # populates self._tt
217
-
218
- for path, node, key, moves, cached in leaves:
219
- logits, value = cached if cached is not None else self._tt[key]
220
- if not node.expanded:
221
- self._expand_node(node, moves, self._compute_priors(moves, logits))
222
- self._backup_vl(path, value)
223
-
224
- # ── tree reuse ───────────────────────────────────────────────────────
225
- def _take_reuse_root(self, board: chess.Board) -> _Node | None:
226
- """Re-root the retained tree to ``board`` if it follows the previous
227
- search position by a few plies. Consumes the stored tree."""
228
- root = self._reuse_root
229
- self._reuse_root = None
230
- if root is None or self._reuse_moves is None:
231
- return None
232
- ms = board.move_stack
233
- rp = len(self._reuse_moves)
234
- gap = len(ms) - rp
235
- if gap < 1 or gap > 4 or list(ms[:rp]) != self._reuse_moves:
236
- return None
237
- node = root
238
- for mv in ms[rp:]:
239
- if not node.expanded:
240
- return None
241
- try:
242
- idx = node.moves.index(mv)
243
- except ValueError:
244
- return None
245
- child = node.children[idx]
246
- if child is None or not child.expanded:
247
- return None
248
- node = child
249
- return node if node.expanded else None
250
-
251
- def predict(self, board: chess.Board) -> tuple[str, float | None]:
252
- self.nodes_searched = 0
253
- self.tt_hits = 0
254
- self.completed_depth = self.num_simulations
255
-
256
- if not any(board.legal_moves):
257
- return chess.Move.null().uci(), 0.0
258
-
259
- root = self._take_reuse_root(board) if self.tree_reuse else None
260
- if root is None:
261
- root = _Node()
262
- self._expand_leaf(root, board)
263
-
264
- if self.sim_batch <= 1:
265
- for _ in range(self.num_simulations):
266
- self._simulate(root, board)
267
- else:
268
- done = 0
269
- while done < self.num_simulations:
270
- n = min(self.sim_batch, self.num_simulations - done)
271
- self._run_batch(root, board, n)
272
- done += n
273
-
274
- ply = 2 * (board.fullmove_number - 1) + (0 if board.turn == chess.WHITE else 1)
275
- if self.move_temp > 0 and ply < self.move_temp_plies and root.child_N.sum() > 0:
276
- # Sample ∝ visit_count**(1/temp) over the opening plies for variety.
277
- weights = root.child_N.astype(np.float64) ** (1.0 / self.move_temp)
278
- total = weights.sum()
279
- best_idx = int(np.random.choice(len(weights), p=weights / total)) if total > 0 \
280
- else int(np.argmax(root.child_N))
281
- else:
282
- best_idx = int(np.argmax(root.child_N))
283
- best_move = root.moves[best_idx]
284
- n = int(root.child_N[best_idx])
285
- # Root value from side-to-move POV (negate child's POV).
286
- best_value = float(-root.child_W[best_idx] / n) if n > 0 else 0.0
287
-
288
- if self.tree_reuse:
289
- self._reuse_root = root
290
- self._reuse_moves = list(board.move_stack)
291
-
292
- return best_move.uci(), best_value
293
-
294
-
295
- if __name__ == "__main__":
296
- import argparse
297
- import time
298
-
299
- parser = argparse.ArgumentParser()
300
- parser.add_argument("model_dir", nargs="?", default=str(_DEFAULT_RUN))
301
- parser.add_argument("--sims", type=int, default=800)
302
- parser.add_argument("--c-puct", type=float, default=1.0)
303
- parser.add_argument("--ema", action="store_true")
304
- args = parser.parse_args()
305
-
306
- bot = Pos2MoveV2MctsBot(
307
- model_dir=args.model_dir, num_simulations=args.sims, c_puct=args.c_puct,
308
- time_limit=0.0, use_ema=args.ema,
309
- )
310
- board = chess.Board()
311
- for _ in range(8):
312
- t0 = time.time()
313
- mv, val = bot.predict(board)
314
- print(f"{'W' if board.turn else 'B'} {mv} val={val:+.3f} "
315
- f"sims={bot.num_simulations} nodes={bot.nodes_searched} time={time.time()-t0:.2f}s")
316
- board.push_uci(mv)
317
- if board.is_game_over():
318
- break
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
chesstransformer/bots/position2move_bot.py DELETED
@@ -1,88 +0,0 @@
1
- from pathlib import Path
2
- import json
3
- import torch
4
- import chess
5
- from safetensors import safe_open
6
- from chesstransformer.models.transformer.position2move import Position2MoveModel
7
- from chesstransformer.models.tokenizer import PostionTokenizer, MoveTokenizer
8
-
9
- data_foler = Path(__file__).parents[1] / "data"
10
-
11
-
12
- class Position2MoveBot:
13
- def __init__(
14
- self,
15
- model_path: str = str((data_foler / "models/position2moveV2.1/best_model/model.safetensors").resolve()),
16
- device: str = "cpu",
17
- ):
18
- self.device = device
19
- self.position_tokenizer = PostionTokenizer()
20
- self.move_tokenizer = MoveTokenizer()
21
-
22
- config_path = Path(model_path).parent / "config.json"
23
-
24
- with open(config_path, "r") as f:
25
- self.config = json.load(f)
26
-
27
- # Load model
28
- self.model = Position2MoveModel(**self.config).to(device)
29
-
30
- with safe_open(model_path, framework="pt", device=device) as f:
31
- state_dict = {k: f.get_tensor(k) for k in f.keys()}
32
-
33
- # Handle compiled model prefix (_orig_mod.)
34
- if any(k.startswith("_orig_mod.") for k in state_dict.keys()):
35
- state_dict = {k.replace("_orig_mod.", ""): v for k, v in state_dict.items()}
36
-
37
- self.model.load_state_dict(state_dict)
38
-
39
- self.model.eval()
40
-
41
- @torch.no_grad()
42
- def predict(self, board: chess.Board):
43
- tokens_ids = self.position_tokenizer.encode(board)
44
- torch_input = torch.tensor(tokens_ids).unsqueeze(0).long().to(self.device) # Add batch dimension
45
-
46
- is_white = board.turn
47
- is_white = torch.tensor([is_white]).bool().to(self.device) # Add batch dimension
48
-
49
- logits = self.model(torch_input, is_white)
50
-
51
- legal_moves = [m.uci() for m in board.legal_moves]
52
-
53
- mask = torch.full((logits.size(-1),), float("-inf")).to(self.device)
54
- for move in legal_moves:
55
- move_id = self.move_tokenizer.encode(move)
56
- mask[move_id] = 0.0
57
-
58
- masked_logits = logits[0] + mask # Assuming batch size of 1
59
- probs = torch.softmax(masked_logits, dim=-1)
60
- # draw the move from the distribution
61
- predicted_move_id = torch.multinomial(probs * 0.2, num_samples=1).item()
62
- proba = probs[predicted_move_id].item()
63
-
64
- predicted_move = self.move_tokenizer.decode(predicted_move_id)
65
- return predicted_move, proba
66
-
67
-
68
- if __name__ == "__main__":
69
- bot = Position2MoveBot()
70
-
71
- board = chess.Board()
72
-
73
- print("Initial board:")
74
- print(board)
75
-
76
- move, proba = bot.predict(board)
77
- print(f"Bot suggests move: {move} with probability {proba:.4f}")
78
-
79
- board.push_uci(move)
80
- print("Board after bot move:")
81
- print(board)
82
-
83
- move, proba = bot.predict(board)
84
- print(f"Bot suggests move: {move} with probability {proba:.4f}")
85
-
86
- board.push_uci(move)
87
- print("Board after bot move:")
88
- print(board)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
chesstransformer/bots/random_bot.py DELETED
@@ -1,33 +0,0 @@
1
- import chess
2
- import random
3
-
4
-
5
- class RandomBot:
6
- def __init__(self):
7
- pass
8
-
9
- def predict(self, board: chess.Board) -> chess.Move:
10
- legal_moves = list(board.legal_moves)
11
- if not legal_moves:
12
- raise ValueError("No legal moves available.")
13
- # pick a random move
14
- move = random.choice(legal_moves)
15
- return move.uci(), None
16
-
17
-
18
- if __name__ == "__main__":
19
- bot = RandomBot()
20
- board = chess.Board()
21
- print("Initial board:")
22
- print(board)
23
- move = bot.predict(board)
24
- print(f"RandomBot suggests move: {move}")
25
- board.push(move)
26
- print("Board after move:")
27
- print(board)
28
-
29
- move = bot.predict(board)
30
- print(f"RandomBot suggests move: {move}")
31
- board.push(move)
32
- print("Board after move:")
33
- print(board)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
chesstransformer/datasets/__init__.py DELETED
@@ -1,10 +0,0 @@
1
- from .lichess import LichessDataset
2
- from .lichess_simple_uci import LichessSimpleUciDataset
3
- from .puzzle_dataset import LichessPuzzleDataset, LichessPuzzleFullSolutionDataset
4
-
5
- __all__ = [
6
- "LichessDataset",
7
- "LichessSimpleUciDataset",
8
- "LichessPuzzleDataset",
9
- "LichessPuzzleFullSolutionDataset",
10
- ]
 
 
 
 
 
 
 
 
 
 
 
chesstransformer/datasets/dataset_h5_convertor.py DELETED
@@ -1,404 +0,0 @@
1
- import h5py
2
- import chess.pgn
3
- import zstandard as zstd
4
- import numpy as np
5
- from tqdm import tqdm
6
- from multiprocessing import Pool, cpu_count
7
- from chesstransformer.models.tokenizer.move_tokenizer import MoveTokenizer
8
- import io
9
- from pathlib import Path
10
-
11
- # Per-worker globals initialised once per process
12
- _worker_tokenizer = None
13
- _worker_exclude_tcs = []
14
-
15
-
16
- def _worker_init(exclude_tcs):
17
- global _worker_tokenizer, _worker_exclude_tcs
18
- _worker_tokenizer = MoveTokenizer()
19
- _worker_exclude_tcs = exclude_tcs
20
-
21
-
22
- def _process_raw_game(raw_pgn: str):
23
- """Full pipeline in each worker: text-filter → parse → tokenize.
24
-
25
- Moving parse into workers parallelises the most expensive step.
26
- The fast text-level filter avoids even calling chess.pgn.read_game
27
- for excluded time controls.
28
- """
29
- global _worker_tokenizer, _worker_exclude_tcs
30
-
31
- # Fast text-level filter before expensive PGN parsing
32
- if _worker_exclude_tcs:
33
- idx = raw_pgn.find('[Event "')
34
- if idx >= 0:
35
- end = raw_pgn.find('"]', idx + 8)
36
- if end >= 0:
37
- event_lower = raw_pgn[idx + 8 : end].lower()
38
- if any(tc in event_lower for tc in _worker_exclude_tcs):
39
- return None
40
-
41
- game = chess.pgn.read_game(io.StringIO(raw_pgn))
42
- if game is None:
43
- return None
44
-
45
- if game.headers.get("Variant", "Standard") != "Standard":
46
- return None
47
-
48
- result_str = game.headers.get("Result", "*")
49
- if result_str == "1/2-1/2":
50
- result = 0
51
- elif result_str == "1-0":
52
- result = 1
53
- elif result_str == "0-1":
54
- result = 2
55
- else:
56
- return None
57
-
58
- try:
59
- white_elo = int(game.headers.get("WhiteElo", "1500"))
60
- except ValueError:
61
- white_elo = 1500
62
- try:
63
- black_elo = int(game.headers.get("BlackElo", "1500"))
64
- except ValueError:
65
- black_elo = 1500
66
-
67
- tokens = []
68
- for move in game.mainline_moves():
69
- try:
70
- tokens.append(_worker_tokenizer.encode(move.uci()))
71
- except ValueError:
72
- return None
73
-
74
- if len(tokens) < 10:
75
- return None
76
-
77
- return np.array(tokens, dtype=np.int16), white_elo, black_elo, result, len(tokens)
78
-
79
-
80
- def _iter_raw_games(stream):
81
- """Yield individual PGN game strings from a text stream.
82
-
83
- Splits on '[Event' lines, which always start each game in standard PGN.
84
- Line-by-line reading keeps memory usage constant regardless of file size.
85
- """
86
- buf = []
87
- for line in stream:
88
- if line.startswith("[Event ") and buf:
89
- yield "".join(buf)
90
- buf = [line]
91
- else:
92
- buf.append(line)
93
- if buf:
94
- text = "".join(buf).strip()
95
- if text:
96
- yield text
97
-
98
-
99
- class PGNtoHDF5Converter:
100
- def __init__(
101
- self,
102
- output_path: str,
103
- chunk_size: int = 10000,
104
- num_workers: int = None,
105
- exclude_time_controls: list = None,
106
- ):
107
- """
108
- Convert PGN files to HDF5 format storing complete games as UCI move sequences.
109
-
110
- Args:
111
- output_path: Path to save the HDF5 file
112
- chunk_size: Number of games to accumulate before writing (default: 10000)
113
- exclude_time_controls: List of time control categories to skip, matched against
114
- the Lichess Event header (e.g. ["Bullet", "Blitz", "UltraBullet"])
115
- """
116
- self.output_path = output_path
117
- self.chunk_size = chunk_size
118
- default_workers = (cpu_count() or 1) - 1
119
- self.num_workers = max(1, num_workers if num_workers is not None else max(1, default_workers))
120
- self.exclude_time_controls = [tc.lower() for tc in (exclude_time_controls or [])]
121
-
122
- def convert(self, input_path: str, max_games: int = None):
123
- """Convert PGN files to HDF5. Input can be a file or folder."""
124
- path = Path(input_path)
125
- if not path.exists():
126
- raise FileNotFoundError(f"Path not found: {input_path}")
127
- if path.is_file():
128
- self._convert_single_file(path, max_games)
129
- elif path.is_dir():
130
- self._convert_folder(path, max_games)
131
- else:
132
- raise ValueError(f"Invalid input path: {input_path}")
133
-
134
- def _convert_folder(self, folder_path: Path, max_games: int = None):
135
- pgn_files = sorted(list(folder_path.glob("*.pgn")) + list(folder_path.glob("*.pgn.zst")))
136
- if not pgn_files:
137
- raise ValueError(f"No PGN files found in {folder_path}")
138
-
139
- done = self._completed_files()
140
- pending = [f for f in pgn_files if f.name not in done]
141
-
142
- print(f"Found {len(pgn_files)} PGN file(s) in {folder_path}")
143
- if done:
144
- print(f"Resuming: {len(done)} already converted, {len(pending)} remaining")
145
- for f in pending:
146
- print(f" - {f.name}")
147
- print()
148
-
149
- if not pending:
150
- print("Nothing to do — every file is already in the output.")
151
- return 0
152
-
153
- total_games = 0
154
- started = bool(done)
155
- for pgn_file in pending:
156
- remaining = None if max_games is None else max(0, max_games - total_games)
157
- if remaining == 0:
158
- break
159
- print(f"\nProcessing: {pgn_file.name}")
160
- total_games += self._convert_single_file(pgn_file, remaining, append=started)
161
- started = True
162
- if max_games and total_games >= max_games:
163
- print(f"\nReached maximum game limit ({max_games})")
164
- break
165
-
166
- print(f"\n{'=' * 60}")
167
- print(f"All files processed! Total games: {total_games}")
168
- print(f"Output: {self.output_path}")
169
-
170
- def _convert_single_file(self, pgn_path: Path, max_games: int = None, append: bool = False):
171
- if pgn_path.suffix == ".zst" or pgn_path.name.endswith(".pgn.zst"):
172
- return self._convert_compressed_file(str(pgn_path), max_games, append)
173
- else:
174
- return self._convert_uncompressed_file(str(pgn_path), max_games, append)
175
-
176
- def convert_pgn_zst(self, pgn_path: str, max_games: int = None):
177
- """Deprecated: Use convert() instead."""
178
- return self._convert_compressed_file(pgn_path, max_games, append=False)
179
-
180
- # ------------------------------------------------------------------
181
- # Internal helpers
182
- # ------------------------------------------------------------------
183
-
184
- _DATASET_NAMES = ("moves", "white_elo", "black_elo", "result", "num_moves")
185
-
186
- def _completed_files(self) -> list[str]:
187
- """Names of PGN files already fully written into the output, for resume."""
188
- path = Path(self.output_path)
189
- if not path.exists():
190
- return []
191
- try:
192
- with h5py.File(path, "r") as f:
193
- return [str(n) for n in f.attrs.get("converted_files", [])]
194
- except OSError:
195
- return []
196
-
197
- def _mark_converted(self, hdf5_file, name: str):
198
- """Commit a finished PGN file: record its name and the row watermark."""
199
- done = [str(n) for n in hdf5_file.attrs.get("converted_files", [])]
200
- if name not in done:
201
- done.append(name)
202
- hdf5_file.attrs["converted_files"] = np.array(done, dtype=h5py.string_dtype())
203
- hdf5_file.attrs["committed_rows"] = int(min(hdf5_file[n].shape[0] for n in self._DATASET_NAMES))
204
-
205
- def _rewind_to_watermark(self, hdf5_file):
206
- """Discard everything written after the last fully converted PGN file.
207
-
208
- Length agreement is not enough to detect an interrupted run: `_write_batch`
209
- resizes all five datasets before writing any of them, so a crash mid-batch
210
- leaves them equal-length with a junk tail. Worse, the rows a dead run *did*
211
- write would be duplicated when its PGN file — never added to `converted_files`
212
- — gets reconverted. Rewinding to the committed watermark drops both.
213
- """
214
- lengths = {n: hdf5_file[n].shape[0] for n in self._DATASET_NAMES}
215
- mark = int(hdf5_file.attrs.get("committed_rows", -1))
216
- if mark < 0: # written before watermarks existed; fall back to length agreement
217
- mark = min(lengths.values())
218
- if any(length != mark for length in lengths.values()):
219
- print(f"Rewinding interrupted output: {lengths} -> {mark} rows")
220
- for name in self._DATASET_NAMES:
221
- hdf5_file[name].resize((mark,))
222
-
223
- def _create_datasets(self, hdf5_file):
224
- """Create all HDF5 datasets (called only when not appending)."""
225
- dt = h5py.vlen_dtype(np.dtype("int16"))
226
- for name, dtype in [
227
- ("moves", dt),
228
- ("white_elo", "int16"),
229
- ("black_elo", "int16"),
230
- ("result", "int8"),
231
- ("num_moves", "int16"),
232
- ]:
233
- hdf5_file.create_dataset(
234
- name,
235
- shape=(0,),
236
- maxshape=(None,),
237
- dtype=dtype,
238
- compression="gzip",
239
- compression_opts=4,
240
- chunks=(self.chunk_size,),
241
- )
242
-
243
- def _write_batch(self, hdf5_file, games, white_elos, black_elos, results, num_moves):
244
- old = hdf5_file["moves"].shape[0]
245
- new = old + len(games)
246
- for ds in ("moves", "white_elo", "black_elo", "result", "num_moves"):
247
- hdf5_file[ds].resize((new,))
248
- # One slice assignment, not one write per game: each per-game write re-read and
249
- # re-compressed the whole gzip chunk, costing ~1 ms/game in the main process and
250
- # starving the workers.
251
- moves_arr = np.empty(len(games), dtype=object)
252
- moves_arr[:] = games
253
- try:
254
- hdf5_file["moves"][old:new] = moves_arr
255
- except TypeError:
256
- # h5py re-normalises the object array internally, and when every game in the
257
- # batch happens to have the same length (always so for a single game) that
258
- # collapses it to 2-D and the vlen write is refused. Only degenerate batches
259
- # hit this — typically the 1-game remainder of a file's final flush — so the
260
- # slow path costs nothing in practice.
261
- for i, g in enumerate(games):
262
- hdf5_file["moves"][old + i] = g
263
- hdf5_file["white_elo"][old:new] = np.array(white_elos, dtype=np.int16)
264
- hdf5_file["black_elo"][old:new] = np.array(black_elos, dtype=np.int16)
265
- hdf5_file["result"][old:new] = np.array(results, dtype=np.int8)
266
- hdf5_file["num_moves"][old:new] = np.array(num_moves, dtype=np.int16)
267
-
268
- def _run_conversion(self, stream, hdf5_file, max_games) -> tuple[int, int]:
269
- """Core loop: stream raw PGN text → workers parse+tokenize → write HDF5.
270
-
271
- Returns (game_count, total_positions).
272
- """
273
- game_count = 0
274
- total_positions = 0
275
- games_buf, we_buf, be_buf, res_buf, nm_buf = [], [], [], [], []
276
-
277
- pbar = tqdm(desc="Processing games", unit="games")
278
-
279
- def flush():
280
- if games_buf:
281
- self._write_batch(hdf5_file, games_buf, we_buf, be_buf, res_buf, nm_buf)
282
- games_buf.clear(); we_buf.clear(); be_buf.clear()
283
- res_buf.clear(); nm_buf.clear()
284
-
285
- def accumulate(result):
286
- nonlocal game_count, total_positions
287
- token_array, we, be, res, nm = result
288
- games_buf.append(token_array)
289
- we_buf.append(we); be_buf.append(be)
290
- res_buf.append(res); nm_buf.append(nm)
291
- total_positions += nm
292
- game_count += 1
293
- pbar.update(1)
294
- if len(games_buf) >= self.chunk_size:
295
- flush()
296
-
297
- use_parallel = self.num_workers > 1
298
- raw_iter = _iter_raw_games(stream)
299
-
300
- if use_parallel:
301
- pool = Pool(
302
- processes=self.num_workers,
303
- initializer=_worker_init,
304
- initargs=(self.exclude_time_controls,),
305
- )
306
- early_stop = False
307
- try:
308
- # imap_unordered streams results as soon as each worker finishes —
309
- # no need to wait for a full chunk_size batch before seeing progress.
310
- for result in pool.imap_unordered(_process_raw_game, raw_iter, chunksize=200):
311
- if result is None:
312
- continue
313
- accumulate(result)
314
- if max_games and game_count >= max_games:
315
- early_stop = True
316
- break
317
- finally:
318
- if early_stop:
319
- pool.terminate()
320
- else:
321
- pool.close()
322
- pool.join()
323
- else:
324
- # Sequential fallback (single worker)
325
- _worker_init(self.exclude_time_controls)
326
- for raw_pgn in raw_iter:
327
- result = _process_raw_game(raw_pgn)
328
- if result is None:
329
- continue
330
- accumulate(result)
331
- if max_games and game_count >= max_games:
332
- break
333
-
334
- flush()
335
- pbar.close()
336
- return game_count, total_positions
337
-
338
- def _convert_compressed_file(self, pgn_path: str, max_games: int = None, append: bool = False):
339
- with open(pgn_path, "rb") as f:
340
- dctx = zstd.ZstdDecompressor()
341
- with dctx.stream_reader(f) as reader:
342
- stream = io.TextIOWrapper(reader, encoding="utf-8")
343
- mode = "a" if append else "w"
344
- with h5py.File(self.output_path, mode) as hdf5_file:
345
- if append and "moves" in hdf5_file:
346
- self._rewind_to_watermark(hdf5_file)
347
- else:
348
- self._create_datasets(hdf5_file)
349
- game_count, total_positions = self._run_conversion(stream, hdf5_file, max_games)
350
- if not max_games or game_count < max_games:
351
- self._mark_converted(hdf5_file, Path(pgn_path).name)
352
-
353
- self._print_summary(game_count, total_positions)
354
- return game_count
355
-
356
- def _convert_uncompressed_file(self, pgn_path: str, max_games: int = None, append: bool = False):
357
- with open(pgn_path, "r", encoding="utf-8") as pgn_file:
358
- mode = "a" if append else "w"
359
- with h5py.File(self.output_path, mode) as hdf5_file:
360
- if append and "moves" in hdf5_file:
361
- self._rewind_to_watermark(hdf5_file)
362
- else:
363
- self._create_datasets(hdf5_file)
364
- game_count, total_positions = self._run_conversion(pgn_file, hdf5_file, max_games)
365
- if not max_games or game_count < max_games:
366
- self._mark_converted(hdf5_file, Path(pgn_path).name)
367
-
368
- self._print_summary(game_count, total_positions)
369
- return game_count
370
-
371
- def _print_summary(self, game_count, total_positions):
372
- print("File conversion complete!")
373
- print(f"Games from this file: {game_count}")
374
- print(f"Positions from this file: {total_positions}")
375
- avg = (total_positions / game_count) if game_count else 0.0
376
- print(f"Average moves per game: {avg:.1f}")
377
-
378
-
379
- # Usage script
380
- if __name__ == "__main__":
381
- import argparse
382
-
383
- parser = argparse.ArgumentParser(description="Convert PGN/PGN.zst file(s) or folder to HDF5")
384
- parser.add_argument("--input", type=str, required=True, help="Input .pgn/.pgn.zst file or folder")
385
- parser.add_argument("--output", type=str, required=True, help="Output .h5 file")
386
- parser.add_argument("--max-games", type=int, default=None, help="Maximum games to process")
387
- parser.add_argument("--chunk-size", type=int, default=10000, help="Batch size for writing")
388
- parser.add_argument("--num-workers", type=int, default=None, help="Worker processes (default: cpu count - 1)")
389
- parser.add_argument(
390
- "--exclude-time-controls",
391
- nargs="+",
392
- default=[],
393
- metavar="TC",
394
- help="Time control categories to skip (e.g. Bullet Blitz UltraBullet)",
395
- )
396
-
397
- args = parser.parse_args()
398
- converter = PGNtoHDF5Converter(
399
- args.output,
400
- chunk_size=args.chunk_size,
401
- num_workers=args.num_workers,
402
- exclude_time_controls=args.exclude_time_controls,
403
- )
404
- converter.convert(args.input, max_games=args.max_games)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
chesstransformer/datasets/flat_shard_dataset.py DELETED
@@ -1,231 +0,0 @@
1
- """Memmap-backed dataset over the flat shards written by ``scripts/build_shards.py``.
2
-
3
- Implements ``doc/dataset_shards.md`` §4. Every expensive thing ``HDF5ChessDataset`` does
4
- per sample — gzip decompression, python-chess replay, legal-move enumeration, tokenizing —
5
- has already been paid at build time. What is left here is a numpy gather out of page cache.
6
-
7
- Two decisions carry most of the speed:
8
-
9
- **Batch-level fetch.** At 202 bytes per sample the per-item Python call overhead dominates
10
- the actual work, so ``__getitem__`` accepts a *list* of indices and does one vectorised
11
- gather per batch. Drive it with :func:`make_dataloader`, which wires up a ``BatchSampler``
12
- and ``batch_size=None`` so the DataLoader hands the whole index list through untouched.
13
-
14
- **The legal mask is not built on CPU.** Returning ``(64, 73)`` bools would put 2.4 MB per
15
- batch of 512 back on the H2D wire — the exact cost the shard format exists to remove. The
16
- loader ships the ``(B, 64)`` uint16 index list (64 KB) and the training loop calls
17
- :func:`scatter_legal_planes` on device.
18
-
19
- **Distributed use.** This is a map-style dataset, so rank splitting is
20
- ``DistributedSampler``'s job, not the dataset's — see :func:`make_dataloader`. Map-style +
21
- ``DistributedSampler`` is preferred over an ``IterableDataset`` that owns shard *k* when
22
- ``k % (world_size * num_workers) == rank * num_workers + worker`` because the latter can
23
- only shuffle within a worker's own shards: sample correlation then survives into the batch,
24
- and the assignment silently unbalances whenever ``world_size * num_workers`` does not
25
- divide the shard count. The shards here are a build-parallelism and train/val/test-split
26
- unit; they deliberately do not constrain who reads what.
27
- """
28
-
29
- import json
30
- import os
31
- from pathlib import Path
32
-
33
- import numpy as np
34
- import torch
35
- from torch.utils.data import BatchSampler, DataLoader, RandomSampler, SequentialSampler
36
-
37
- from chesstransformer.datasets.shard_format import (
38
- FORMAT_VERSION,
39
- NUM_ACTION_PLANES,
40
- NUM_SQUARES,
41
- PAD_ACTION,
42
- dtype_from_meta,
43
- )
44
-
45
- #: Fields handed to the trainer, in the dtype the model wants them in. Kept narrow on the
46
- #: wire (uint8/int16) and widened on device — an int64 ``position`` would be 8x the copy
47
- #: for an embedding lookup that accepts int32 anyway.
48
- _SCALAR_FIELDS = ("from_square", "action_plane", "castling_rights", "en_passant_file", "is_white")
49
-
50
-
51
- class FlatShardDataset(torch.utils.data.Dataset):
52
- """One split of a shard directory, as one addressable array of samples."""
53
-
54
- def __init__(self, shard_dir: str | Path, split: str = "train"):
55
- self.shard_dir = Path(shard_dir)
56
- self.split = split
57
-
58
- with (self.shard_dir / "meta.json").open() as fh:
59
- self.meta = json.load(fh)
60
-
61
- if self.meta["format_version"] != FORMAT_VERSION:
62
- raise ValueError(
63
- f"{self.shard_dir} was built with format_version "
64
- f"{self.meta['format_version']}, this code reads {FORMAT_VERSION}. Rebuild."
65
- )
66
-
67
- if split not in self.meta["splits"]:
68
- raise KeyError(f"unknown split {split!r}; have {list(self.meta['splits'])}")
69
-
70
- self.dtype = dtype_from_meta(self.meta["record_dtype"])
71
- self.pad_action = int(self.meta["pad_action"])
72
-
73
- wanted = set(self.meta["splits"][split])
74
- self.shard_meta = [m for m in self.meta["shards"] if m["shard_id"] in wanted]
75
- if not self.shard_meta:
76
- raise ValueError(f"split {split!r} of {self.shard_dir} is empty")
77
-
78
- # Cumulative offsets: a global index resolves to (shard, row) with one searchsorted.
79
- counts = np.array([m["num_samples"] for m in self.shard_meta], dtype=np.int64)
80
- self.offsets = np.concatenate([[0], np.cumsum(counts)])
81
- self._len = int(self.offsets[-1])
82
-
83
- # Opened lazily, per process. np.memmap survives fork harmlessly, but building the
84
- # handles in __init__ would map every shard in the parent for no reason.
85
- self._shards = None
86
- self._shards_pid = None
87
-
88
- # The whole point of memmap is that the page cache holds one copy that all workers
89
- # share. np.load(...) without mmap_mode would give each of N workers its own.
90
- self.game_ranges = [(m["game_start"], m["game_end"]) for m in self.shard_meta]
91
-
92
- def __len__(self):
93
- return self._len
94
-
95
- @property
96
- def shards(self):
97
- pid = os.getpid()
98
- if self._shards is None or self._shards_pid != pid:
99
- self._shards = [
100
- np.load(self.shard_dir / m["path"], mmap_mode="r") for m in self.shard_meta
101
- ]
102
- self._shards_pid = pid
103
- return self._shards
104
-
105
- def __getstate__(self):
106
- state = self.__dict__.copy()
107
- state["_shards"] = None
108
- state["_shards_pid"] = None
109
- return state
110
-
111
- def _gather(self, idx: np.ndarray) -> np.ndarray:
112
- """Fetch records for a batch of global indices, preserving their order."""
113
- shard_of = np.searchsorted(self.offsets, idx, side="right") - 1
114
- rows = idx - self.offsets[shard_of]
115
-
116
- shards = self.shards
117
- out = np.empty(len(idx), dtype=self.dtype)
118
- for sid in np.unique(shard_of):
119
- where = np.nonzero(shard_of == sid)[0]
120
- r = rows[where]
121
- # Read each shard in ascending row order: sequential page touches, even though
122
- # the batch itself is shuffled.
123
- order = np.argsort(r, kind="stable")
124
- out[where[order]] = shards[sid][r[order]]
125
- return out
126
-
127
- def __getitem__(self, index):
128
- """Accepts a single index or — the fast path — a list/array of them."""
129
- if isinstance(index, (int, np.integer)):
130
- batch = self._gather(np.array([index], dtype=np.int64))
131
- else:
132
- batch = self._gather(np.asarray(index, dtype=np.int64))
133
-
134
- # Everything stays at its on-disk width. A batch of 512 is then 103 KB rather than
135
- # the 5.81 MB the HDF5 path collates, and the widening to int64/bool happens on
136
- # device in the trainer's unpack_batch, where it is free.
137
- #
138
- # int16 rather than uint16 for the two u2 fields because torch has no uint16; both
139
- # are safely in range (legal_idx <= 4672, move_number max observed 601).
140
- out = {
141
- "position": torch.from_numpy(np.ascontiguousarray(batch["position"])),
142
- "legal_idx": torch.from_numpy(np.ascontiguousarray(batch["legal_idx"]).astype(np.int16)),
143
- "legal_cnt": torch.from_numpy(np.ascontiguousarray(batch["legal_cnt"])),
144
- "result": torch.from_numpy(np.ascontiguousarray(batch["result"])),
145
- "move_number": torch.from_numpy(
146
- np.ascontiguousarray(batch["move_number"]).astype(np.int16)
147
- ),
148
- }
149
- for name in _SCALAR_FIELDS:
150
- out[name] = torch.from_numpy(np.ascontiguousarray(batch[name]))
151
- return out
152
-
153
- def game_id_set(self) -> set:
154
- """Game indices this split covers. Used by the split-disjointness assertion."""
155
- ids = set()
156
- for g0, g1 in self.game_ranges:
157
- ids.update(range(g0, g1))
158
- return ids
159
-
160
-
161
- def scatter_legal_planes(legal_idx: torch.Tensor, pad_action: int = PAD_ACTION) -> torch.Tensor:
162
- """Rebuild the ``(B, 64, 73)`` bool legal mask from the packed index list, on device.
163
-
164
- ``legal_idx`` is ``(B, C)`` with unused slots set to ``pad_action`` (== 64*73), one past
165
- the end of the action space. Scattering into a tensor with one extra column parks that
166
- padding in a scratch slot which is then sliced off — if padding were 0 instead, every
167
- sample would come back with square a1 / plane 0 marked legal.
168
- """
169
- b = legal_idx.size(0)
170
- flat = torch.zeros(b, pad_action + 1, dtype=torch.bool, device=legal_idx.device)
171
- flat.scatter_(1, legal_idx.long(), True)
172
- return flat[:, :pad_action].view(b, NUM_SQUARES, NUM_ACTION_PLANES)
173
-
174
-
175
- def worker_init_fn(worker_id: int):
176
- """Give every worker (and every rank) its own numpy RNG stream.
177
-
178
- PyTorch reseeds ``random`` and ``torch`` per worker but *not* ``numpy``, so all workers
179
- inherit the parent's numpy state and draw the identical sequence — and under DDP so
180
- would every rank. Nothing in this dataset's hot path uses ``np.random`` any more, but
181
- ``HDF5ChessDataset._sample_move_idx`` does, and the fix belongs wherever a worker starts.
182
- """
183
- rank = int(os.environ.get("RANK", 0))
184
- np.random.seed((torch.initial_seed() + 977 * rank) % (2**32))
185
-
186
-
187
- def make_dataloader(
188
- dataset: FlatShardDataset,
189
- batch_size: int,
190
- shuffle: bool,
191
- num_workers: int = 4,
192
- drop_last: bool = False,
193
- pin_memory: bool = True,
194
- prefetch_factor: int = 4,
195
- seed: int = 0,
196
- distributed: bool = False,
197
- ) -> DataLoader:
198
- """Build the batch-fetch DataLoader this dataset is designed for.
199
-
200
- ``batch_size=None`` disables the DataLoader's own batching so the ``BatchSampler``'s
201
- index *list* reaches ``__getitem__`` intact and the gather happens once per batch
202
- rather than once per sample. Without this the dataset still works and runs roughly an
203
- order of magnitude slower.
204
-
205
- ``distributed=True`` wraps a ``DistributedSampler`` so each rank sees a disjoint
206
- 1/world_size of the split. Remember to call ``loader.sampler.sampler.set_epoch(epoch)``
207
- — the BatchSampler nests it one level down.
208
- """
209
- if distributed:
210
- from torch.utils.data import DistributedSampler
211
-
212
- inner = DistributedSampler(dataset, shuffle=shuffle, seed=seed, drop_last=drop_last)
213
- elif shuffle:
214
- generator = torch.Generator()
215
- generator.manual_seed(seed)
216
- inner = RandomSampler(dataset, generator=generator)
217
- else:
218
- inner = SequentialSampler(dataset)
219
-
220
- sampler = BatchSampler(inner, batch_size=batch_size, drop_last=drop_last)
221
-
222
- return DataLoader(
223
- dataset,
224
- batch_size=None,
225
- sampler=sampler,
226
- num_workers=num_workers,
227
- pin_memory=pin_memory,
228
- persistent_workers=num_workers > 0,
229
- prefetch_factor=prefetch_factor if num_workers > 0 else None,
230
- worker_init_fn=worker_init_fn,
231
- )
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
chesstransformer/datasets/h5_lichess_dataset.py DELETED
@@ -1,295 +0,0 @@
1
- import os
2
-
3
- import h5py
4
- import torch
5
- from torch.utils.data import Dataset
6
- import numpy as np
7
- import chess
8
- from chesstransformer.models.tokenizer.position_tokenizer import PostionTokenizer
9
- from chesstransformer.models.tokenizer.move_tokenizer import MoveTokenizer
10
- from chesstransformer.models.tokenizer.alphazero_move_encoder import (
11
- move_to_action_plane,
12
- NUM_ACTION_PLANES,
13
- )
14
-
15
-
16
- class HDF5ChessDataset(Dataset):
17
- def __init__(
18
- self,
19
- hdf5_path: str,
20
- cache_size: int = 500,
21
- min_elo: int = None,
22
- max_elo: int = None,
23
- sample_weighting: str = "middlegame",
24
- skip_opening_plies: int = 0,
25
- rdcc_nbytes: int = 32 << 20,
26
- ):
27
- """
28
- Efficient dataset that loads games as UCI sequences and samples random positions.
29
-
30
- Each dataset index corresponds to one game. When a sample is requested, it:
31
- 1. Uses the index to select a game
32
- 2. Samples a position within that game (uniform or phase-weighted)
33
- 3. Replays the game to that position to get the board state
34
- 4. Returns the encoded board state + the next move
35
-
36
- Args:
37
- hdf5_path: Path to HDF5 file containing games
38
- cache_size: Number of games to cache in memory
39
- min_elo: Minimum average ELO for filtering games (optional)
40
- max_elo: Maximum average ELO for filtering games (optional)
41
- sample_weighting: "uniform" (legacy) or "middlegame" (triangular peak around move 25)
42
- skip_opening_plies: Force sampling to start at this ply (0 = no restriction)
43
- rdcc_nbytes: HDF5 chunk cache per dataset. ``moves`` is gzipped in chunks of
44
- 10,000 games, so a random read decompresses a whole chunk; h5py's 1 MB
45
- default cannot hold one and every read pays full decompression. 32 MB
46
- takes a random ``moves[g]`` from 0.22 ms to 0.12 ms.
47
- """
48
- self.hdf5_path = hdf5_path
49
- self.cache_size = cache_size
50
- self.game_cache = {} # Cache for game move sequences
51
- self.position_tokenizer = PostionTokenizer()
52
- self.move_tokenizer = MoveTokenizer()
53
- self.sample_weighting = sample_weighting
54
- self.skip_opening_plies = skip_opening_plies
55
- self.rdcc_nbytes = rdcc_nbytes
56
-
57
- # Long-lived read handle, opened lazily by _h5(). See the note there for why
58
- # this must never be opened in __init__.
59
- self._file = None
60
- self._file_pid = None
61
-
62
- # Load metadata and filter games by ELO
63
- with h5py.File(hdf5_path, "r") as f:
64
- self.num_games = f["moves"].shape[0]
65
- num_moves = f["num_moves"][:]
66
- white_elos = f["white_elo"][:]
67
- black_elos = f["black_elo"][:]
68
- results = f["result"][:]
69
-
70
- # Filter games by ELO if specified
71
- valid_games = np.ones(self.num_games, dtype=bool)
72
- if min_elo is not None or max_elo is not None:
73
- avg_elos = (white_elos + black_elos) / 2
74
- if min_elo is not None:
75
- valid_games &= avg_elos >= min_elo
76
- if max_elo is not None:
77
- valid_games &= avg_elos <= max_elo
78
-
79
- # Store valid game indices and their move counts
80
- self.valid_game_indices = np.where(valid_games)[0]
81
- self.num_moves_per_game = num_moves[valid_games]
82
-
83
- # Per-game metadata stays in RAM: 1.53M games is 1.5 MB of int8 plus
84
- # 3 MB of int16, and reading these three scalars back out of HDF5 per
85
- # sample cost more than the game decode itself. Kept *unfiltered*, so
86
- # they are indexed by the actual game index, not the dataset index.
87
- self.white_elos = white_elos
88
- self.black_elos = black_elos
89
- self.results = results
90
-
91
- total_positions = int(self.num_moves_per_game.sum())
92
- print(f"Loaded dataset: {len(self.valid_game_indices)} games with {total_positions} total positions")
93
- if min_elo or max_elo:
94
- print(f"Filtered by ELO: {min_elo or 0} - {max_elo or 'inf'}")
95
-
96
- def __len__(self):
97
- """Return the number of games in the dataset."""
98
- return len(self.valid_game_indices)
99
-
100
- def _sample_move_idx(self, max_move_idx: int) -> int:
101
- """Sample a position index within a game.
102
-
103
- - "uniform": legacy behavior, np.random.randint(0, max_move_idx)
104
- - "middlegame": triangular weight peaking around ply 25,
105
- downweights opening (rare unique positions) and very late
106
- endgame (sparse signal). Falls back to uniform for short games.
107
-
108
- Optionally enforces ``skip_opening_plies`` as a hard floor.
109
- """
110
- lo = min(self.skip_opening_plies, max_move_idx - 1) if max_move_idx > 1 else 0
111
- hi = max_move_idx # exclusive upper bound
112
-
113
- if self.sample_weighting == "uniform" or hi - lo <= 8:
114
- return int(np.random.randint(lo, hi))
115
-
116
- # Triangular weights: ramp up over plies 0->20, flat 20-40, taper 40->80
117
- plies = np.arange(lo, hi)
118
- ramp_up = np.minimum(plies / 20.0, 1.0)
119
- taper = np.minimum(np.maximum((80.0 - plies) / 40.0, 0.1), 1.0)
120
- weights = ramp_up * taper
121
- weights = weights / weights.sum()
122
- return int(np.random.choice(plies, p=weights))
123
-
124
- def _h5(self) -> h5py.File:
125
- """Return this *process's* HDF5 handle, opening it on first use.
126
-
127
- Deliberately not opened in ``__init__``. DataLoader workers are forked, and
128
- HDF5 is not fork-safe: a handle inherited across ``fork()`` is shared library
129
- state that two processes then mutate independently. It does not raise — it
130
- returns corrupted data or deadlocks, which is far worse. Opening on first
131
- ``__getitem__`` means every worker gets its own handle after the fork.
132
-
133
- The PID check covers the case where the parent process reads a sample (e.g.
134
- ``num_workers=0``, or a shape probe) before spawning workers, which would
135
- otherwise leave an open handle for the children to inherit.
136
- """
137
- pid = os.getpid()
138
- if self._file is None or self._file_pid != pid:
139
- self._file = h5py.File(
140
- self.hdf5_path,
141
- "r",
142
- rdcc_nbytes=self.rdcc_nbytes,
143
- rdcc_nslots=10007, # prime, comfortably > chunks held in the cache
144
- )
145
- self._file_pid = pid
146
- return self._file
147
-
148
- def __getstate__(self):
149
- """Drop the handle when pickled (``spawn`` start method); h5py.File is not
150
- picklable and each process must open its own anyway."""
151
- state = self.__dict__.copy()
152
- state["_file"] = None
153
- state["_file_pid"] = None
154
- return state
155
-
156
- def _get_game_moves(self, game_idx: int) -> np.ndarray:
157
- """Load and cache a game's move sequence.
158
-
159
- Only ``moves`` comes from HDF5; the per-game scalars live in RAM (see __init__).
160
- """
161
- if game_idx in self.game_cache:
162
- return self.game_cache[game_idx]
163
-
164
- moves = self._h5()["moves"][game_idx]
165
-
166
- # Update cache (simple LRU-like behavior)
167
- if len(self.game_cache) >= self.cache_size:
168
- # Remove oldest item
169
- self.game_cache.pop(next(iter(self.game_cache)))
170
-
171
- self.game_cache[game_idx] = moves
172
- return moves
173
-
174
- def __getitem__(self, idx):
175
- """
176
- Returns a position-move pair by:
177
- 1. Using idx to select a game
178
- 2. Uniformly sampling a random position within that game
179
- 3. Replaying the game to get the board state
180
- 4. Returning encoded board + next move
181
- """
182
- # Map dataset index to actual game index
183
- actual_game_idx = self.valid_game_indices[idx]
184
-
185
- # Load game moves; the scalars are RAM lookups (indexed by actual game index)
186
- game_moves = self._get_game_moves(actual_game_idx)
187
- white_elo = int(self.white_elos[actual_game_idx])
188
- black_elo = int(self.black_elos[actual_game_idx])
189
- result = int(self.results[actual_game_idx])
190
- num_moves = self.num_moves_per_game[idx]
191
-
192
- # Uniformly sample a position within the game (exclude last move - no next move)
193
- max_move_idx = num_moves - 1
194
- if max_move_idx <= 0:
195
- # Edge case: game with only one move, use position 0
196
- move_idx = 0
197
- else:
198
- move_idx = self._sample_move_idx(int(max_move_idx))
199
-
200
- # Replay game to position
201
- board = chess.Board()
202
- for i in range(move_idx):
203
- # Decode move token to UCI string
204
- move_token = int(game_moves[i])
205
- move_uci = self._decode_move_token(move_token)
206
- try:
207
- move = chess.Move.from_uci(move_uci)
208
- board.push(move)
209
- except (ValueError, AssertionError):
210
- # If move is invalid, skip to next position
211
- # This shouldn't happen with properly encoded data
212
- pass
213
-
214
- # Encode current position
215
- position = self.position_tokenizer.encode(board)
216
- position_tensor = torch.tensor(position, dtype=torch.long)
217
-
218
- legal_moves_planes = torch.zeros(64, NUM_ACTION_PLANES, dtype=torch.bool)
219
- for move in board.legal_moves:
220
- plane = move_to_action_plane(move.from_square, move.to_square, move.promotion)
221
- legal_moves_planes[move.from_square, plane] = True
222
-
223
- # Get next move
224
- next_move_token = torch.tensor(int(game_moves[move_idx]), dtype=torch.long)
225
-
226
- # Parse next move for grid format (from/to squares + promotion)
227
- next_move_uci = self._decode_move_token(int(game_moves[move_idx]))
228
- next_move = chess.Move.from_uci(next_move_uci)
229
- from_square = next_move.from_square
230
- to_square = next_move.to_square
231
-
232
- # AlphaZero action plane for this move
233
- action_plane = move_to_action_plane(from_square, to_square, next_move.promotion)
234
-
235
- # Get turn indicator
236
- is_white = board.turn # True for white, False for black
237
-
238
- # Get castling rights as a single integer (0-15)
239
- # Bit 0: white kingside, Bit 1: white queenside, Bit 2: black kingside, Bit 3: black queenside
240
- castling_rights = 0
241
- if board.has_kingside_castling_rights(chess.WHITE):
242
- castling_rights |= 1
243
- if board.has_queenside_castling_rights(chess.WHITE):
244
- castling_rights |= 2
245
- if board.has_kingside_castling_rights(chess.BLACK):
246
- castling_rights |= 4
247
- if board.has_queenside_castling_rights(chess.BLACK):
248
- castling_rights |= 8
249
-
250
- # Get en passant file (0-7 for a-h, 8 for none)
251
- if board.has_legal_en_passant():
252
- en_passant_file = chess.square_file(board.ep_square)
253
- else:
254
- en_passant_file = 8 # No en passant
255
-
256
- return {
257
- "position": position_tensor,
258
- "move": next_move_token,
259
- "is_white": is_white,
260
- "castling_rights": castling_rights,
261
- "en_passant_file": en_passant_file,
262
- "move_number": move_idx,
263
- "white_elo": white_elo,
264
- "black_elo": black_elo,
265
- "result": result,
266
- "from_square": from_square,
267
- "action_plane": action_plane,
268
- "legal_moves_planes": legal_moves_planes,
269
- }
270
-
271
- def _decode_move_token(self, token: int) -> str:
272
- """Decode a move token back to UCI string."""
273
- return self.move_tokenizer.decode(token)
274
-
275
-
276
- if __name__ == "__main__":
277
- # Example usage
278
- import sys
279
-
280
- if len(sys.argv) < 2:
281
- print("Usage: python h5_lichess_dataset.py <path_to_h5_file>")
282
- sys.exit(1)
283
-
284
- dataset = HDF5ChessDataset(sys.argv[1], cache_size=100)
285
- print(f"Dataset length: {len(dataset)}")
286
-
287
- # Test a few samples
288
- for i in range(min(5, len(dataset))):
289
- sample = dataset[i]
290
- print(f"\nSample {i}:")
291
- print(f" Position shape: {sample['position'].shape}")
292
- print(f" Move token: {sample['move'].item()}")
293
- print(f" Turn: {'White' if sample['is_white'] else 'Black'}")
294
- print(f" ELOs: {sample['white_elo']} vs {sample['black_elo']}")
295
- print(f" Move number: {sample['move_number']} over {dataset.num_moves_per_game[i]} moves")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
chesstransformer/datasets/lichess.py DELETED
@@ -1,223 +0,0 @@
1
- from torch.utils.data import Dataset
2
- import zstandard as zstd
3
- import chess.pgn
4
- import io
5
- import torch
6
-
7
-
8
- def extract_games_optimized(filepath, num_games=1000000, batch_size=1000):
9
- """
10
- Optimized game extraction with progress tracking.
11
- Process in batches and show estimated time remaining.
12
-
13
- Args:
14
- filepath: Path to the compressed PGN file
15
- num_games: Number of games to extract (default: 1,000,000)
16
- batch_size: Number of games to process in each batch (default: 1,000)
17
-
18
- Returns:
19
- List of extracted games
20
- """
21
- games = []
22
- dctx = zstd.ZstdDecompressor()
23
- with open(filepath, "rb") as f:
24
- with dctx.stream_reader(f) as reader:
25
- text_stream = io.TextIOWrapper(reader, encoding="utf-8")
26
- pgn = chess.pgn.read_game(text_stream)
27
- game_count = 0
28
- batch_count = 0
29
-
30
- while pgn is not None and game_count < num_games:
31
- # check if the game ended by a win/loss/draw
32
- if pgn.headers.get("Result") not in ["1-0", "0-1", "1/2-1/2"]:
33
- pgn = chess.pgn.read_game(text_stream)
34
- continue
35
-
36
- # Skip Chess960 games - only use classical chess
37
- variant = pgn.headers.get("Variant", "Standard")
38
- if variant != "Standard":
39
- pgn = chess.pgn.read_game(text_stream)
40
- continue
41
-
42
- games.append(pgn)
43
- game_count += 1
44
- batch_count += 1
45
-
46
- if batch_count >= batch_size:
47
- batch_count = 0
48
- print(f"Extracted {game_count}/{num_games} games...")
49
-
50
- pgn = chess.pgn.read_game(text_stream)
51
-
52
- print(f"Finished extracting {len(games)} games.")
53
- return games
54
-
55
-
56
- def encode_game_moves(game, notation="uci", seprator=" <STEP> "):
57
- """
58
- Encode the moves of a single game using the provided tokenizer.
59
-
60
- Args:
61
- game: A chess.pgn.Game object
62
- notation: Notation format for moves ("uci" or "san", default: "uci")
63
- seprator: String to separate moves (default: " <STEP> ")
64
-
65
- Returns:
66
- List of moves as a single formatted string
67
- """
68
- if notation == "uci":
69
- move_sequence = seprator.join([move.uci() for move in game.mainline_moves()])
70
- elif notation == "san":
71
- board = game.board()
72
- move_sequence = seprator.join(
73
- [
74
- board.san(move)
75
- for move in game.mainline_moves()
76
- if not board.push(move) # Push the move to update the board state
77
- ]
78
- )
79
- else:
80
- raise ValueError("Unsupported notation format. Use 'uci' or 'san'.")
81
-
82
- ending_condition = seprator + {
83
- "1-0": "<1-0>",
84
- "0-1": "<0-1>",
85
- "1/2-1/2": "<1/2-1/2>",
86
- }.get(game.headers.get("Result", ""), "")
87
- if ending_condition:
88
- move_sequence += ending_condition
89
-
90
- return move_sequence
91
-
92
-
93
- class LichessDataset(Dataset):
94
- def __init__(
95
- self,
96
- dataset_path,
97
- tokenizer,
98
- notation="uci",
99
- context_length=512,
100
- nb_games=100000,
101
- ):
102
- """
103
- Lichess dataset for training chess move prediction models.
104
-
105
- Args:
106
- dataset_path: Path to the lichess dataset file (PGN format)
107
- tokenizer: Tokenizer object to convert moves to token IDs
108
- notation: Notation format for moves ("uci" or "san", default: "uci")
109
- context_length: Maximum sequence length for model input (default: 512)
110
- """
111
- self.tokenizer = tokenizer
112
- self.context_length = context_length
113
- self.notation = notation
114
-
115
- print(f"Loading dataset from {dataset_path}...")
116
- self.games = extract_games_optimized(dataset_path, num_games=nb_games)
117
- print(f"Total games loaded: {len(self.games)}")
118
-
119
- self.encoded_games = []
120
- # encode and stack games together
121
- for idx, game in enumerate(self.games):
122
- encoded_moves = encode_game_moves(game, notation=self.notation)
123
- token_ids = self.tokenizer.encode(encoded_moves).ids
124
-
125
- self.encoded_games.append(token_ids)
126
-
127
- # flatten the list of token ids
128
- self.encoded_games = [token_id for game in self.encoded_games for token_id in game]
129
-
130
- def __len__(self):
131
- return len(self.encoded_games) // self.context_length
132
-
133
- def __getitem__(self, idx):
134
- """
135
- Get a sequence of token IDs for training.
136
-
137
- Args:
138
- idx: Index of the game to retrieve
139
- Returns:
140
- input, target: Tuple of input and target token ID lists
141
- """
142
- start_idx = idx * self.context_length
143
- end_idx = start_idx + self.context_length + 1 # +1 for target shift
144
-
145
- if end_idx > len(self.encoded_games):
146
- raise IndexError("Index out of range for the dataset.")
147
-
148
- input_ids = self.encoded_games[start_idx : end_idx - 1]
149
- target_ids = self.encoded_games[start_idx + 1 : end_idx]
150
-
151
- input_tensor = torch.tensor(input_ids, dtype=torch.long)
152
- target_tensor = torch.tensor(target_ids, dtype=torch.long)
153
-
154
- return input_tensor, target_tensor
155
-
156
-
157
- if __name__ == "__main__":
158
- from pathlib import Path
159
- import argparse
160
- from chesstransformer.models.tokenizer import create_bpe_tokenizer
161
-
162
- def arg_parser():
163
- parser = argparse.ArgumentParser(description="Test LichessDataset")
164
- parser.add_argument(
165
- "--dataset_path",
166
- type=str,
167
- default="data/Lichess Rated Games 2017.pgn.zst",
168
- help="Path to the lichess dataset file (PGN format)",
169
- )
170
- parser.add_argument(
171
- "--num_games",
172
- type=int,
173
- default=10000,
174
- help="Number of games to load for testing (default: 1000)",
175
- )
176
- parser.add_argument(
177
- "--vocab_path",
178
- type=str,
179
- default="src/chesstransformer/data/tokenizer_models/bpe_tokenizer_vocab2000.json",
180
- help="Path to the trained tokenizer file",
181
- )
182
- parser.add_argument(
183
- "--context_length",
184
- type=int,
185
- default=512,
186
- help="Context length for sequences (default: 512)",
187
- )
188
- parser.add_argument(
189
- "--notation",
190
- type=str,
191
- choices=["uci", "san"],
192
- default="uci",
193
- help="Notation format for moves (default: uci)",
194
- )
195
- args = parser.parse_args()
196
- return args
197
-
198
- args = arg_parser()
199
- dataset_path = Path(args.dataset_path)
200
- num_games = args.num_games
201
- vocab_path = Path(args.vocab_path)
202
- context_length = args.context_length
203
- notation = args.notation
204
-
205
- import tokenizers
206
-
207
- tokenizer = tokenizers.Tokenizer.from_file(str(vocab_path))
208
- print(f"Loading Lichess dataset from {dataset_path}...")
209
- dataset = LichessDataset(
210
- dataset_path=dataset_path,
211
- tokenizer=tokenizer,
212
- notation=notation,
213
- context_length=context_length,
214
- nb_games=num_games,
215
- )
216
-
217
- print(f"Dataset loaded with {len(dataset)} games.")
218
- input_ids, target_ids = dataset[0]
219
- print(f"Sample input IDs: {input_ids}")
220
- print(f"Sample target IDs: {target_ids}")
221
- print(f"Encoded and stacked {len(dataset.encoded_games)} token IDs.")
222
-
223
- print(f"Decoded sample moves: {tokenizer.decode(input_ids)}")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
chesstransformer/datasets/lichess_simple_uci.py DELETED
@@ -1,136 +0,0 @@
1
- from pathlib import Path
2
- import json
3
- from torch.utils.data import Dataset
4
- import zstandard as zstd
5
- import chess.pgn
6
- import io
7
- import torch
8
- import chess
9
-
10
-
11
- root_path = Path(__file__).parents[1]
12
- print(f"Root path: {root_path}")
13
-
14
-
15
- class LichessSimpleUciDataset(Dataset):
16
- def __init__(
17
- self,
18
- dataset_path,
19
- vocab_path=root_path / "data/tokenizer_models/uci_vocab.json",
20
- context_length=512,
21
- num_samples=100_000,
22
- ):
23
- """
24
- Lichess dataset in the simple UCI format."""
25
-
26
- self.dataset_path = dataset_path
27
- self.num_samples = num_samples
28
- self.vocab_path = vocab_path
29
- self.context_length = context_length
30
- self._load_vocab()
31
- self.games = self._extract_games()
32
-
33
- def _extract_games(self, only_check_mate=True):
34
- games = []
35
- dctx = zstd.ZstdDecompressor()
36
- with open(self.dataset_path, "rb") as f:
37
- with dctx.stream_reader(f) as reader:
38
- text_stream = io.TextIOWrapper(reader, encoding="utf-8")
39
- game_count = 0
40
-
41
- while game_count < self.num_samples:
42
- pgn = chess.pgn.read_game(text_stream)
43
- if pgn is None:
44
- break
45
-
46
- # Check if the game ended by a win/loss/draw
47
- if pgn.headers.get("Result") not in ["1-0", "0-1", "1/2-1/2"]:
48
- continue
49
-
50
- # Skip Chess960 games - only use classical chess
51
- variant = pgn.headers.get("Variant", "Standard")
52
- if variant != "Standard":
53
- continue
54
-
55
- games.append(pgn)
56
- game_count += 1
57
-
58
- if game_count % 10000 == 0:
59
- print(f"Extracted {game_count}/{self.num_samples} games...", end="\r")
60
-
61
- print(f"\nFinished extracting {len(games)} games.")
62
- return games
63
-
64
- def _load_vocab(self):
65
- with open(self.vocab_path, "r") as f:
66
- self.vocab = json.load(f)
67
- self.token_to_id = {token: idx for idx, token in enumerate(self.vocab)}
68
-
69
- def __len__(self):
70
- return len(self.games)
71
-
72
- def __getitem__(self, idx):
73
- game = self.games[idx]
74
- token_ids = self.encode_game(game)
75
- legals_mask = self.get_legal_moves_mask(game)
76
- target_ids = token_ids[1:] + [-100] # Shifted targets with -100 for padding
77
- return {
78
- "input_ids": torch.tensor(token_ids, dtype=torch.long),
79
- "target_ids": torch.tensor(target_ids, dtype=torch.long),
80
- "legal_moves_mask": legals_mask,
81
- }
82
-
83
- def encode_game(self, game):
84
- move_list = [move.uci() for move in game.mainline_moves()]
85
- move_list = ["<START>"] + move_list + ["<END>"]
86
- token_ids = self.encode_moves(move_list)
87
- return token_ids
88
-
89
- def encode_moves(self, moves):
90
- token_ids = []
91
- for move in moves:
92
- if move in self.token_to_id:
93
- token_ids.append(self.token_to_id[move])
94
- else:
95
- token_ids.append(self.token_to_id["<UNK>"])
96
-
97
- return token_ids
98
-
99
- def get_legal_moves_mask(self, game):
100
- """
101
- For each move in the game, generate a mask of legal moves at that position.
102
- Returns a list of masks, where each mask is a tensor of shape (vocab_size,)
103
- with 1.0 for legal moves and 0.0 for illegal moves.
104
- """
105
- board = game.board()
106
- masks = []
107
-
108
- for move in game.mainline_moves():
109
- # Get all legal moves at current position
110
- legal_moves = [m.uci() for m in board.legal_moves]
111
-
112
- # Create mask for all tokens in vocabulary
113
- mask = torch.zeros(len(self.vocab), dtype=torch.float32)
114
-
115
- # Set 1.0 for legal moves
116
- for legal_move in legal_moves:
117
- if legal_move in self.token_to_id:
118
- mask[self.token_to_id[legal_move]] = 1.0
119
-
120
- masks.append(mask)
121
-
122
- # Apply the move to the board for next iteration
123
- board.push(move)
124
-
125
- # Add mask for <START> and <END> tokens
126
- start_end_mask = torch.zeros(len(self.vocab), dtype=torch.float32)
127
- start_end_mask[self.token_to_id["<START>"]] = 1.0
128
- start_end_mask[self.token_to_id["<END>"]] = 1.0
129
- masks = [start_end_mask] + masks + [start_end_mask]
130
-
131
- return torch.stack(masks)
132
-
133
- def decode_game(self, token_ids):
134
- id_to_token = {idx: token for token, idx in self.token_to_id.items()}
135
- moves = [id_to_token.get(tid, "<UNK>") for tid in token_ids if tid in id_to_token]
136
- return " ".join(moves)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
chesstransformer/datasets/puzzle_dataset.py DELETED
@@ -1,259 +0,0 @@
1
- import csv
2
- import zstandard as zstd
3
- import torch
4
- from torch.utils.data import Dataset
5
- import chess
6
- from typing import Optional
7
- from chesstransformer.models.tokenizer.position_tokenizer import PostionTokenizer
8
- from chesstransformer.models.tokenizer.move_tokenizer import MoveTokenizer
9
-
10
-
11
- class LichessPuzzleDataset(Dataset):
12
- """Dataset for Lichess chess puzzles.
13
-
14
- Each puzzle consists of:
15
- - A FEN position (the position where the puzzle starts)
16
- - A sequence of moves (the solution to the puzzle)
17
- - Rating and themes
18
-
19
- The dataset returns position-move pairs where:
20
- - position: The current board state encoded as 64 tokens
21
- - move: The next move in the puzzle solution
22
- - is_white: Whether white is to move
23
- """
24
-
25
- def __init__(
26
- self,
27
- puzzle_path: str,
28
- min_rating: int = None,
29
- max_rating: int = None,
30
- themes: list[str] = None,
31
- max_puzzles: int = None,
32
- ):
33
- """
34
- Args:
35
- puzzle_path: Path to the .csv.zst puzzle file
36
- min_rating: Minimum puzzle rating to include
37
- max_rating: Maximum puzzle rating to include
38
- themes: List of themes to filter by (e.g., ['mate', 'mateIn2'])
39
- max_puzzles: Maximum number of puzzles to load (for testing)
40
- """
41
- self.position_tokenizer = PostionTokenizer()
42
- self.move_tokenizer = MoveTokenizer()
43
- self.puzzles = []
44
-
45
- print(f"Loading puzzles from {puzzle_path}...")
46
-
47
- # Load and filter puzzles
48
- with zstd.open(puzzle_path, "rt", encoding="utf-8") as f:
49
- reader = csv.DictReader(f)
50
- for i, row in enumerate(reader):
51
- if max_puzzles and i >= max_puzzles:
52
- break
53
-
54
- # Parse puzzle data
55
- puzzle_rating = int(row["Rating"])
56
- puzzle_themes = row["Themes"].split()
57
-
58
- # Apply filters
59
- if min_rating and puzzle_rating < min_rating:
60
- continue
61
- if max_rating and puzzle_rating > max_rating:
62
- continue
63
- if themes and not any(theme in puzzle_themes for theme in themes):
64
- continue
65
-
66
- # Parse moves (space-separated UCI moves)
67
- moves = row["Moves"].split()
68
- if len(moves) < 2: # Need at least opponent move + our response
69
- continue
70
-
71
- self.puzzles.append(
72
- {
73
- "puzzle_id": row["PuzzleId"],
74
- "fen": row["FEN"],
75
- "moves": moves,
76
- "rating": puzzle_rating,
77
- "themes": puzzle_themes,
78
- "game_url": row["GameUrl"],
79
- }
80
- )
81
-
82
- if (i + 1) % 10000 == 0:
83
- print(f"Loaded {i + 1} puzzles...")
84
-
85
- print(f"Total puzzles loaded: {len(self.puzzles)}")
86
-
87
- # Calculate total position-move pairs
88
- self.total_pairs = sum(len(p["moves"]) - 1 for p in self.puzzles)
89
- print(f"Total position-move pairs: {self.total_pairs}")
90
-
91
- def __len__(self):
92
- """Return total number of position-move pairs across all puzzles."""
93
- return self.total_pairs
94
-
95
- def __getitem__(self, idx):
96
- """
97
- Returns a position-move pair from the puzzle dataset.
98
-
99
- The dataset is indexed as a flat sequence of all position-move pairs
100
- across all puzzles. This means we need to:
101
- 1. Find which puzzle this index belongs to
102
- 2. Find which move within that puzzle
103
- 3. Replay the game to get the board state
104
- 4. Return the encoded position + next move
105
- """
106
- # Find the puzzle and move index
107
- cumulative = 0
108
- for puzzle in self.puzzles:
109
- puzzle_pairs = len(puzzle["moves"]) - 1
110
- if idx < cumulative + puzzle_pairs:
111
- # Found the right puzzle
112
- move_idx = idx - cumulative
113
- break
114
- cumulative += puzzle_pairs
115
- else:
116
- raise IndexError(f"Index {idx} out of range")
117
-
118
- # Set up the board from FEN
119
- board = chess.Board(puzzle["fen"])
120
-
121
- # Apply moves up to (but not including) the target move
122
- for i in range(move_idx + 1): # +1 because first move is opponent's
123
- move = chess.Move.from_uci(puzzle["moves"][i])
124
- board.push(move)
125
-
126
- # Get the target move
127
- target_move = puzzle["moves"][move_idx + 1]
128
-
129
- # Encode the position
130
- position_tokens = self.position_tokenizer.encode(board)
131
-
132
- # Encode the move
133
- try:
134
- move_token = self.move_tokenizer.encode(target_move)
135
- except ValueError:
136
- # If move not in vocabulary, skip this sample
137
- # This shouldn't happen often with proper vocab
138
- print(f"Warning: Move {target_move} not in vocabulary for puzzle {puzzle['puzzle_id']}")
139
- move_token = 0 # Use a default
140
-
141
- # Get legal moves mask for loss calculation
142
- legal_moves = list(board.legal_moves)
143
- legal_moves_mask = torch.zeros(self.move_tokenizer.vocab_size, dtype=torch.float32)
144
- for move in legal_moves:
145
- try:
146
- move_id = self.move_tokenizer.encode(move.uci())
147
- legal_moves_mask[move_id] = 1.0
148
- except ValueError:
149
- continue
150
-
151
- return {
152
- "position": torch.tensor(position_tokens, dtype=torch.long),
153
- "move": torch.tensor(move_token, dtype=torch.long),
154
- "is_white": board.turn == chess.WHITE,
155
- "puzzle_id": puzzle["puzzle_id"],
156
- "puzzle_rating": puzzle["rating"],
157
- "legal_moves_mask": legal_moves_mask,
158
- "themes": puzzle["themes"],
159
- }
160
-
161
- def get_puzzle_by_id(self, puzzle_id: str):
162
- """Get a complete puzzle by its ID."""
163
- for puzzle in self.puzzles:
164
- if puzzle["puzzle_id"] == puzzle_id:
165
- return puzzle
166
- return None
167
-
168
-
169
- class LichessPuzzleFullSolutionDataset(Dataset):
170
- """Alternative puzzle dataset that returns complete puzzles (not individual moves).
171
-
172
- Useful for evaluation where you want to test solving the entire puzzle.
173
- """
174
-
175
- def __init__(
176
- self,
177
- puzzle_path: str,
178
- min_rating: int = None,
179
- max_rating: int = None,
180
- themes: list[str] = None,
181
- max_puzzles: int = None,
182
- ):
183
- """
184
- Args:
185
- puzzle_path: Path to the .csv.zst puzzle file
186
- min_rating: Minimum puzzle rating to include
187
- max_rating: Maximum puzzle rating to include
188
- themes: List of themes to filter by
189
- max_puzzles: Maximum number of puzzles to load
190
- """
191
- self.position_tokenizer = PostionTokenizer()
192
- self.move_tokenizer = MoveTokenizer()
193
- self.puzzles = []
194
-
195
- print(f"Loading puzzles from {puzzle_path}...")
196
-
197
- with zstd.open(puzzle_path, "rt", encoding="utf-8") as f:
198
- reader = csv.DictReader(f)
199
- for i, row in enumerate(reader):
200
- if max_puzzles and i >= max_puzzles:
201
- break
202
-
203
- puzzle_rating = int(row["Rating"])
204
- puzzle_themes = row["Themes"].split()
205
-
206
- if min_rating and puzzle_rating < min_rating:
207
- continue
208
- if max_rating and puzzle_rating > max_rating:
209
- continue
210
- if themes and not any(theme in puzzle_themes for theme in themes):
211
- continue
212
-
213
- moves = row["Moves"].split()
214
- if len(moves) < 2:
215
- continue
216
-
217
- self.puzzles.append(
218
- {
219
- "puzzle_id": row["PuzzleId"],
220
- "fen": row["FEN"],
221
- "moves": moves,
222
- "rating": puzzle_rating,
223
- "themes": puzzle_themes,
224
- "game_url": row["GameUrl"],
225
- }
226
- )
227
-
228
- print(f"Total puzzles loaded: {len(self.puzzles)}")
229
-
230
- def __len__(self):
231
- return len(self.puzzles)
232
-
233
- def __getitem__(self, idx):
234
- """Returns a complete puzzle with starting position and full solution."""
235
- puzzle = self.puzzles[idx]
236
-
237
- # Starting position
238
- board = chess.Board(puzzle["fen"])
239
- position_tokens = self.position_tokenizer.encode(board)
240
-
241
- # Encode solution moves
242
- solution_tokens = []
243
- for move_uci in puzzle["moves"]:
244
- try:
245
- move_token = self.move_tokenizer.encode(move_uci)
246
- solution_tokens.append(move_token)
247
- except ValueError:
248
- solution_tokens.append(0)
249
-
250
- return {
251
- "puzzle_id": puzzle["puzzle_id"],
252
- "position": torch.tensor(position_tokens, dtype=torch.long),
253
- "solution": torch.tensor(solution_tokens, dtype=torch.long),
254
- "fen": puzzle["fen"],
255
- "moves_uci": puzzle["moves"],
256
- "rating": puzzle["rating"],
257
- "themes": puzzle["themes"],
258
- "is_white": board.turn == chess.WHITE,
259
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
chesstransformer/datasets/shard_format.py DELETED
@@ -1,129 +0,0 @@
1
- """On-disk record format for the flat, memory-mappable training shards.
2
-
3
- Shared by ``scripts/build_shards.py`` (writer), ``FlatShardDataset`` (reader) and
4
- ``scripts/verify_shards.py`` (equivalence harness), so the layout is defined exactly
5
- once. See ``doc/dataset_shards.md`` §3 for the design rationale and the measurements
6
- behind the choices below.
7
-
8
- One sample is one fixed-width row. Only the nine fields ``pos2move_v2_trainer.py``
9
- actually reads are stored; everything else ``HDF5ChessDataset`` returns is dropped.
10
-
11
- The legal-move mask is the whole design problem: ``(64, 73)`` bool is 4,672 bytes and
12
- would dominate the record 20:1. It is stored instead as a fixed-width list of at most
13
- ``LEGAL_CAP`` flat ``from_square * 73 + plane`` indices (129 bytes), which is what makes
14
- one memmap row per sample — and therefore trivial rank/worker slicing — possible. The
15
- mask is rebuilt on GPU by :func:`~chesstransformer.datasets.flat_shard_dataset.scatter_legal_planes`.
16
-
17
- Fixed-width was chosen over CSR (~72 B/sample, no cap) deliberately: CSR is ~28 % smaller
18
- and is what Megatron's ``IndexedDataset`` does, but it needs a custom ragged collate, and
19
- the 1.4 GB it saves is on a dataset that already fits in page cache. The cap is not a
20
- correctness risk because the *target* move's index is always written first (see
21
- ``LEGAL_CAP`` below), so clipping can only ever drop alternative legal moves, never the
22
- label. Observed legal-move counts over 400 real positions: mean 33.9, p99.9 = 53, max 53.
23
- """
24
-
25
- import numpy as np
26
-
27
- NUM_ACTION_PLANES = 73
28
- NUM_SQUARES = 64
29
-
30
- #: Number of legal-move indices stored per sample. Sits well above the observed max of
31
- #: 53. The builder counts overflows and prints the rate; if it ever climbs above ~0.01 %
32
- #: raise this to 96 and rebuild.
33
- LEGAL_CAP = 64
34
-
35
- #: Flat action index space is ``[0, 4672)``. Unused slots in ``legal_idx`` are filled with
36
- #: ``PAD_ACTION`` (== 4672), one past the end, so a GPU ``scatter_`` can write padding into
37
- #: a scratch column that is then sliced off. Writing padding as 0 instead would silently
38
- #: mark square a1 / plane 0 legal on every single sample.
39
- PAD_ACTION = NUM_SQUARES * NUM_ACTION_PLANES
40
-
41
- #: Packed record layout. Offsets are explicit rather than auto-computed so that both
42
- #: ``uint16`` fields land on even offsets — an unaligned memmap view of ``legal_idx``
43
- #: would force numpy into a slow per-element copy on every batch gather.
44
- #:
45
- #: ========== ====== =========================================================
46
- #: offset dtype field
47
- #: ========== ====== =========================================================
48
- #: 0-63 u1×64 position board tokens, vocab 0-12
49
- #: 64-191 u2×64 legal_idx flat legal actions, target first, PAD-filled
50
- #: 192 u1 legal_cnt number of valid entries in legal_idx
51
- #: 193 u1 from_square target move source square, 0-63
52
- #: 194 u1 action_plane target move AlphaZero plane, 0-72
53
- #: 195 u1 castling_rights 4-bit mask, 0-15
54
- #: 196 u1 en_passant_file 0-7 = file a-h, 8 = none
55
- #: 197 u1 is_white side to move
56
- #: 198 i1 result 0 = draw, 1 = white win, 2 = black win (as in the HDF5)
57
- #: 199 u1 (pad) keeps move_number 2-byte aligned
58
- #: 200-201 u2 move_number ply index, max observed 601
59
- #: ========== ====== =========================================================
60
- RECORD_DTYPE = np.dtype(
61
- {
62
- "names": [
63
- "position",
64
- "legal_idx",
65
- "legal_cnt",
66
- "from_square",
67
- "action_plane",
68
- "castling_rights",
69
- "en_passant_file",
70
- "is_white",
71
- "result",
72
- "move_number",
73
- ],
74
- "formats": [
75
- (np.uint8, NUM_SQUARES),
76
- (np.uint16, LEGAL_CAP),
77
- np.uint8,
78
- np.uint8,
79
- np.uint8,
80
- np.uint8,
81
- np.uint8,
82
- np.uint8,
83
- np.int8,
84
- np.uint16,
85
- ],
86
- "offsets": [0, 64, 192, 193, 194, 195, 196, 197, 198, 200],
87
- "itemsize": 202,
88
- }
89
- )
90
-
91
- RECORD_BYTES = RECORD_DTYPE.itemsize
92
-
93
- #: Bumped whenever the layout above changes in a way that makes old shards unreadable.
94
- #: ``FlatShardDataset`` refuses to load a directory built by a different version.
95
- FORMAT_VERSION = 1
96
-
97
-
98
- def dtype_to_meta() -> dict:
99
- """Serialise :data:`RECORD_DTYPE` into ``meta.json``.
100
-
101
- Stored so a shard directory is self-describing: a reader can reconstruct the exact
102
- layout it was written with instead of trusting that this module never changed.
103
- """
104
- return {
105
- "names": list(RECORD_DTYPE.names),
106
- "formats": [
107
- (
108
- [RECORD_DTYPE[n].subdtype[0].str, RECORD_DTYPE[n].subdtype[1][0]]
109
- if RECORD_DTYPE[n].subdtype
110
- else RECORD_DTYPE[n].str
111
- )
112
- for n in RECORD_DTYPE.names
113
- ],
114
- "offsets": [RECORD_DTYPE.fields[n][1] for n in RECORD_DTYPE.names],
115
- "itemsize": RECORD_BYTES,
116
- }
117
-
118
-
119
- def dtype_from_meta(meta: dict) -> np.dtype:
120
- """Rebuild a record dtype from what ``meta.json`` recorded."""
121
- formats = [tuple(f) if isinstance(f, list) else f for f in meta["formats"]]
122
- return np.dtype(
123
- {
124
- "names": list(meta["names"]),
125
- "formats": formats,
126
- "offsets": list(meta["offsets"]),
127
- "itemsize": int(meta["itemsize"]),
128
- }
129
- )
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
chesstransformer/distributed_muon.py DELETED
@@ -1,222 +0,0 @@
1
- """Muon with Newton-Schulz distributed across ranks, plus its schedules.
2
-
3
- Under plain DDP every rank all-reduces to the *same* gradient and then every rank runs the
4
- *same* Newton-Schulz iteration on it. That is pure redundant compute: at world size N,
5
- N-1 copies of the NS work are thrown away, and every rank stores a full momentum buffer.
6
-
7
- The fix is to partition the work — but the partition axis is the whole point. NS is
8
- ``X @ X.T`` plus a global spectral-norm division, so it needs the *complete* matrix.
9
- Measured on a 64x32 weight, orthogonalizing per-shard and concatenating gives a **52-58 %
10
- wrong update** at world size 2-4, and nothing raises: a row-slice of a 2D matrix is still a
11
- 2D matrix. That is the ZeRO-2 trap, and it is why sharding *inside* a tensor is off limits
12
- here even though it is perfectly safe for elementwise optimizers like AdamW.
13
-
14
- So this partitions by **whole parameter**, following nanochat's ``MuonAdamW``
15
- (https://github.com/karpathy/nanochat, ``nanochat/optim.py``): params of a given shape are
16
- stacked, the K of them are divided into contiguous chunks across ranks, each rank runs NS
17
- only on the complete matrices it owns, and one ``all_gather`` per shape bucket republishes
18
- the results. Momentum state exists only for owned params, so it is sharded too.
19
-
20
- **Communication note, stated honestly.** nanochat does not use DDP: it ``reduce_scatter``s
21
- gradients inside the optimizer, so grad-sync and state-sharding share one collective and
22
- the total wire cost equals what DDP's all-reduce would have been. This class runs *on top
23
- of* DDP, which has already all-reduced the gradients, so the ``all_gather`` here is
24
- **additional** traffic — bytes DDP did not previously move. It buys NS compute (÷N) and
25
- momentum memory (÷N) at the cost of one param-sized gather per step. On a fast interconnect
26
- that is a clear win; on PCIe it may not be. ``scripts/bench_distributed_muon.py`` measures
27
- it rather than assuming.
28
-
29
- **Measured so far (gloo/CPU, preset 46m, 121 2D tensors, 16-core box):**
30
-
31
- =========== =========== ==================
32
- world size step time momentum state
33
- =========== =========== ==================
34
- 1 243 ms 177.0 MiB/rank
35
- 2 638 ms 89.0 MiB/rank
36
- 4 491 ms 45.0 MiB/rank
37
- =========== =========== ==================
38
-
39
- The memory result is the designed one and is exact: state divides by world size. **The step
40
- time is currently a regression**, and the honest reading is that this configuration has not
41
- been shown to be a compute win. Three things confound the CPU numbers -- gloo is far slower
42
- relative to compute than NCCL, N ranks on one 16-core box contend for the same cores so the
43
- "NS compute / N" never materialises, and the benchmark is pure optimizer with no forward or
44
- backward, so the all_gather is 100 % of the measured time instead of a few percent of a real
45
- step. It still should not be reported as a win until multi-GPU says so.
46
-
47
- The structural fix is the fused variant, which is why nanochat does not use DDP at all: it
48
- ``reduce_scatter``s gradients inside the optimizer, so grad-sync and state-sharding share one
49
- collective and the total wire cost equals DDP's all-reduce -- the ``all_gather`` stops being
50
- *extra*. Doing that here means excluding the Muon params from DDP's reducer
51
- (``DistributedDataParallel._set_params_and_buffers_to_ignore_for_model``) and doing the
52
- reduce-scatter in ``step()``. That is the next step, and it is what turns this from "memory
53
- win, compute unproven" into a pure win.
54
- """
55
-
56
- import math
57
-
58
- import torch
59
- import torch.distributed as dist
60
-
61
- # Reuse torch's own kernels rather than reimplementing the algorithm: it makes this
62
- # optimizer numerically identical to torch.optim.Muon by construction, which is exactly
63
- # what tests/test_distributed_muon.py asserts. Private, so guarded.
64
- try:
65
- from torch.optim._muon import _adjust_lr, _zeropower_via_newtonschulz
66
- except ImportError as exc: # pragma: no cover
67
- raise ImportError(
68
- "DistributedMuon reuses torch.optim._muon internals; needs a torch with "
69
- "torch.optim.Muon (2.9+)."
70
- ) from exc
71
-
72
- #: Params smaller than this stay replicated: every rank computes them redundantly and no
73
- #: collective is issued. For Pos2MoveV2-large that is 9 of 13 shape buckets holding 0.2M of
74
- #: 156.4M params — distributing them would cost 9 extra collectives to save nothing.
75
- DEFAULT_MIN_DISTRIBUTED_NUMEL = 65_536
76
-
77
-
78
- def muon_momentum(step: int, total_steps: int, warmup: int = 400,
79
- warmdown_ratio: float = 0.2, lo: float = 0.85,
80
- hi: float = 0.97, end: float = 0.90) -> float:
81
- """Momentum schedule: warm up 0.85 -> 0.97, then down to 0.90 during LR warmdown.
82
-
83
- Muon's momentum controls how much history the orthogonalized direction carries. Early
84
- on the gradient is changing fast and a long memory averages over a moving target, so it
85
- starts low; late in training a slightly *shorter* memory helps the run settle, which is
86
- why it comes back down rather than staying pinned at the peak. Schedule follows
87
- nanochat's ``get_muon_momentum``.
88
- """
89
- if step < warmup:
90
- frac = step / max(1, warmup)
91
- return (1 - frac) * lo + frac * hi
92
- warmdown_iters = max(1, round(warmdown_ratio * total_steps))
93
- warmdown_start = total_steps - warmdown_iters
94
- if step >= warmdown_start:
95
- progress = min(1.0, (step - warmdown_start) / warmdown_iters)
96
- return hi * (1 - progress) + end * progress
97
- return hi
98
-
99
-
100
- def muon_weight_decay(step: int, total_steps: int, base_wd: float) -> float:
101
- """Cosine-decay weight decay to zero over the run, as nanochat's ``get_weight_decay``."""
102
- return base_wd * 0.5 * (1 + math.cos(math.pi * min(1.0, step / max(1, total_steps))))
103
-
104
-
105
- class DistributedMuon(torch.optim.Optimizer):
106
- """Drop-in for ``torch.optim.Muon`` that splits NS work and momentum across ranks.
107
-
108
- Assumes gradients are already synchronized (DDP has all-reduced them), so every rank
109
- starts from an identical gradient and only the *work* is divided. With no process group
110
- initialized it degenerates to exactly ``torch.optim.Muon``.
111
- """
112
-
113
- def __init__(self, params, lr=1e-3, weight_decay=0.1, momentum=0.95, nesterov=True,
114
- ns_coefficients=(3.4445, -4.7750, 2.0315), eps=1e-7, ns_steps=5,
115
- adjust_lr_fn=None, min_distributed_numel=DEFAULT_MIN_DISTRIBUTED_NUMEL):
116
- defaults = dict(lr=lr, weight_decay=weight_decay, momentum=momentum,
117
- nesterov=nesterov, ns_coefficients=ns_coefficients, eps=eps,
118
- ns_steps=ns_steps, adjust_lr_fn=adjust_lr_fn)
119
- super().__init__(params, defaults)
120
-
121
- self.world_size = dist.get_world_size() if dist.is_initialized() else 1
122
- self.rank = dist.get_rank() if dist.is_initialized() else 0
123
- self.min_distributed_numel = min_distributed_numel
124
- self._plan = [self._plan_group(g) for g in self.param_groups]
125
-
126
- owned = sum(len(b["owned"]) for p in self._plan for b in p["buckets"])
127
- total = sum(len(g["params"]) for g in self.param_groups)
128
- self.owned_param_count, self.total_param_count = owned, total
129
-
130
- def _plan_group(self, group):
131
- """Bucket a group's params by shape and assign contiguous chunks to ranks.
132
-
133
- Contiguous rather than strided so that ``all_gather_into_tensor`` — which lays rank
134
- i's contribution at output[i*chunk:(i+1)*chunk] — reassembles the bucket in order
135
- with no extra permutation.
136
- """
137
- by_shape: dict[tuple, list] = {}
138
- for p in group["params"]:
139
- if p.ndim != 2:
140
- raise ValueError(f"DistributedMuon handles 2D params only, got {p.ndim}D")
141
- by_shape.setdefault(tuple(p.shape), []).append(p)
142
-
143
- buckets = []
144
- for shape, plist in sorted(by_shape.items(), key=lambda kv: -kv[1][0].numel() * len(kv[1])):
145
- k = len(plist)
146
- distributed = (self.world_size > 1
147
- and plist[0].numel() >= self.min_distributed_numel)
148
- if not distributed:
149
- # Replicated: every rank updates every param, no collective.
150
- buckets.append(dict(shape=shape, params=plist, owned=plist,
151
- offset=0, chunk=k, distributed=False))
152
- continue
153
- chunk = math.ceil(k / self.world_size)
154
- start = self.rank * chunk
155
- owned = plist[start:start + chunk] # may be short on the last rank
156
- buckets.append(dict(shape=shape, params=plist, owned=owned,
157
- offset=start, chunk=chunk, distributed=True))
158
- return dict(buckets=buckets)
159
-
160
- @torch.no_grad()
161
- def step(self, closure=None):
162
- loss = None
163
- if closure is not None:
164
- with torch.enable_grad():
165
- loss = closure()
166
-
167
- for group, plan in zip(self.param_groups, self._plan):
168
- lr, wd = group["lr"], group["weight_decay"]
169
- momentum, nesterov = group["momentum"], group["nesterov"]
170
- for bucket in plan["buckets"]:
171
- for p in bucket["owned"]:
172
- if p.grad is None:
173
- continue
174
- self._update_one(p, lr, wd, momentum, nesterov, group)
175
- for bucket in plan["buckets"]:
176
- if bucket["distributed"]:
177
- self._publish(bucket)
178
- return loss
179
-
180
- def _update_one(self, p, lr, wd, momentum, nesterov, group):
181
- """Exactly torch.optim.Muon's ``_single_tensor_muon`` body, for one param."""
182
- grad = p.grad
183
- if grad.ndim != 2:
184
- raise ValueError("Param gradient must be a 2D matrix")
185
- state = self.state[p]
186
- if "momentum_buffer" not in state:
187
- state["momentum_buffer"] = torch.zeros_like(p)
188
- buf = state["momentum_buffer"]
189
- buf.lerp_(grad, 1 - momentum)
190
- update = grad.lerp(buf, momentum) if nesterov else buf
191
- update = _zeropower_via_newtonschulz(
192
- update, group["ns_coefficients"], group["ns_steps"], group["eps"])
193
- adjusted_lr = _adjust_lr(lr, group["adjust_lr_fn"], p.shape)
194
- p.mul_(1 - lr * wd)
195
- p.add_(update, alpha=-adjusted_lr)
196
-
197
- def _publish(self, bucket):
198
- """One all_gather per shape bucket to give every rank the updated params.
199
-
200
- Zero-padding when K does not divide world_size: the tail slots are gathered and
201
- discarded, which costs a little bandwidth and keeps the collective's shape uniform.
202
- """
203
- plist, chunk, shape = bucket["params"], bucket["chunk"], bucket["shape"]
204
- k, ws = len(plist), self.world_size
205
- ref = plist[0]
206
- local = torch.zeros(chunk, *shape, dtype=ref.dtype, device=ref.device)
207
- for i, p in enumerate(bucket["owned"]):
208
- local[i].copy_(p)
209
- out = torch.empty(chunk * ws, *shape, dtype=ref.dtype, device=ref.device)
210
- dist.all_gather_into_tensor(out, local)
211
- for i in range(k):
212
- plist[i].copy_(out[i])
213
-
214
- def sharding_summary(self) -> str:
215
- parts = []
216
- for plan in self._plan:
217
- for b in plan["buckets"]:
218
- tag = "dist" if b["distributed"] else "repl"
219
- parts.append(f"{tuple(b['shape'])}x{len(b['params'])}:{tag}")
220
- return (f"DistributedMuon rank {self.rank}/{self.world_size}: owns "
221
- f"{self.owned_param_count}/{self.total_param_count} params | "
222
- + " ".join(parts))
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
chesstransformer/introspection/__init__.py DELETED
@@ -1,5 +0,0 @@
1
- """Model introspection: attention capture, logit lens, occlusion saliency."""
2
-
3
- from chesstransformer.introspection.analyze import PositionAnalyzer, load_model
4
-
5
- __all__ = ["PositionAnalyzer", "load_model"]
 
 
 
 
 
 
chesstransformer/introspection/analyze.py DELETED
@@ -1,450 +0,0 @@
1
- """Per-position introspection of a Pos2MoveV2 network.
2
-
3
- Everything here is read-only with respect to the model: attention maps are
4
- recomputed from the captured layer inputs using the module's own projections,
5
- so the numbers match what SDPA saw during the real forward pass.
6
-
7
- Produced per position:
8
- * policy over legal moves (probability, raw logit, action plane)
9
- * value head output, plus the value/policy read off *every* layer
10
- (logit lens) so you can see at which depth the model makes up its mind
11
- * full attention tensor (L, H, 67, 67)
12
- * attention rollout (Abnar & Zuidema 2020) over the layer stack
13
- * occlusion saliency: remove one piece, measure the swing in value and in
14
- the probability of the move the model wanted to play
15
- * 1-ply child values, i.e. what the value head says about the position
16
- *after* each legal move — the fastest way to spot policy/value disagreement
17
- """
18
-
19
- from __future__ import annotations
20
-
21
- import json
22
- import math
23
- import zlib
24
- from base64 import b64encode
25
- from dataclasses import dataclass
26
- from pathlib import Path
27
-
28
- import chess
29
- import numpy as np
30
- import torch
31
- from safetensors import safe_open
32
-
33
- from chesstransformer.models.tokenizer import PostionTokenizer
34
- from chesstransformer.models.tokenizer.alphazero_move_encoder import (
35
- NUM_ACTION_PLANES,
36
- move_to_action_plane,
37
- )
38
- from chesstransformer.models.transformer.pos2move_v2 import Pos2MoveV2
39
-
40
- BOARD_SQUARES = 64
41
- CONTEXT_SIZE = 67
42
- STATE_TOKEN_NAMES = ["castling", "en_passant", "player"]
43
-
44
-
45
- def load_model(model_dir: str | Path, device: str = "cpu", use_ema: bool = False):
46
- """Load a Pos2MoveV2 checkpoint in fp32 (no compile, no bf16).
47
-
48
- Introspection wants clean numbers more than it wants speed, and the model
49
- is small enough that fp32 eager is instant either way.
50
- """
51
- model_dir = Path(model_dir)
52
-
53
- config_path = model_dir / "model_config.json"
54
- if not config_path.exists():
55
- config_path = model_dir.parent.parent / "model_config.json"
56
- if not config_path.exists():
57
- raise FileNotFoundError(f"No model_config.json found for {model_dir}")
58
- with open(config_path) as f:
59
- config = json.load(f)
60
-
61
- model = Pos2MoveV2(**config)
62
-
63
- weights = model_dir / "model.safetensors"
64
- if not weights.exists():
65
- raise FileNotFoundError(f"No model.safetensors in {model_dir}")
66
- with safe_open(str(weights), framework="pt", device="cpu") as f:
67
- state_dict = {k: f.get_tensor(k) for k in f.keys()}
68
- state_dict = {k.replace("_orig_mod.", ""): v for k, v in state_dict.items()}
69
- model.load_state_dict(state_dict)
70
-
71
- if use_ema:
72
- ema_path = model_dir / "ema_state.pt"
73
- if not ema_path.exists():
74
- raise FileNotFoundError(f"No ema_state.pt in {model_dir}")
75
- ema_state = torch.load(str(ema_path), map_location="cpu", weights_only=True)
76
- ema_state = {k.replace("_orig_mod.", ""): v for k, v in ema_state.items()}
77
- model.load_state_dict(ema_state, strict=False)
78
-
79
- model.eval()
80
- return model.float().to(device), config
81
-
82
-
83
- def encode_board(board: chess.Board, tokenizer: PostionTokenizer):
84
- """Same encoding the bots use: 64 piece tokens + castling/ep/player."""
85
- tokens = tokenizer.encode(board)
86
- castling = 0
87
- if board.has_kingside_castling_rights(chess.WHITE):
88
- castling |= 1
89
- if board.has_queenside_castling_rights(chess.WHITE):
90
- castling |= 2
91
- if board.has_kingside_castling_rights(chess.BLACK):
92
- castling |= 4
93
- if board.has_queenside_castling_rights(chess.BLACK):
94
- castling |= 8
95
- ep_file = chess.square_file(board.ep_square) if board.has_legal_en_passant() else 8
96
- return tokens, int(board.turn), castling, ep_file
97
-
98
-
99
- @dataclass
100
- class _Capture:
101
- """Residual stream and attention input for one layer."""
102
-
103
- attn_input: torch.Tensor # normed input handed to the attention module
104
- output: torch.Tensor # residual stream leaving the layer
105
-
106
-
107
- class PositionAnalyzer:
108
- def __init__(
109
- self,
110
- model: Pos2MoveV2,
111
- device: str = "cpu",
112
- max_moves: int = 64,
113
- saliency: bool = True,
114
- ):
115
- self.model = model
116
- self.device = device
117
- self.tokenizer = PostionTokenizer()
118
- self.max_moves = max_moves
119
- self.saliency = saliency
120
- self.n_layers = len(model.transformer_layers)
121
- self.n_heads = model.transformer_layers[0].attn.num_heads
122
-
123
- # ── raw forward helpers ────────────────────────────────────────────
124
- @torch.no_grad()
125
- def _forward_batch(self, boards: list[chess.Board]):
126
- enc = [encode_board(b, self.tokenizer) for b in boards]
127
- tokens = torch.tensor([e[0] for e in enc], dtype=torch.long, device=self.device)
128
- player = torch.tensor([e[1] for e in enc], dtype=torch.long, device=self.device)
129
- castling = torch.tensor([e[2] for e in enc], dtype=torch.long, device=self.device)
130
- ep = torch.tensor([e[3] for e in enc], dtype=torch.long, device=self.device)
131
- return self.model(tokens, player, castling, ep)
132
-
133
- @torch.no_grad()
134
- def _forward_tokens(self, tokens, player, castling, ep):
135
- return self.model(tokens, player, castling, ep)
136
-
137
- @torch.no_grad()
138
- def _forward_with_capture(self, board: chess.Board):
139
- """One forward pass, keeping every layer's input and output."""
140
- captures: list[_Capture] = []
141
- pending: dict[int, torch.Tensor] = {}
142
- handles = []
143
-
144
- def make_pre_hook(idx):
145
- def hook(_module, args):
146
- pending[idx] = args[0].detach()
147
-
148
- return hook
149
-
150
- def make_post_hook(idx):
151
- def hook(_module, _args, output):
152
- captures.append(_Capture(attn_input=pending[idx], output=output.detach()))
153
-
154
- return hook
155
-
156
- for i, layer in enumerate(self.model.transformer_layers):
157
- handles.append(layer.attn.register_forward_pre_hook(make_pre_hook(i)))
158
- handles.append(layer.register_forward_hook(make_post_hook(i)))
159
-
160
- try:
161
- tokens, player, castling, ep = encode_board(board, self.tokenizer)
162
- out = self._forward_tokens(
163
- torch.tensor([tokens], dtype=torch.long, device=self.device),
164
- torch.tensor([player], dtype=torch.long, device=self.device),
165
- torch.tensor([castling], dtype=torch.long, device=self.device),
166
- torch.tensor([ep], dtype=torch.long, device=self.device),
167
- )
168
- finally:
169
- for h in handles:
170
- h.remove()
171
-
172
- return out, captures
173
-
174
- @torch.no_grad()
175
- def _attention_probs(self, layer_idx: int, attn_input: torch.Tensor) -> torch.Tensor:
176
- """Recompute softmax attention for one layer: (H, 67, 67).
177
-
178
- Mirrors ChessGroupedQueryAttention.forward up to the softmax — SDPA
179
- adds the chess-relation bias to the *scaled* scores, so we do too.
180
- """
181
- attn = self.model.transformer_layers[layer_idx].attn
182
- x = attn_input
183
- b, t, _ = x.shape
184
-
185
- q = attn.q_proj(x).view(b, t, attn.num_heads, attn.head_dim).transpose(1, 2)
186
- k = attn.k_proj(x).view(b, t, attn.num_kv_groups, attn.head_dim).transpose(1, 2)
187
- q = attn.q_norm(q)
188
- k = attn.k_norm(k)
189
- k = k.repeat_interleave(attn.group_size, dim=1)
190
-
191
- scores = (q @ k.transpose(-2, -1)) / math.sqrt(attn.head_dim)
192
- scores = scores + attn.bias_table[:, attn.rel_idx].unsqueeze(0)
193
- return torch.softmax(scores, dim=-1)[0]
194
-
195
- # ── analysis ───────────────────────────────────────────────────────
196
- def analyze(self, board: chess.Board, include_attention: bool = True) -> dict:
197
- (move_logits, value), captures = self._forward_with_capture(board)
198
- logits = move_logits[0].float() # (64, 73)
199
- value = float(value[0].item())
200
-
201
- result: dict = {
202
- "fen": board.fen(),
203
- "turn": "w" if board.turn else "b",
204
- "fullmove": board.fullmove_number,
205
- "check": board.is_check(),
206
- "pieces": [
207
- (board.piece_at(sq).symbol() if board.piece_at(sq) else None)
208
- for sq in range(64)
209
- ],
210
- "value": value,
211
- }
212
-
213
- legal = list(board.legal_moves)
214
- result["legal_count"] = len(legal)
215
- if not legal:
216
- result["policy"] = {"moves": [], "entropy": 0.0, "top1_prob": 0.0}
217
- result["terminal"] = board.result()
218
- return result
219
-
220
- result.update(self._policy_block(board, logits, legal))
221
- result.update(self._depth_block(captures, logits, board, legal))
222
-
223
- if include_attention:
224
- result["attention"] = self._attention_block(captures)
225
-
226
- if self.saliency:
227
- result["saliency"] = self._saliency_block(board, value, result)
228
-
229
- return result
230
-
231
- def _policy_block(self, board: chess.Board, logits: torch.Tensor, legal) -> dict:
232
- planes = [
233
- move_to_action_plane(m.from_square, m.to_square, m.promotion) for m in legal
234
- ]
235
- idx = torch.tensor(
236
- [m.from_square * NUM_ACTION_PLANES + p for m, p in zip(legal, planes)],
237
- dtype=torch.long,
238
- )
239
- flat = logits.reshape(-1).cpu()
240
- legal_logits = flat[idx]
241
- probs = torch.softmax(legal_logits, dim=0)
242
-
243
- # Illegal mass: how much of the raw 64x73 softmax the model spends on
244
- # moves that do not exist. A cheap sanity signal for a broken head.
245
- full_probs = torch.softmax(flat, dim=0)
246
- legal_mass = float(full_probs[idx].sum())
247
-
248
- order = torch.argsort(probs, descending=True)
249
- moves = []
250
- for rank, i in enumerate(order.tolist()):
251
- m = legal[i]
252
- moves.append(
253
- {
254
- "uci": m.uci(),
255
- "san": board.san(m),
256
- "from": m.from_square,
257
- "to": m.to_square,
258
- "plane": planes[i],
259
- "prob": float(probs[i]),
260
- "logit": float(legal_logits[i]),
261
- "rank": rank,
262
- "capture": board.is_capture(m),
263
- "check": board.gives_check(m),
264
- }
265
- )
266
-
267
- p = probs[probs > 0]
268
- entropy = float(-(p * p.log()).sum())
269
- return {
270
- "policy": {
271
- "moves": moves[: self.max_moves],
272
- "n_moves": len(moves),
273
- "entropy": entropy,
274
- "max_entropy": math.log(len(legal)),
275
- "top1_prob": moves[0]["prob"],
276
- "legal_mass": legal_mass,
277
- }
278
- }
279
-
280
- @torch.no_grad()
281
- def _depth_block(self, captures, final_logits, board, legal) -> dict:
282
- """Logit lens: decode value + policy from every layer's residual."""
283
- norm = self.model.final_norm
284
- value_by_layer = []
285
- top_by_layer = []
286
- write_norms = []
287
-
288
- planes = [
289
- move_to_action_plane(m.from_square, m.to_square, m.promotion) for m in legal
290
- ]
291
- idx = torch.tensor(
292
- [m.from_square * NUM_ACTION_PLANES + p for m, p in zip(legal, planes)],
293
- dtype=torch.long,
294
- device=self.device,
295
- )
296
- final_probs = torch.softmax(final_logits.reshape(-1)[idx.cpu()], dim=0)
297
- final_top = int(torch.argmax(final_probs))
298
-
299
- prev = None
300
- for cap in captures:
301
- x = norm(cap.output)
302
- state = x[:, -3:, :].mean(dim=1)
303
- value_by_layer.append(float(self.model.value_head(state)[0].item()))
304
-
305
- lens_logits = self.model.move_head(x[:, :64, :])[0].reshape(-1)[idx]
306
- lens_probs = torch.softmax(lens_logits.float(), dim=0)
307
- best = int(torch.argmax(lens_probs))
308
- top_by_layer.append(
309
- {
310
- "uci": legal[best].uci(),
311
- "san": board.san(legal[best]),
312
- "prob": float(lens_probs[best]),
313
- "final_move_prob": float(lens_probs[final_top]),
314
- "agrees": best == final_top,
315
- }
316
- )
317
-
318
- out = cap.output
319
- write_norms.append(
320
- float((out - prev).norm().item()) if prev is not None else float(out.norm().item())
321
- )
322
- prev = out
323
-
324
- return {
325
- "depth": {
326
- "value_by_layer": value_by_layer,
327
- "top_by_layer": top_by_layer,
328
- "residual_write_norm": write_norms,
329
- }
330
- }
331
-
332
- @torch.no_grad()
333
- def _attention_block(self, captures) -> dict:
334
- """Full (L, H, 67, 67) attention, u8-quantised per row, deflated."""
335
- maps = []
336
- for i, cap in enumerate(captures):
337
- maps.append(self._attention_probs(i, cap.attn_input).cpu())
338
- attn = torch.stack(maps) # (L, H, 67, 67)
339
-
340
- rollout = self._rollout(attn)
341
-
342
- # u8 per row, companded through a square root: attention rows are long
343
- # tails of small weights, and linear u8 would quantise those to 20-30%
344
- # relative error. sqrt spreads the error evenly across magnitudes.
345
- row_max = attn.amax(dim=-1, keepdim=True).clamp(min=1e-9)
346
- quant = ((attn / row_max).sqrt() * 255).round().clamp(0, 255).to(torch.uint8).numpy()
347
- payload = zlib.compress(quant.tobytes(), 6)
348
-
349
- return {
350
- "shape": list(attn.shape),
351
- "scale": row_max.squeeze(-1).flatten().numpy().astype(np.float32).tolist(),
352
- "data": b64encode(payload).decode("ascii"),
353
- "rollout": [round(v, 5) for v in rollout.tolist()],
354
- }
355
-
356
- @staticmethod
357
- def _rollout(attn: torch.Tensor) -> torch.Tensor:
358
- """Attention rollout: how much of each token the output actually sees.
359
-
360
- Head-averaged, residual-corrected (0.5*A + 0.5*I), multiplied through
361
- the stack, then read from the three game-state token rows — those are
362
- what the value head consumes.
363
- """
364
- a = attn.mean(dim=1) # (L, T, T)
365
- eye = torch.eye(a.shape[-1])
366
- joint = None
367
- for layer in a:
368
- m = 0.5 * layer + 0.5 * eye
369
- m = m / m.sum(dim=-1, keepdim=True)
370
- joint = m if joint is None else m @ joint
371
- return joint[-3:, :].mean(dim=0)
372
-
373
- @torch.no_grad()
374
- def _saliency_block(self, board: chess.Board, base_value: float, result: dict) -> dict:
375
- """Occlusion: delete one piece at a time, measure what moves.
376
-
377
- Kings are left in place (an occluded king is not a chess position the
378
- model has ever seen). Everything runs as one batch.
379
- """
380
- top = result["policy"]["moves"][0]
381
- top_idx = top["from"] * NUM_ACTION_PLANES + top["plane"]
382
-
383
- squares = [
384
- sq
385
- for sq in range(64)
386
- if board.piece_at(sq) is not None and board.piece_at(sq).piece_type != chess.KING
387
- ]
388
- value_delta = [0.0] * 64
389
- policy_delta = [0.0] * 64
390
- if not squares:
391
- return {"value_delta": value_delta, "policy_delta": policy_delta, "squares": []}
392
-
393
- tokens, player, castling, ep = encode_board(board, self.tokenizer)
394
- base = torch.tensor(tokens, dtype=torch.long).repeat(len(squares), 1)
395
- for row, sq in enumerate(squares):
396
- base[row, sq] = 0
397
- n = len(squares)
398
- move_logits, values = self._forward_tokens(
399
- base.to(self.device),
400
- torch.full((n,), player, dtype=torch.long, device=self.device),
401
- torch.full((n,), castling, dtype=torch.long, device=self.device),
402
- torch.full((n,), ep, dtype=torch.long, device=self.device),
403
- )
404
-
405
- # Probability of the top move under the *full* 64x73 softmax: occlusion
406
- # changes the legal move set, so a legal-masked comparison would not be
407
- # measuring the same quantity across rows.
408
- flat = move_logits.float().reshape(n, -1).cpu()
409
- probs = torch.softmax(flat, dim=1)[:, top_idx]
410
-
411
- ref_logits, _ = self._forward_batch([board])
412
- ref_prob = float(torch.softmax(ref_logits[0].float().reshape(-1), dim=0)[top_idx])
413
-
414
- for row, sq in enumerate(squares):
415
- value_delta[sq] = float(values[row].item()) - base_value
416
- policy_delta[sq] = float(probs[row]) - ref_prob
417
-
418
- return {
419
- "value_delta": [round(v, 5) for v in value_delta],
420
- "policy_delta": [round(v, 5) for v in policy_delta],
421
- "squares": squares,
422
- "top_move": top["uci"],
423
- "ref_prob": ref_prob,
424
- }
425
-
426
- @torch.no_grad()
427
- def child_values(self, board: chess.Board, moves: list[dict]) -> None:
428
- """Fill in the value of the position *after* each candidate move.
429
-
430
- Stored from the mover's point of view (the raw head is side-to-move
431
- POV, so it gets negated) — directly comparable to ``value``.
432
- """
433
- if not moves:
434
- return
435
- boards = []
436
- terminal = {}
437
- for i, m in enumerate(moves):
438
- b = board.copy(stack=False)
439
- b.push(chess.Move.from_uci(m["uci"]))
440
- if b.is_checkmate():
441
- terminal[i] = 1.0
442
- elif b.is_stalemate() or b.is_insufficient_material():
443
- terminal[i] = 0.0
444
- boards.append(b)
445
-
446
- _, values = self._forward_batch(boards)
447
- for i, m in enumerate(moves):
448
- m["child_value"] = (
449
- terminal[i] if i in terminal else -float(values[i].item())
450
- )
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
chesstransformer/introspection/viewer.html DELETED
@@ -1,860 +0,0 @@
1
- <title>Pos2MoveV2 Glass Box</title>
2
- <meta name="viewport" content="width=device-width, initial-scale=1">
3
- <link rel="preconnect" href="https://fonts.googleapis.com">
4
- <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
5
- <link rel="stylesheet" href="https://fonts.googleapis.com/css2?family=Archivo:wdth,wght@62..125,400..700&family=JetBrains+Mono:wght@400..600&display=swap">
6
-
7
- <style>
8
- :root {
9
- --bg: #e9edec;
10
- --panel: #f8faf9;
11
- --panel-2: #f0f3f2;
12
- --ink: #101a1c;
13
- --ink-2: #47585b;
14
- --ink-3: #7c8b8d;
15
- --line: #d3dbd9;
16
- --line-strong: #b9c4c2;
17
- --accent: #8a6208;
18
- --pos: #0d6b66;
19
- --neg: #a4382a;
20
- --neutral: #93a0a1;
21
- --seq: #2b3a8f;
22
- --sq-light: #dad5c8;
23
- --sq-dark: #b0a996;
24
- --piece-w: #fbf8f1;
25
- --piece-b: #171c1e;
26
- --shadow: 0 1px 2px rgba(16,26,28,.06), 0 8px 24px rgba(16,26,28,.06);
27
- --radius: 10px;
28
- }
29
- @media (prefers-color-scheme: dark) {
30
- :root:not([data-theme="light"]) {
31
- --bg: #10171a;
32
- --panel: #172023;
33
- --panel-2: #1d282b;
34
- --ink: #e8efee;
35
- --ink-2: #a7b7b8;
36
- --ink-3: #75878a;
37
- --line: #27353a;
38
- --line-strong: #3a4b50;
39
- --accent: #d6a63c;
40
- --pos: #3fb3a6;
41
- --neg: #e0705c;
42
- --neutral: #6f8083;
43
- --seq: #8fa2ff;
44
- --sq-light: #414c4f;
45
- --sq-dark: #2d383b;
46
- --piece-w: #f2f5f2;
47
- --piece-b: #0b1012;
48
- --shadow: 0 1px 2px rgba(0,0,0,.4), 0 10px 30px rgba(0,0,0,.28);
49
- }
50
- }
51
- :root[data-theme="dark"] {
52
- --bg: #10171a;
53
- --panel: #172023;
54
- --panel-2: #1d282b;
55
- --ink: #e8efee;
56
- --ink-2: #a7b7b8;
57
- --ink-3: #75878a;
58
- --line: #27353a;
59
- --line-strong: #3a4b50;
60
- --accent: #d6a63c;
61
- --pos: #3fb3a6;
62
- --neg: #e0705c;
63
- --neutral: #6f8083;
64
- --seq: #8fa2ff;
65
- --sq-light: #414c4f;
66
- --sq-dark: #2d383b;
67
- --piece-w: #f2f5f2;
68
- --piece-b: #0b1012;
69
- --shadow: 0 1px 2px rgba(0,0,0,.4), 0 10px 30px rgba(0,0,0,.28);
70
- }
71
-
72
- * { box-sizing: border-box; }
73
-
74
- body {
75
- margin: 0;
76
- background: var(--bg);
77
- color: var(--ink);
78
- font-family: Archivo, "Helvetica Neue", Arial, sans-serif;
79
- font-size: 15px;
80
- line-height: 1.5;
81
- -webkit-font-smoothing: antialiased;
82
- }
83
-
84
- .wrap { max-width: 1400px; margin: 0 auto; padding: 28px 24px 64px; }
85
-
86
- /* ── header ─────────────────────────────────────────────── */
87
- header { display: flex; flex-wrap: wrap; align-items: flex-end; gap: 20px; justify-content: space-between; padding-bottom: 18px; border-bottom: 1px solid var(--line); }
88
- h1 {
89
- margin: 0;
90
- font-size: clamp(26px, 3.4vw, 40px);
91
- font-variation-settings: "wdth" 112, "wght" 700;
92
- letter-spacing: -.02em;
93
- line-height: 1.02;
94
- text-wrap: balance;
95
- }
96
- h1 .sub { display: block; font-size: 13px; font-variation-settings: "wdth" 88, "wght" 500; letter-spacing: .14em; text-transform: uppercase; color: var(--accent); margin-bottom: 8px; }
97
- .meta-grid { display: flex; flex-wrap: wrap; gap: 6px 22px; font-family: "JetBrains Mono", ui-monospace, monospace; font-size: 11.5px; color: var(--ink-3); }
98
- .meta-grid b { color: var(--ink-2); font-weight: 500; }
99
-
100
- /* ── generic panel ──────────────────────────────────────── */
101
- .panel { background: var(--panel); border: 1px solid var(--line); border-radius: var(--radius); box-shadow: var(--shadow); }
102
- .panel > h2, .head { display: flex; align-items: baseline; justify-content: space-between; gap: 12px; margin: 0; padding: 12px 16px; border-bottom: 1px solid var(--line); font-size: 11px; letter-spacing: .13em; text-transform: uppercase; font-variation-settings: "wdth" 88, "wght" 600; color: var(--ink-2); }
103
- .head .note { font-family: "JetBrains Mono", ui-monospace, monospace; font-size: 10.5px; letter-spacing: 0; text-transform: none; color: var(--ink-3); }
104
-
105
- /* ── position strip ─────────────────────────────────────── */
106
- .strip { display: flex; gap: 6px; overflow-x: auto; padding: 14px 0 4px; }
107
- .ply {
108
- flex: 0 0 auto; border: 1px solid var(--line); background: var(--panel); color: var(--ink-2);
109
- border-radius: 999px; padding: 5px 13px; cursor: pointer;
110
- font-family: "JetBrains Mono", ui-monospace, monospace; font-size: 12px;
111
- }
112
- .ply:hover { border-color: var(--line-strong); color: var(--ink); }
113
- .ply[aria-current="true"] { background: var(--accent); border-color: var(--accent); color: var(--bg); }
114
-
115
- /* ── main layout ────────────────────────────────────────── */
116
- .main { display: grid; grid-template-columns: minmax(360px, 520px) minmax(340px, 1fr); gap: 20px; margin-top: 20px; align-items: start; }
117
- @media (max-width: 980px) { .main { grid-template-columns: 1fr; } }
118
-
119
- /* ── board ──────────────────────────────────────────────── */
120
- .board-shell { padding: 16px; }
121
- @media (min-width: 981px) { .board-shell { position: sticky; top: 20px; } }
122
- .board-frame { position: relative; }
123
- .board { position: relative; display: grid; grid-template-columns: repeat(8, 1fr); aspect-ratio: 1; border-radius: 6px; overflow: hidden; border: 1px solid var(--line-strong); }
124
- .sq { position: relative; display: flex; align-items: center; justify-content: center; cursor: pointer; }
125
- .sq.light { background: var(--sq-light); }
126
- .sq.dark { background: var(--sq-dark); }
127
- .sq .heat { position: absolute; inset: 0; pointer-events: none; }
128
- .sq .glyph { position: relative; font-size: min(6.2vw, 40px); line-height: 1; user-select: none; pointer-events: none; font-family: "Noto Sans Symbols 2", "DejaVu Sans", "Segoe UI Symbol", serif; }
129
- .sq .glyph.w { color: var(--piece-w); -webkit-text-stroke: 1.1px var(--piece-b); }
130
- .sq .glyph.b { color: var(--piece-b); -webkit-text-stroke: 1.1px rgba(250,250,245,.55); }
131
- .sq .val { position: absolute; bottom: 2px; right: 3px; font-family: "JetBrains Mono", ui-monospace, monospace; font-size: 9px; color: var(--ink); opacity: .72; pointer-events: none; font-variant-numeric: tabular-nums; }
132
- .sq.sel { outline: 2px solid var(--accent); outline-offset: -2px; }
133
- .sq.lastmove::after { content: ""; position: absolute; inset: 0; border: 2px solid var(--accent); opacity: .45; pointer-events: none; }
134
- .sq .rank-lbl, .sq .file-lbl { position: absolute; font-family: "JetBrains Mono", ui-monospace, monospace; font-size: 9px; color: var(--ink-2); opacity: .6; pointer-events: none; }
135
- .sq .rank-lbl { top: 2px; left: 3px; }
136
- .sq .file-lbl { bottom: 1px; left: 3px; }
137
- #arrows { position: absolute; inset: 0; width: 100%; height: 100%; pointer-events: none; }
138
-
139
- .modes { display: flex; flex-wrap: wrap; gap: 6px; margin-top: 14px; }
140
- .mode {
141
- border: 1px solid var(--line); background: var(--panel-2); color: var(--ink-2);
142
- border-radius: 7px; padding: 6px 11px; cursor: pointer; font-size: 12.5px;
143
- font-variation-settings: "wdth" 92, "wght" 500;
144
- }
145
- .mode:hover { border-color: var(--line-strong); color: var(--ink); }
146
- .mode[aria-pressed="true"] { background: var(--accent); border-color: var(--accent); color: var(--bg); }
147
- .scalebar { display: flex; align-items: center; gap: 9px; margin-top: 12px; font-family: "JetBrains Mono", ui-monospace, monospace; font-size: 10.5px; color: var(--ink-3); }
148
- .ramp { flex: 1; height: 7px; border-radius: 4px; border: 1px solid var(--line); }
149
- .fen { margin-top: 12px; font-family: "JetBrains Mono", ui-monospace, monospace; font-size: 10.5px; color: var(--ink-3); word-break: break-all; }
150
- .hint { margin-top: 10px; font-size: 12px; color: var(--ink-3); }
151
-
152
- /* ── stat tiles ─────────────────────────────────────────── */
153
- .tiles { display: grid; grid-template-columns: repeat(auto-fit, minmax(168px, 1fr)); gap: 12px; }
154
- .tile { background: var(--panel); border: 1px solid var(--line); border-radius: var(--radius); padding: 13px 15px; box-shadow: var(--shadow); }
155
- .tile .lbl { font-size: 10px; letter-spacing: .13em; text-transform: uppercase; color: var(--ink-3); font-variation-settings: "wdth" 88, "wght" 600; }
156
- .tile .big { font-size: 26px; font-variation-settings: "wdth" 105, "wght" 700; letter-spacing: -.02em; font-variant-numeric: tabular-nums; margin-top: 2px; }
157
- .tile .small { font-family: "JetBrains Mono", ui-monospace, monospace; font-size: 11px; color: var(--ink-3); font-variant-numeric: tabular-nums; }
158
- .meter { height: 6px; border-radius: 3px; background: var(--panel-2); border: 1px solid var(--line); margin-top: 8px; position: relative; overflow: hidden; }
159
- .meter i { position: absolute; top: 0; bottom: 0; display: block; }
160
- .meter .mid { position: absolute; top: -1px; bottom: -1px; left: 50%; width: 1px; background: var(--line-strong); }
161
-
162
- /* ── move table ─────────────────────────────────────────── */
163
- .tablewrap { overflow-x: auto; }
164
- table { width: 100%; border-collapse: collapse; font-size: 13px; }
165
- th { text-align: left; font-size: 10px; letter-spacing: .11em; text-transform: uppercase; color: var(--ink-3); font-variation-settings: "wdth" 88, "wght" 600; padding: 9px 12px; border-bottom: 1px solid var(--line); white-space: nowrap; }
166
- td { padding: 6px 12px; border-bottom: 1px solid var(--line); vertical-align: middle; font-variant-numeric: tabular-nums; }
167
- tbody tr { cursor: pointer; }
168
- tbody tr:hover { background: var(--panel-2); }
169
- tbody tr.played td { box-shadow: inset 2px 0 0 var(--accent); }
170
- td.san { font-family: "JetBrains Mono", ui-monospace, monospace; font-weight: 600; white-space: nowrap; }
171
- td.num { font-family: "JetBrains Mono", ui-monospace, monospace; font-size: 12px; }
172
- .bar { position: relative; height: 9px; min-width: 60px; background: var(--panel-2); border-radius: 4px; }
173
- .bar i { position: absolute; left: 0; top: 0; bottom: 0; background: var(--seq); border-radius: 0 4px 4px 0; }
174
- .chip { display: inline-block; padding: 1px 7px; border-radius: 999px; font-family: "JetBrains Mono", ui-monospace, monospace; font-size: 11px; border: 1px solid var(--line-strong); }
175
-
176
- /* ── charts ─────────────────────────────────────────────── */
177
- .chartbox { padding: 14px 16px 8px; }
178
- svg.chart { width: 100%; height: auto; display: block; overflow: visible; }
179
- .axis text { font-family: "JetBrains Mono", ui-monospace, monospace; font-size: 9px; fill: var(--ink-3); }
180
- .grid line { stroke: var(--line); stroke-width: 1; }
181
- .zero { stroke: var(--line-strong); stroke-width: 1; stroke-dasharray: 3 3; }
182
- .vline { stroke: var(--pos); stroke-width: 2; fill: none; stroke-linejoin: round; }
183
- .tip { position: fixed; z-index: 40; pointer-events: none; background: var(--panel); border: 1px solid var(--line-strong); border-radius: 8px; padding: 7px 10px; font-family: "JetBrains Mono", ui-monospace, monospace; font-size: 11.5px; color: var(--ink); box-shadow: var(--shadow); opacity: 0; transition: opacity .1s; max-width: 260px; }
184
- .tip.on { opacity: 1; }
185
-
186
- /* ── attention explorer ─────────────────────────────────── */
187
- .attn-section { margin-top: 22px; }
188
- .attn-grid { display: grid; grid-template-columns: minmax(300px, 480px) 1fr; gap: 20px; align-items: start; }
189
- @media (max-width: 980px) { .attn-grid { grid-template-columns: 1fr; } }
190
- .controls { display: flex; flex-wrap: wrap; gap: 14px; align-items: center; padding: 12px 16px; border-bottom: 1px solid var(--line); }
191
- .controls label { font-size: 11px; letter-spacing: .1em; text-transform: uppercase; color: var(--ink-3); font-variation-settings: "wdth" 88, "wght" 600; }
192
- input[type=range] { accent-color: var(--accent); width: 190px; }
193
- select, button.ghost { font-family: inherit; font-size: 12.5px; color: var(--ink); background: var(--panel-2); border: 1px solid var(--line); border-radius: 7px; padding: 5px 9px; }
194
- canvas { display: block; image-rendering: pixelated; width: 100%; height: auto; border-radius: 6px; }
195
- .heads { display: grid; grid-template-columns: repeat(auto-fill, minmax(84px, 1fr)); gap: 10px; padding: 14px 16px; }
196
- .headcell { border: 1px solid var(--line); border-radius: 7px; padding: 5px; cursor: pointer; background: var(--panel-2); }
197
- .headcell[aria-pressed="true"] { border-color: var(--accent); box-shadow: 0 0 0 1px var(--accent); }
198
- .headcell figcaption { font-family: "JetBrains Mono", ui-monospace, monospace; font-size: 10px; color: var(--ink-3); text-align: center; margin-top: 4px; }
199
- .biasrow { display: flex; gap: 3px; padding: 0 16px 14px; }
200
- .biasrow div { flex: 1; text-align: center; font-family: "JetBrains Mono", ui-monospace, monospace; font-size: 9.5px; color: var(--ink-3); }
201
- .biasrow .cell { height: 26px; border-radius: 4px; border: 1px solid var(--line); margin-bottom: 3px; }
202
-
203
- details.reading { margin-top: 22px; }
204
- details.reading summary { cursor: pointer; font-size: 11px; letter-spacing: .13em; text-transform: uppercase; color: var(--ink-2); font-variation-settings: "wdth" 88, "wght" 600; padding: 12px 0; }
205
- .reading .cols { display: grid; grid-template-columns: repeat(auto-fit, minmax(260px, 1fr)); gap: 18px; }
206
- .reading p { margin: 0 0 8px; color: var(--ink-2); font-size: 13.5px; max-width: 62ch; }
207
- .reading code { font-family: "JetBrains Mono", ui-monospace, monospace; font-size: 12px; background: var(--panel-2); border: 1px solid var(--line); border-radius: 4px; padding: 1px 5px; }
208
- .reading h3 { font-size: 13px; margin: 0 0 6px; font-variation-settings: "wdth" 100, "wght" 700; }
209
- :focus-visible { outline: 2px solid var(--accent); outline-offset: 2px; }
210
- @media (prefers-reduced-motion: reduce) { * { transition: none !important; animation: none !important; } }
211
- </style>
212
-
213
- <div class="wrap">
214
- <header>
215
- <h1><span class="sub" id="eyebrow">Model introspection</span><span id="title">Glass Box</span></h1>
216
- <div class="meta-grid" id="meta"></div>
217
- </header>
218
-
219
- <div class="strip" id="strip"></div>
220
-
221
- <div class="main">
222
- <section class="panel board-shell">
223
- <div class="board-frame">
224
- <div class="board" id="board"></div>
225
- <svg id="arrows" viewBox="0 0 800 800" aria-hidden="true"></svg>
226
- </div>
227
- <div class="modes" id="modes"></div>
228
- <div class="scalebar" id="scalebar"></div>
229
- <div class="hint" id="overlay-hint"></div>
230
- <div class="fen" id="fen"></div>
231
- </section>
232
-
233
- <section style="display:grid; gap:16px;">
234
- <div class="tiles" id="tiles"></div>
235
-
236
- <div class="panel">
237
- <div class="head"><span>Legal moves — policy vs. value</span><span class="note" id="moves-note"></span></div>
238
- <div class="tablewrap"><table id="moves"></table></div>
239
- </div>
240
-
241
- <div class="panel chartbox">
242
- <div class="head" style="margin:-14px -16px 10px; padding:12px 16px;"><span>Where the answer forms</span><span class="note" id="depth-note"></span></div>
243
- <svg class="chart" id="depthchart" viewBox="0 0 720 260" preserveAspectRatio="xMidYMid meet"></svg>
244
- </div>
245
- </section>
246
- </div>
247
-
248
- <section class="panel attn-section" id="attn-panel">
249
- <div class="controls">
250
- <span style="font-size:11px;letter-spacing:.13em;text-transform:uppercase;font-variation-settings:'wdth' 88,'wght' 600;color:var(--ink-2)">Attention</span>
251
- <label for="layer">Layer</label>
252
- <input type="range" id="layer" min="0" max="15" value="15">
253
- <span class="note" id="layer-val" style="font-family:'JetBrains Mono',monospace;font-size:12px"></span>
254
- <label for="head">Head</label>
255
- <select id="head"></select>
256
- <label for="attnmode">Read</label>
257
- <select id="attnmode">
258
- <option value="row">row — where a token looks</option>
259
- <option value="col">column — who looks at a token</option>
260
- </select>
261
- <span class="note" id="attn-status" style="color:var(--ink-3);font-family:'JetBrains Mono',monospace;font-size:11px"></span>
262
- </div>
263
- <div class="attn-grid">
264
- <div style="padding:16px">
265
- <canvas id="heatmap" width="67" height="67"></canvas>
266
- <div class="scalebar"><span>0</span><span class="ramp" id="attn-ramp"></span><span id="attn-max">max</span></div>
267
- <div class="hint">Rows are queries (the token doing the looking), columns are keys. The last three rows/columns are the castling, en-passant and side-to-move tokens. Click a cell to select that square on the board.</div>
268
- </div>
269
- <div>
270
- <div class="head" style="border-top:0"><span>All heads, layer <span id="hl-layer"></span></span><span class="note">click to select</span></div>
271
- <div class="heads" id="heads"></div>
272
- <div class="head"><span>Relation bias — head <span id="hb-head"></span></span><span class="note">learned, position-independent</span></div>
273
- <div class="biasrow" id="biasrow"></div>
274
- </div>
275
- </div>
276
- </section>
277
-
278
- <details class="reading">
279
- <summary>How to read this / how it was made</summary>
280
- <div class="cols">
281
- <div>
282
- <h3>Policy vs. value disagreement</h3>
283
- <p>Every legal move carries its policy probability and the <em>child value</em> — the value head's verdict on the position after that move, from the mover's side. A move the policy loves but the value dislikes (or vice versa) is where search and network fight each other.</p>
284
- </div>
285
- <div>
286
- <h3>Where the answer forms</h3>
287
- <p>The value and policy are decoded from <em>every</em> layer's residual stream through the final norm and the real heads (a logit lens). If the top move is already locked in by layer 6 and nothing moves after, the last ten layers are not doing policy work.</p>
288
- </div>
289
- <div>
290
- <h3>Occlusion saliency</h3>
291
- <p>Each non-king piece is deleted in turn and the position re-evaluated. The square shows the swing in value, or in the probability of the move the model wanted. Big swings mark the pieces the verdict actually rests on.</p>
292
- </div>
293
- <div>
294
- <h3>Attention and rollout</h3>
295
- <p>Attention is recomputed from each layer's captured input using the model's own projections and its chess-relation bias, then checked against the real forward pass. Rollout multiplies the head-averaged, residual-corrected maps through the stack and reads the game-state token rows.</p>
296
- </div>
297
- </div>
298
- <p style="margin-top:10px"><code id="repro"></code></p>
299
- </details>
300
- </div>
301
-
302
- <div class="tip" id="tip"></div>
303
-
304
- <script id="dump" type="application/json">__PAYLOAD__</script>
305
- <script>
306
- "use strict";
307
- const DUMP = JSON.parse(document.getElementById("dump").textContent);
308
- const META = DUMP.meta;
309
- const POS = DUMP.positions;
310
- const L = META.n_layers, H = META.n_heads, T = 67;
311
- const GLYPH = { p: "♟", n: "♞", b: "♝", r: "♜", q: "♛", k: "♚" };
312
- const FILES = "abcdefgh";
313
- const STATE_NAMES = ["castling", "en-passant", "side-to-move"];
314
-
315
- const state = {
316
- pos: 0,
317
- mode: "policy",
318
- layer: L - 1,
319
- head: -1, // -1 = mean over heads
320
- attnMode: "row",
321
- square: null, // selected token index 0..66
322
- hoverMove: null,
323
- };
324
- const attnCache = new Map(); // pos index -> {q: Uint8Array, scale: Float32Array}
325
-
326
- const css = (n) => getComputedStyle(document.documentElement).getPropertyValue(n).trim();
327
- const sqName = (i) => (i < 64 ? FILES[i % 8] + (Math.floor(i / 8) + 1) : STATE_NAMES[i - 64]);
328
- const fmt = (x, d = 3) => (x >= 0 ? "+" : "−") + Math.abs(x).toFixed(d);
329
- const pct = (x) => (100 * x).toFixed(1) + "%";
330
-
331
- /* ── colour ramps ──────────────────────────────────────────
332
- sequential (attention, one hue, light→dark) and diverging
333
- (value / saliency, two poles with a neutral middle). */
334
- function mix(a, b, t) { return a.map((v, i) => Math.round(v + (b[i] - v) * t)); }
335
- function rgb(c, alpha) { return `rgba(${c[0]},${c[1]},${c[2]},${alpha == null ? 1 : alpha})`; }
336
- function hex2rgb(h) { h = h.replace("#", ""); return [0, 2, 4].map((i) => parseInt(h.slice(i, i + 2), 16)); }
337
- function seqColor(t) { // 0..1
338
- const lo = hex2rgb(css("--panel-2")), hi = hex2rgb(css("--seq"));
339
- return mix(lo, hi, Math.max(0, Math.min(1, Math.pow(t, 0.45))));
340
- }
341
- function divColor(t) { // -1..1
342
- const mid = hex2rgb(css("--neutral"));
343
- const pole = hex2rgb(t >= 0 ? css("--pos") : css("--neg"));
344
- return mix(mid, pole, Math.min(1, Math.abs(t)));
345
- }
346
-
347
- /* ── attention decoding ────────────────────────────────── */
348
- async function inflate(b64) {
349
- const bin = atob(b64);
350
- const bytes = new Uint8Array(bin.length);
351
- for (let i = 0; i < bin.length; i++) bytes[i] = bin.charCodeAt(i);
352
- if (typeof DecompressionStream === "undefined") throw new Error("no DecompressionStream");
353
- const stream = new Blob([bytes]).stream().pipeThrough(new DecompressionStream("deflate"));
354
- return new Uint8Array(await new Response(stream).arrayBuffer());
355
- }
356
- async function getAttn(i) {
357
- if (attnCache.has(i)) return attnCache.get(i);
358
- const a = POS[i].attention;
359
- if (!a) return null;
360
- const q = await inflate(a.data);
361
- const rec = { q, scale: Float32Array.from(a.scale) };
362
- attnCache.set(i, rec);
363
- return rec;
364
- }
365
- // attention[layer][head][from][to], de-quantised with the per-row scale.
366
- // The stored byte is sqrt-companded, so square it on the way back out.
367
- function attnAt(rec, l, h, i, j) {
368
- const row = (l * H + h) * T + i;
369
- const u = rec.q[row * T + j] / 255;
370
- return u * u * rec.scale[row];
371
- }
372
- function headMeanAt(rec, l, i, j) {
373
- let s = 0;
374
- for (let h = 0; h < H; h++) s += attnAt(rec, l, h, i, j);
375
- return s / H;
376
- }
377
-
378
- /* ── board ─────────────────────────────────────────────── */
379
- const boardEl = document.getElementById("board");
380
- const cells = [];
381
- for (let row = 0; row < 8; row++) {
382
- for (let col = 0; col < 8; col++) {
383
- const sq = (7 - row) * 8 + col;
384
- const d = document.createElement("div");
385
- d.className = "sq " + ((row + col) % 2 === 0 ? "light" : "dark");
386
- d.dataset.sq = sq;
387
- d.innerHTML =
388
- '<span class="heat"></span>' +
389
- (col === 0 ? `<span class="rank-lbl">${8 - row}</span>` : "") +
390
- (row === 7 ? `<span class="file-lbl">${FILES[col]}</span>` : "") +
391
- '<span class="glyph"></span><span class="val"></span>';
392
- d.addEventListener("click", () => {
393
- state.square = state.square === sq ? null : sq;
394
- render();
395
- });
396
- boardEl.appendChild(d);
397
- cells[sq] = d;
398
- }
399
- }
400
-
401
- const MODES = [
402
- { id: "policy", label: "Policy", hint: "Arrow width and opacity follow the move probability. The dashed arrow is the move actually played in the game." },
403
- { id: "attention", label: "Attention", hint: "Attention for the selected layer/head. Click a square to pin a query token; with nothing selected the board shows attention received from the three game-state tokens." },
404
- { id: "rollout", label: "Rollout", hint: "Attention rollout through the whole stack, read from the game-state tokens the value head consumes." },
405
- { id: "value", label: "Value saliency", hint: "Delete this piece, and the value moves by this much. Teal = the model likes the position more without it." },
406
- { id: "policysal", label: "Move saliency", hint: "Delete this piece, and the probability of the model's chosen move changes by this much." },
407
- ];
408
- const modesEl = document.getElementById("modes");
409
- MODES.forEach((m) => {
410
- const b = document.createElement("button");
411
- b.className = "mode"; b.textContent = m.label; b.dataset.mode = m.id;
412
- b.addEventListener("click", () => { state.mode = m.id; render(); });
413
- modesEl.appendChild(b);
414
- });
415
-
416
- /* ── overlays ──────────────────────────────────────────── */
417
- function overlayValues(p, rec) {
418
- // Returns {v: Float64Array(64), kind: "seq"|"div", max, label}
419
- const out = new Float64Array(64);
420
- if (state.mode === "rollout" && p.attention) {
421
- for (let i = 0; i < 64; i++) out[i] = p.attention.rollout[i];
422
- return { v: out, kind: "seq", max: Math.max(...out), label: "rollout weight" };
423
- }
424
- if (state.mode === "value" && p.saliency) {
425
- for (let i = 0; i < 64; i++) out[i] = p.saliency.value_delta[i];
426
- return { v: out, kind: "div", max: Math.max(...out.map(Math.abs)), label: "Δ value" };
427
- }
428
- if (state.mode === "policysal" && p.saliency) {
429
- for (let i = 0; i < 64; i++) out[i] = p.saliency.policy_delta[i];
430
- return { v: out, kind: "div", max: Math.max(...out.map(Math.abs)), label: "Δ p(" + p.saliency.top_move + ")" };
431
- }
432
- if (state.mode === "attention" && rec) {
433
- const l = state.layer;
434
- const get = (i, j) => (state.head < 0 ? headMeanAt(rec, l, i, j) : attnAt(rec, l, state.head, i, j));
435
- if (state.square !== null) {
436
- for (let i = 0; i < 64; i++) out[i] = state.attnMode === "row" ? get(state.square, i) : get(i, state.square);
437
- } else {
438
- for (let i = 0; i < 64; i++) { let s = 0; for (let t = 64; t < 67; t++) s += get(t, i); out[i] = s / 3; }
439
- }
440
- return { v: out, kind: "seq", max: Math.max(...out), label: "attention" };
441
- }
442
- return null;
443
- }
444
-
445
- function paintBoard(p, rec) {
446
- const ov = overlayValues(p, rec);
447
- const lastFrom = p.last_move ? sqIndex(p.last_move.slice(0, 2)) : null;
448
- const lastTo = p.last_move ? sqIndex(p.last_move.slice(2, 4)) : null;
449
-
450
- for (let sq = 0; sq < 64; sq++) {
451
- const cell = cells[sq];
452
- const glyph = cell.querySelector(".glyph");
453
- const piece = p.pieces[sq];
454
- if (piece) {
455
- glyph.textContent = GLYPH[piece.toLowerCase()];
456
- glyph.className = "glyph " + (piece === piece.toUpperCase() ? "w" : "b");
457
- } else { glyph.textContent = ""; glyph.className = "glyph"; }
458
-
459
- const heat = cell.querySelector(".heat");
460
- const val = cell.querySelector(".val");
461
- if (ov && ov.max > 0) {
462
- const x = ov.v[sq];
463
- if (ov.kind === "seq") {
464
- heat.style.background = rgb(seqColor(x / ov.max), Math.min(0.88, 0.12 + 0.88 * (x / ov.max)));
465
- } else {
466
- const t = x / ov.max;
467
- heat.style.background = Math.abs(t) < 0.02 ? "transparent" : rgb(divColor(t), Math.min(0.85, Math.abs(t) * 0.9));
468
- }
469
- val.textContent = ov.kind === "div"
470
- ? (Math.abs(x) >= ov.max * 0.25 ? (x >= 0 ? "+" : "−") + Math.abs(x).toFixed(2).slice(1) : "")
471
- : (x >= ov.max * 0.3 ? (100 * x).toFixed(0) + "%" : "");
472
- } else { heat.style.background = "transparent"; val.textContent = ""; }
473
-
474
- cell.classList.toggle("sel", state.square === sq);
475
- cell.classList.toggle("lastmove", sq === lastFrom || sq === lastTo);
476
- }
477
-
478
- const bar = document.getElementById("scalebar");
479
- if (ov && ov.max > 0) {
480
- const stops = ov.kind === "seq"
481
- ? [0, .25, .5, .75, 1].map((t) => rgb(seqColor(t))).join(",")
482
- : [-1, -.5, 0, .5, 1].map((t) => rgb(divColor(t))).join(",");
483
- bar.innerHTML = `<span>${ov.kind === "div" ? fmt(-ov.max, 2) : "0"}</span>
484
- <span class="ramp" style="background:linear-gradient(90deg,${stops})"></span>
485
- <span>${ov.kind === "div" ? fmt(ov.max, 2) : (100 * ov.max).toFixed(1) + "%"}</span>
486
- <span style="margin-left:6px">${ov.label}</span>`;
487
- } else bar.innerHTML = "";
488
- }
489
-
490
- function sqIndex(name) { return (parseInt(name[1], 10) - 1) * 8 + FILES.indexOf(name[0]); }
491
- function sqCenter(sq) { const c = sq % 8, r = Math.floor(sq / 8); return [c * 100 + 50, (7 - r) * 100 + 50]; }
492
-
493
- function paintArrows(p) {
494
- const svg = document.getElementById("arrows");
495
- svg.innerHTML = "";
496
- if (state.mode !== "policy" && state.hoverMove === null) return;
497
- const moves = state.hoverMove !== null
498
- ? [p.policy.moves[state.hoverMove]]
499
- : p.policy.moves.slice(0, 6);
500
- const defs = document.createElementNS("http://www.w3.org/2000/svg", "defs");
501
- svg.appendChild(defs);
502
- moves.forEach((m, i) => {
503
- const [x1, y1] = sqCenter(m.from), [x2, y2] = sqCenter(m.to);
504
- const opacity = state.hoverMove !== null ? 0.95 : Math.max(0.18, Math.min(0.92, 0.25 + m.prob));
505
- const w = 6 + 20 * Math.sqrt(m.prob);
506
- const color = i === 0 || state.hoverMove !== null ? css("--accent") : css("--seq");
507
- const dx = x2 - x1, dy = y2 - y1, len = Math.hypot(dx, dy);
508
- const ux = dx / len, uy = dy / len;
509
- const hx = x2 - ux * 26, hy = y2 - uy * 26; // stop short for the head
510
- const line = document.createElementNS("http://www.w3.org/2000/svg", "path");
511
- line.setAttribute("d", `M ${x1 + ux * 16} ${y1 + uy * 16} L ${hx} ${hy}`);
512
- line.setAttribute("stroke", color);
513
- line.setAttribute("stroke-width", w);
514
- line.setAttribute("stroke-linecap", "round");
515
- line.setAttribute("opacity", opacity);
516
- line.setAttribute("fill", "none");
517
- svg.appendChild(line);
518
- const head = document.createElementNS("http://www.w3.org/2000/svg", "path");
519
- const px = -uy, py = ux, s = w * 1.5;
520
- head.setAttribute("d", `M ${x2 - ux * 4} ${y2 - uy * 4} L ${hx + px * s} ${hy + py * s} L ${hx - px * s} ${hy - py * s} Z`);
521
- head.setAttribute("fill", color);
522
- head.setAttribute("opacity", opacity);
523
- svg.appendChild(head);
524
- });
525
- const topUci = p.policy.moves[0] && p.policy.moves[0].uci;
526
- if (p.played_uci && p.played_uci !== topUci && state.hoverMove === null) {
527
- const [x1, y1] = sqCenter(sqIndex(p.played_uci.slice(0, 2)));
528
- const [x2, y2] = sqCenter(sqIndex(p.played_uci.slice(2, 4)));
529
- const dash = document.createElementNS("http://www.w3.org/2000/svg", "path");
530
- dash.setAttribute("d", `M ${x1} ${y1} L ${x2} ${y2}`);
531
- dash.setAttribute("stroke", css("--ink"));
532
- dash.setAttribute("stroke-width", "3");
533
- dash.setAttribute("stroke-dasharray", "10 8");
534
- dash.setAttribute("opacity", ".65");
535
- dash.setAttribute("fill", "none");
536
- svg.appendChild(dash);
537
- }
538
- }
539
-
540
- /* ── tiles ─────────────────────────────────────────────── */
541
- // The move the value head would pick if it, not the policy, chose. When this
542
- // disagrees with the policy's move, search will pull away from the prior —
543
- // and that gap is usually where the interesting bugs live.
544
- function valuePick(p) {
545
- const scored = p.policy.moves.filter((m) => m.child_value != null);
546
- if (scored.length < 2) return null;
547
- let best = scored[0];
548
- for (const m of scored) if (m.child_value > best.child_value) best = m;
549
- const same = best.uci === p.policy.moves[0].uci;
550
- return {
551
- lbl: "Value head's pick",
552
- big: best.san,
553
- small: same ? "agrees with the policy" : `policy rank #${best.rank + 1} · ${pct(best.prob)} · ${fmt(best.child_value, 3)}`,
554
- meter: { from: 0, to: best.prob, color: same ? css("--pos") : css("--neg") },
555
- };
556
- }
557
-
558
- function paintTiles(p) {
559
- const top = p.policy.moves[0];
560
- const ent = p.policy.entropy, maxEnt = p.policy.max_entropy || 1;
561
- const played = p.played_san;
562
- const rankOfPlayed = played ? p.policy.moves.findIndex((m) => m.san === played) : -1;
563
- const vPct = (p.value + 1) / 2;
564
- const cmp = p.compare;
565
- const tiles = [
566
- { lbl: "Model's move", big: top ? top.san : "—", small: top ? pct(top.prob) + " of the legal mass" + (cmp ? " · other: " + cmp.top1 : "") : "",
567
- meter: { from: 0, to: top ? top.prob : 0, color: css("--seq") } },
568
- { lbl: "Value (side to move)", big: top ? fmt(p.value, 3) : fmt(p.value, 3),
569
- small: (p.turn === "w" ? "White" : "Black") + " to move" + (cmp ? " · other: " + fmt(cmp.value, 3) : ""),
570
- meter: { from: Math.min(0.5, vPct), to: Math.max(0.5, vPct), color: p.value >= 0 ? css("--pos") : css("--neg"), mid: true } },
571
- { lbl: "Policy entropy", big: ent.toFixed(2) + " b", small: "of " + maxEnt.toFixed(2) + " b over " + p.legal_count + " moves",
572
- meter: { from: 0, to: ent / maxEnt, color: css("--accent") } },
573
- valuePick(p),
574
- played ? { lbl: "Move played", big: played, small: rankOfPlayed >= 0 ? "model rank #" + (rankOfPlayed + 1) + " · " + pct(p.policy.moves[rankOfPlayed].prob) : "not in the legal list",
575
- meter: { from: 0, to: rankOfPlayed >= 0 ? p.policy.moves[rankOfPlayed].prob : 0, color: css("--accent") } } : null,
576
- ].filter(Boolean);
577
-
578
- document.getElementById("tiles").innerHTML = tiles.map((t) => `
579
- <div class="tile">
580
- <div class="lbl">${t.lbl}</div>
581
- <div class="big">${t.big}</div>
582
- <div class="small">${t.small}</div>
583
- <div class="meter">${t.meter.mid ? '<span class="mid"></span>' : ""}
584
- <i style="left:${100 * t.meter.from}%;width:${100 * (t.meter.to - t.meter.from)}%;background:${t.meter.color}"></i>
585
- </div>
586
- </div>`).join("");
587
- }
588
-
589
- /* ── move table ────────────────────────────────────────── */
590
- function paintMoves(p) {
591
- const hasCmp = p.policy.moves.some((m) => m.compare_prob != null);
592
- const rows = p.policy.moves.slice(0, 18);
593
- const head = `<thead><tr><th>#</th><th>Move</th><th style="width:34%">Policy</th><th>p</th>
594
- <th>Child value</th><th>Value gain</th>${hasCmp ? "<th>Other model</th>" : ""}<th></th></tr></thead>`;
595
- const body = rows.map((m, i) => {
596
- const cv = m.child_value;
597
- const tension = cv == null ? null : cv - p.policy.moves[0].child_value;
598
- return `<tr data-i="${i}" class="${p.played_san && m.san === p.played_san ? "played" : ""}">
599
- <td class="num" style="color:var(--ink-3)">${i + 1}</td>
600
- <td class="san">${m.san}</td>
601
- <td><div class="bar"><i style="width:${Math.max(1.5, 100 * m.prob)}%"></i></div></td>
602
- <td class="num">${(100 * m.prob).toFixed(1)}</td>
603
- <td class="num"><span class="chip" style="color:${cv == null ? "var(--ink-3)" : cv >= 0 ? "var(--pos)" : "var(--neg)"}">${cv == null ? "—" : fmt(cv, 3)}</span></td>
604
- <td class="num" style="color:${tension > 0.05 ? "var(--pos)" : "var(--ink-3)"}">${tension == null ? "" : (tension > 0 ? "+" : "") + tension.toFixed(3)}</td>
605
- ${hasCmp ? `<td class="num" style="color:var(--ink-3)">${m.compare_prob == null ? "—" : (100 * m.compare_prob).toFixed(1)}</td>` : ""}
606
- <td class="num" style="color:var(--ink-3)">${sqName(m.from)}→${sqName(m.to)}</td>
607
- </tr>`;
608
- }).join("");
609
- const t = document.getElementById("moves");
610
- t.innerHTML = head + "<tbody>" + body + "</tbody>";
611
- t.querySelectorAll("tbody tr").forEach((tr) => {
612
- tr.addEventListener("mouseenter", () => { state.hoverMove = +tr.dataset.i; paintArrows(p); });
613
- tr.addEventListener("mouseleave", () => { state.hoverMove = null; paintArrows(p); });
614
- tr.addEventListener("click", () => { state.square = +p.policy.moves[+tr.dataset.i].from; render(); });
615
- });
616
- document.getElementById("moves-note").textContent =
617
- `${p.policy.n_moves} legal · ${pct(p.policy.legal_mass)} of the raw 64×73 softmax is legal` +
618
- (rows.length < p.policy.n_moves ? ` · top ${rows.length} shown` : "");
619
- }
620
-
621
- /* ── depth chart ───────────────────────────────────────── */
622
- function paintDepth(p) {
623
- const d = p.depth;
624
- if (!d) return;
625
- const W = 720, Ht = 260, m = { l: 44, r: 16, t: 16, b: 54 };
626
- const iw = W - m.l - m.r, ih = Ht - m.t - m.b;
627
- const vals = d.value_by_layer;
628
- const lim = Math.max(0.15, ...vals.map((v) => Math.abs(v)), Math.abs(p.value)) * 1.15;
629
- const x = (i) => m.l + (iw * i) / Math.max(1, vals.length - 1);
630
- const y = (v) => m.t + ih * (1 - (v + lim) / (2 * lim));
631
-
632
- let g = `<g class="grid">`;
633
- [-lim, -lim / 2, 0, lim / 2, lim].forEach((v) => { g += `<line x1="${m.l}" y1="${y(v).toFixed(1)}" x2="${W - m.r}" y2="${y(v).toFixed(1)}"/>`; });
634
- g += `</g><line class="zero" x1="${m.l}" y1="${y(0)}" x2="${W - m.r}" y2="${y(0)}"/>`;
635
- g += `<g class="axis">` + [-lim, 0, lim].map((v) => `<text x="${m.l - 8}" y="${y(v) + 3}" text-anchor="end">${v.toFixed(2)}</text>`).join("") + `</g>`;
636
-
637
- const path = vals.map((v, i) => `${i ? "L" : "M"} ${x(i).toFixed(1)} ${y(v).toFixed(1)}`).join(" ");
638
- const area = `${path} L ${x(vals.length - 1).toFixed(1)} ${y(0)} L ${m.l} ${y(0)} Z`;
639
- g += `<path d="${area}" fill="${css("--pos")}" opacity=".12"/>`;
640
- g += `<path class="vline" d="${path}" stroke="${css("--pos")}"/>`;
641
- g += `<circle cx="${x(vals.length - 1)}" cy="${y(vals[vals.length - 1])}" r="4.5" fill="${css("--pos")}"/>`;
642
- g += `<text x="${x(vals.length - 1) - 8}" y="${y(vals[vals.length - 1]) - 10}" text-anchor="end" font-family="JetBrains Mono, monospace" font-size="11" fill="${css("--ink")}">${fmt(p.value, 3)}</text>`;
643
-
644
- // Agreement strip: does the logit lens at this layer already pick the final move?
645
- const sy = m.t + ih + 16;
646
- d.top_by_layer.forEach((t, i) => {
647
- const w = Math.max(6, iw / vals.length - 3);
648
- g += `<rect x="${(x(i) - w / 2).toFixed(1)}" y="${sy}" width="${w.toFixed(1)}" height="10" rx="3"
649
- fill="${t.agrees ? css("--accent") : css("--line-strong")}" opacity="${t.agrees ? 0.9 : 0.6}"/>`;
650
- g += `<rect class="hitbox" data-i="${i}" x="${(x(i) - w / 2 - 2).toFixed(1)}" y="${m.t}" width="${(w + 4).toFixed(1)}" height="${ih + 30}" fill="transparent"/>`;
651
- });
652
- g += `<g class="axis">` + vals.map((_, i) =>
653
- (i % 2 === 0 || i === vals.length - 1) ? `<text x="${x(i)}" y="${sy + 26}" text-anchor="middle">${i + 1}</text>` : "").join("") + `</g>`;
654
- g += `<text class="axis" x="${m.l}" y="${Ht - 4}" font-family="JetBrains Mono, monospace" font-size="9" fill="${css("--ink-3")}">layer → · value read from each layer (line), lens already picks the final move (gold)</text>`;
655
-
656
- const svg = document.getElementById("depthchart");
657
- svg.innerHTML = g;
658
- svg.querySelectorAll(".hitbox").forEach((r) => {
659
- r.addEventListener("mousemove", (e) => {
660
- const i = +r.dataset.i, t = d.top_by_layer[i];
661
- showTip(e, `layer ${i + 1}<br>value ${fmt(vals[i], 3)}<br>lens top: ${t.san} ${pct(t.prob)}<br>final move here: ${pct(t.final_move_prob)}<br>write norm ${d.residual_write_norm[i].toFixed(1)}`);
662
- });
663
- r.addEventListener("mouseleave", hideTip);
664
- });
665
-
666
- const firstAgree = d.top_by_layer.findIndex((t, i) => d.top_by_layer.slice(i).every((u) => u.agrees));
667
- document.getElementById("depth-note").textContent =
668
- firstAgree >= 0 ? `final move locked in from layer ${firstAgree + 1} of ${L}` : "the final move is never stable across layers";
669
- }
670
-
671
- const tipEl = document.getElementById("tip");
672
- function showTip(e, html) {
673
- tipEl.innerHTML = html; tipEl.classList.add("on");
674
- const r = tipEl.getBoundingClientRect();
675
- tipEl.style.left = Math.min(window.innerWidth - r.width - 10, e.clientX + 14) + "px";
676
- tipEl.style.top = Math.max(8, e.clientY - r.height - 12) + "px";
677
- }
678
- function hideTip() { tipEl.classList.remove("on"); }
679
-
680
- /* ── attention explorer ────────────────────────────────── */
681
- function paintHeatmap(rec) {
682
- const cv = document.getElementById("heatmap");
683
- const ctx = cv.getContext("2d");
684
- if (!rec) { ctx.clearRect(0, 0, T, T); return; }
685
- const img = ctx.createImageData(T, T);
686
- const get = (i, j) => (state.head < 0 ? headMeanAt(rec, state.layer, i, j) : attnAt(rec, state.layer, state.head, i, j));
687
- let max = 0;
688
- const buf = new Float64Array(T * T);
689
- for (let i = 0; i < T; i++) for (let j = 0; j < T; j++) { const v = get(i, j); buf[i * T + j] = v; if (v > max) max = v; }
690
- for (let i = 0; i < T; i++) for (let j = 0; j < T; j++) {
691
- const c = seqColor(buf[i * T + j] / (max || 1));
692
- const o = (i * T + j) * 4;
693
- img.data[o] = c[0]; img.data[o + 1] = c[1]; img.data[o + 2] = c[2]; img.data[o + 3] = 255;
694
- }
695
- ctx.putImageData(img, 0, 0);
696
- document.getElementById("attn-max").textContent = max.toFixed(3);
697
- document.getElementById("attn-ramp").style.background =
698
- `linear-gradient(90deg,${[0, .25, .5, .75, 1].map((t) => rgb(seqColor(t))).join(",")})`;
699
- }
700
-
701
- function paintHeads(rec) {
702
- const wrap = document.getElementById("heads");
703
- wrap.innerHTML = "";
704
- document.getElementById("hl-layer").textContent = state.layer + 1;
705
- if (!rec) return;
706
- const opts = [{ h: -1, name: "mean" }].concat(Array.from({ length: H }, (_, h) => ({ h, name: "h" + h })));
707
- opts.forEach(({ h, name }) => {
708
- const fig = document.createElement("figure");
709
- fig.className = "headcell"; fig.style.margin = "0";
710
- fig.setAttribute("aria-pressed", state.head === h);
711
- const cv = document.createElement("canvas");
712
- cv.width = T; cv.height = T;
713
- const ctx = cv.getContext("2d");
714
- const img = ctx.createImageData(T, T);
715
- let max = 0; const buf = new Float64Array(T * T);
716
- for (let i = 0; i < T; i++) for (let j = 0; j < T; j++) {
717
- const v = h < 0 ? headMeanAt(rec, state.layer, i, j) : attnAt(rec, state.layer, h, i, j);
718
- buf[i * T + j] = v; if (v > max) max = v;
719
- }
720
- for (let i = 0; i < T * T; i++) {
721
- const c = seqColor(buf[i] / (max || 1)), o = i * 4;
722
- img.data[o] = c[0]; img.data[o + 1] = c[1]; img.data[o + 2] = c[2]; img.data[o + 3] = 255;
723
- }
724
- ctx.putImageData(img, 0, 0);
725
- fig.appendChild(cv);
726
- const cap = document.createElement("figcaption");
727
- cap.textContent = name;
728
- fig.appendChild(cap);
729
- fig.addEventListener("click", () => { state.head = h; render(); });
730
- wrap.appendChild(fig);
731
- });
732
- }
733
-
734
- function paintBias() {
735
- const row = document.getElementById("biasrow");
736
- const names = META.relation_names || [];
737
- const bias = META.relation_bias;
738
- document.getElementById("hb-head").textContent = state.head < 0 ? "mean" : state.head;
739
- if (!bias) { row.innerHTML = ""; return; }
740
- const vals = state.head < 0
741
- ? names.map((_, k) => bias[state.layer].reduce((s, h) => s + h[k], 0) / H)
742
- : bias[state.layer][state.head];
743
- const max = Math.max(...vals.map(Math.abs), 1e-6);
744
- row.innerHTML = vals.map((v, k) => `
745
- <div title="${names[k]}: ${fmt(v, 3)}">
746
- <div class="cell" style="background:${rgb(divColor(v / max), Math.min(0.9, 0.15 + Math.abs(v / max)))}"></div>
747
- ${names[k]}<br><span style="color:var(--ink-2)">${v.toFixed(2)}</span>
748
- </div>`).join("");
749
- }
750
-
751
- /* ── position strip, header ────────────────────────────── */
752
- function paintStrip() {
753
- const strip = document.getElementById("strip");
754
- if (POS.length < 2) { strip.style.display = "none"; return; }
755
- strip.innerHTML = POS.map((p, i) => {
756
- const label = p.ply != null ? `${Math.floor(p.ply / 2) + 1}${p.ply % 2 === 0 ? "." : "…"} ${p.played_san || "—"}` : `#${i + 1}`;
757
- return `<button class="ply" data-i="${i}" aria-current="${i === state.pos}">${label}</button>`;
758
- }).join("");
759
- strip.querySelectorAll(".ply").forEach((b) =>
760
- b.addEventListener("click", () => { state.pos = +b.dataset.i; state.square = null; render(); }));
761
- }
762
-
763
- function paintHeader() {
764
- document.getElementById("title").textContent = META.title || "Glass Box";
765
- const bits = [
766
- ["model", (META.model_dir || "").split("/").pop() + (META.ema ? " (EMA)" : "")],
767
- ["shape", `${META.n_layers}L · ${META.n_heads}H · d${META.config ? META.config.embed_dim : "?"}`],
768
- META.compare_dir ? ["vs", META.compare_dir.split("/").pop()] : null,
769
- ["attn check", "Δ " + Number(META.attention_check).toExponential(1)],
770
- ["generated", META.generated],
771
- ].filter(Boolean);
772
- document.getElementById("meta").innerHTML = bits.map(([k, v]) => `<span>${k} <b>${v}</b></span>`).join("");
773
- document.getElementById("repro").textContent =
774
- `python scripts/introspect_model.py --model-dir ${META.model_dir} --fen "<fen>" --out out/think.html`;
775
- }
776
-
777
- /* ── render ────────────────────────────────────────────── */
778
- let renderToken = 0;
779
- async function render() {
780
- const p = POS[state.pos];
781
- const my = ++renderToken;
782
- paintStrip();
783
- paintTiles(p);
784
- paintMoves(p);
785
- paintDepth(p);
786
- paintBias();
787
- document.getElementById("fen").textContent = p.fen;
788
- document.getElementById("layer-val").textContent = `${state.layer + 1}/${L}`;
789
- document.getElementById("overlay-hint").textContent =
790
- (MODES.find((m) => m.id === state.mode) || {}).hint +
791
- (state.square !== null ? ` · selected: ${sqName(state.square)}` : "");
792
- modesEl.querySelectorAll(".mode").forEach((b) => b.setAttribute("aria-pressed", b.dataset.mode === state.mode));
793
- document.getElementById("head").value = String(state.head);
794
- document.getElementById("attnmode").value = state.attnMode;
795
-
796
- let rec = null;
797
- if (p.attention) {
798
- const status = document.getElementById("attn-status");
799
- if (!attnCache.has(state.pos)) status.textContent = "decoding…";
800
- try { rec = await getAttn(state.pos); status.textContent = ""; }
801
- catch (e) { status.textContent = "attention unavailable: " + e.message; }
802
- } else {
803
- document.getElementById("attn-panel").style.display = "none";
804
- }
805
- if (my !== renderToken) return;
806
- paintBoard(p, rec);
807
- paintArrows(p);
808
- paintHeatmap(rec);
809
- paintHeads(rec);
810
- }
811
-
812
- /* ── controls ──────────────────────────────────────────── */
813
- const layerEl = document.getElementById("layer");
814
- layerEl.max = String(L - 1); layerEl.value = String(state.layer);
815
- layerEl.addEventListener("input", () => { state.layer = +layerEl.value; render(); });
816
-
817
- const headEl = document.getElementById("head");
818
- headEl.innerHTML = `<option value="-1">mean of ${H}</option>` +
819
- Array.from({ length: H }, (_, h) => `<option value="${h}">head ${h}</option>`).join("");
820
- headEl.addEventListener("change", () => { state.head = +headEl.value; render(); });
821
-
822
- document.getElementById("attnmode").addEventListener("change", (e) => { state.attnMode = e.target.value; render(); });
823
-
824
- document.getElementById("heatmap").addEventListener("click", async (e) => {
825
- const r = e.target.getBoundingClientRect();
826
- const j = Math.floor(((e.clientX - r.left) / r.width) * T);
827
- const i = Math.floor(((e.clientY - r.top) / r.height) * T);
828
- state.square = state.attnMode === "row" ? i : j;
829
- if (state.square > 63) state.square = null;
830
- state.mode = "attention";
831
- render();
832
- });
833
- document.getElementById("heatmap").addEventListener("mousemove", async (e) => {
834
- const rec = attnCache.get(state.pos);
835
- if (!rec) return;
836
- const r = e.target.getBoundingClientRect();
837
- const j = Math.floor(((e.clientX - r.left) / r.width) * T);
838
- const i = Math.floor(((e.clientY - r.top) / r.height) * T);
839
- if (i < 0 || j < 0 || i >= T || j >= T) return hideTip();
840
- const v = state.head < 0 ? headMeanAt(rec, state.layer, i, j) : attnAt(rec, state.layer, state.head, i, j);
841
- showTip(e, `${sqName(i)} → ${sqName(j)}<br>${v.toFixed(4)}`);
842
- });
843
- document.getElementById("heatmap").addEventListener("mouseleave", hideTip);
844
-
845
- window.addEventListener("keydown", (e) => {
846
- if (e.target.tagName === "SELECT" || e.target.tagName === "INPUT") return;
847
- const k = e.key;
848
- if (k === "ArrowRight" && state.pos < POS.length - 1) { state.pos++; state.square = null; render(); }
849
- else if (k === "ArrowLeft" && state.pos > 0) { state.pos--; state.square = null; render(); }
850
- else if (k === "ArrowUp" && state.layer < L - 1) { state.layer++; layerEl.value = state.layer; render(); }
851
- else if (k === "ArrowDown" && state.layer > 0) { state.layer--; layerEl.value = state.layer; render(); }
852
- else if (k === "]" && state.head < H - 1) { state.head++; render(); }
853
- else if (k === "[" && state.head >= 0) { state.head--; render(); }
854
- else return;
855
- e.preventDefault();
856
- });
857
-
858
- paintHeader();
859
- render();
860
- </script>
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
chesstransformer/models/__init__.py DELETED
@@ -1 +0,0 @@
1
- """This module contains all the models."""
 
 
chesstransformer/models/tokenizer/__init__.py DELETED
@@ -1,93 +0,0 @@
1
- from tokenizers import Tokenizer
2
- from tokenizers import models, pre_tokenizers, trainers, processors, normalizers
3
- from .move_tokenizer import MoveTokenizer
4
- from .position_tokenizer import PostionTokenizer
5
-
6
-
7
- def create_bpe_tokenizer(
8
- vocab_size: int = 2000,
9
- ) -> tuple[Tokenizer, trainers.BpeTrainer]:
10
- """
11
- Create a BPE tokenizer for chess moves.
12
-
13
- The tokenizer uses Byte-Pair Encoding (BPE) with ByteLevel pre-tokenization
14
- to handle the unique structure of chess moves. The <STEP> token is used to
15
- explicitly separate each move, ensuring clear move boundaries.
16
-
17
- Special tokens included:
18
- - <PAD>: Padding token
19
- - <START>: Start of sequence token
20
- - <END>: End of sequence token
21
- - <STEP>: Move separator token
22
- - <1-0>: White wins
23
- - <0-1>: Black wins
24
- - <1/2-1/2>: Draw
25
- - <UNK>: Unknown token
26
-
27
- Args:
28
- vocab_size: Size of the vocabulary (default: 2000)
29
-
30
- Returns:
31
- A Tokenizer object and trainer
32
- """
33
-
34
- # Initialize a tokenizer with BPE model
35
- encoder = Tokenizer(models.BPE(unk_token="<UNK>"))
36
- encoder.normalizer = normalizers.Sequence([]) # No normalization needed for chess moves
37
-
38
- # remove the white space pre-tokenizer
39
- encoder.pre_tokenizer = pre_tokenizers.Whitespace()
40
-
41
- # Setup trainer with special tokens, including space
42
- trainer = trainers.BpeTrainer(
43
- vocab_size=vocab_size,
44
- special_tokens=[
45
- "<PAD>",
46
- "<START>",
47
- "<END>",
48
- "<1-0>",
49
- "<0-1>",
50
- "<1/2-1/2>",
51
- "<UNK>",
52
- "<STEP>",
53
- ],
54
- show_progress=True,
55
- min_frequency=2,
56
- initial_alphabet=[
57
- "a",
58
- "b",
59
- "c",
60
- "d",
61
- "e",
62
- "f",
63
- "g",
64
- "h",
65
- "1",
66
- "2",
67
- "3",
68
- "4",
69
- "5",
70
- "6",
71
- "7",
72
- "8",
73
- "q",
74
- "k",
75
- "r",
76
- "b",
77
- "n", # Piece notations
78
- ],
79
- )
80
-
81
- # Add post-processing to add special tokens
82
- encoder.post_processor = processors.TemplateProcessing(
83
- single="<START> $A <END>",
84
- special_tokens=[
85
- ("<START>", 1),
86
- ("<END>", 2),
87
- ],
88
- )
89
-
90
- return encoder, trainer
91
-
92
-
93
- __all__ = ["create_bpe_tokenizer", "MoveTokenizer", "PostionTokenizer"]
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
chesstransformer/models/tokenizer/alphazero_move_encoder.py DELETED
@@ -1,127 +0,0 @@
1
- """AlphaZero-style action encoding for chess moves.
2
-
3
- Encodes moves as (from_square, action_plane) pairs using the 8×8×73
4
- representation from the AlphaZero paper (Silver et al., 2018).
5
-
6
- The 73 action planes decompose as:
7
- 0–55: Queen-type moves — 8 directions × 7 distances
8
- 56–63: Knight moves — 8 L-shaped jumps
9
- 64–72: Underpromotions — 3 directions × 3 piece types (knight/bishop/rook)
10
-
11
- Queen promotions are encoded as regular queen-type moves (distance-1 to the
12
- last rank). Underpromotions get their own dedicated planes.
13
- """
14
-
15
- import chess
16
- import torch
17
-
18
- NUM_ACTION_PLANES = 73
19
-
20
- # Direction vectors (file_delta, rank_delta) for queen-type moves
21
- QUEEN_DIRECTIONS = [
22
- (0, 1), # N
23
- (1, 1), # NE
24
- (1, 0), # E
25
- (1, -1), # SE
26
- (0, -1), # S
27
- (-1, -1), # SW
28
- (-1, 0), # W
29
- (-1, 1), # NW
30
- ]
31
-
32
- # Knight jump offsets (file_delta, rank_delta)
33
- KNIGHT_JUMPS = [
34
- (1, 2), (2, 1), (2, -1), (1, -2),
35
- (-1, -2), (-2, -1), (-2, 1), (-1, 2),
36
- ]
37
-
38
- # Underpromotion piece types (queen promotion is a regular queen-type move)
39
- UNDERPROMO_PIECES = [chess.KNIGHT, chess.BISHOP, chess.ROOK]
40
-
41
- # Pre-compute knight jump lookup for O(1) detection
42
- _KNIGHT_JUMP_INDEX = {(df, dr): i for i, (df, dr) in enumerate(KNIGHT_JUMPS)}
43
-
44
-
45
- def move_to_action_plane(from_sq: int, to_sq: int, promotion: int | None = None) -> int:
46
- """Convert a chess move to its action plane index (0–72).
47
-
48
- Args:
49
- from_sq: Source square (0–63, python-chess ordering).
50
- to_sq: Destination square (0–63).
51
- promotion: Promotion piece type (chess.QUEEN, chess.KNIGHT, etc.) or None.
52
-
53
- Returns:
54
- Action plane index in [0, 72].
55
- """
56
- df = (to_sq % 8) - (from_sq % 8)
57
- dr = (to_sq // 8) - (from_sq // 8)
58
-
59
- # ── Underpromotion ───────────────────────────────────────────────────
60
- if promotion is not None and promotion != chess.QUEEN:
61
- # Direction: file delta ∈ {-1, 0, +1} → index 0, 1, 2
62
- dir_idx = df + 1
63
- piece_idx = UNDERPROMO_PIECES.index(promotion)
64
- return 64 + dir_idx * 3 + piece_idx
65
-
66
- # ── Knight move ──────────────────────────────────────────────────────
67
- knight_idx = _KNIGHT_JUMP_INDEX.get((df, dr))
68
- if knight_idx is not None:
69
- return 56 + knight_idx
70
-
71
- # ── Queen-type move (includes queen promotions) ──────────────────────
72
- distance = max(abs(df), abs(dr))
73
- dir_f = (df // distance) if df != 0 else 0
74
- dir_r = (dr // distance) if dr != 0 else 0
75
- dir_idx = QUEEN_DIRECTIONS.index((dir_f, dir_r))
76
- return dir_idx * 7 + (distance - 1)
77
-
78
-
79
- def action_to_move(from_sq: int, plane: int) -> tuple[int, int | None]:
80
- """Decode an action plane back to (to_square, promotion_piece | None).
81
-
82
- For queen-type moves landing on promotion ranks, the caller must check
83
- whether the source piece is a pawn and add ``chess.QUEEN`` promotion.
84
-
85
- Args:
86
- from_sq: Source square (0–63).
87
- plane: Action plane index (0–72).
88
-
89
- Returns:
90
- (to_sq, promotion_piece) — promotion_piece is set only for underpromotions.
91
- """
92
- from_file = from_sq % 8
93
- from_rank = from_sq // 8
94
-
95
- if plane < 56:
96
- # Queen-type move
97
- dir_idx = plane // 7
98
- distance = plane % 7 + 1
99
- df, dr = QUEEN_DIRECTIONS[dir_idx]
100
- to_file = from_file + df * distance
101
- to_rank = from_rank + dr * distance
102
- return to_rank * 8 + to_file, None
103
-
104
- if plane < 64:
105
- # Knight move
106
- df, dr = KNIGHT_JUMPS[plane - 56]
107
- to_file = from_file + df
108
- to_rank = from_rank + dr
109
- return to_rank * 8 + to_file, None
110
-
111
- # Underpromotion
112
- promo_idx = plane - 64
113
- dir_idx = promo_idx // 3
114
- piece_idx = promo_idx % 3
115
- df = dir_idx - 1 # 0→-1, 1→0, 2→+1
116
- to_file = from_file + df
117
- to_rank = 7 if from_rank == 6 else 0 # white promotes to rank 8, black to rank 1
118
- return to_rank * 8 + to_file, UNDERPROMO_PIECES[piece_idx]
119
-
120
-
121
- def build_legal_moves_planes(board: chess.Board) -> torch.Tensor:
122
- """Build a (64, 73) boolean mask of legal moves in action-plane format."""
123
- mask = torch.zeros(64, NUM_ACTION_PLANES, dtype=torch.bool)
124
- for move in board.legal_moves:
125
- plane = move_to_action_plane(move.from_square, move.to_square, move.promotion)
126
- mask[move.from_square, plane] = True
127
- return mask
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
chesstransformer/models/tokenizer/move_tokenizer.py DELETED
@@ -1,38 +0,0 @@
1
- import json
2
- from pathlib import Path
3
-
4
- DATA_DIR = Path(__file__).parents[2] / "data"
5
-
6
-
7
- class MoveTokenizer:
8
- def __init__(self, vocab_path=DATA_DIR / "tokenizer_models/chess_moves_vocab.json"):
9
- with open(vocab_path, "r") as f:
10
- self.vocab = json.load(f)
11
- self.inv_vocab = {v: k for k, v in self.vocab.items()}
12
-
13
- def encode(self, move: str) -> int:
14
- """Encode a chess move string into a token ID.
15
- Args:
16
- move: A string representing a chess move in UCI format (e.g., "e2e4", "g1f3").
17
- Returns:
18
- An integer token ID representing the move.
19
- """
20
- if move not in self.vocab:
21
- raise ValueError(f"Move '{move}' not in vocabulary.")
22
- return self.vocab[move]
23
-
24
- def decode(self, token_id: int) -> str:
25
- """Decode a token ID back into a chess move string.
26
- Args:
27
- token_id: An integer token ID representing the move.
28
- Returns:
29
- A string representing a chess move in UCI format.
30
- """
31
- if token_id not in self.inv_vocab:
32
- raise ValueError(f"Token ID '{token_id}' not in vocabulary.")
33
- return self.inv_vocab[token_id]
34
-
35
- @property
36
- def vocab_size(self) -> int:
37
- """Return the size of the vocabulary."""
38
- return len(self.vocab)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
chesstransformer/models/tokenizer/position_tokenizer.py DELETED
@@ -1,88 +0,0 @@
1
- import chess
2
-
3
- _PIECE_TOKEN = {
4
- (chess.PAWN, chess.WHITE): 1, (chess.KNIGHT, chess.WHITE): 2,
5
- (chess.BISHOP, chess.WHITE): 3, (chess.ROOK, chess.WHITE): 4,
6
- (chess.QUEEN, chess.WHITE): 5, (chess.KING, chess.WHITE): 6,
7
- (chess.PAWN, chess.BLACK): 7, (chess.KNIGHT, chess.BLACK): 8,
8
- (chess.BISHOP, chess.BLACK): 9, (chess.ROOK, chess.BLACK): 10,
9
- (chess.QUEEN, chess.BLACK): 11, (chess.KING, chess.BLACK): 12,
10
- }
11
-
12
-
13
- class PostionTokenizer:
14
- """A simple tokenizer for chess board positions.
15
- Each square on the board is represented by a token ID based on the piece occupying it.
16
- Empty squares are represented by the token ID 0.
17
- The board is represented as an 8x8 grid, flattened into a list of 64 tokens.
18
- """
19
-
20
- def __init__(self):
21
- self.vocab = {
22
- "P": 1,
23
- "N": 2,
24
- "B": 3,
25
- "R": 4,
26
- "Q": 5,
27
- "K": 6,
28
- "p": 7,
29
- "n": 8,
30
- "b": 9,
31
- "r": 10,
32
- "q": 11,
33
- "k": 12,
34
- ".": 0,
35
- }
36
- self.inv_vocab = {v: k for k, v in self.vocab.items()}
37
-
38
- def encode(self, board: chess.Board) -> list[int]:
39
- """Encode a chess.Board object into a list of token IDs.
40
- The board is represented as an 8x8 grid, flattened into a list of 64 tokens.
41
-
42
- Args:
43
- board: A chess.Board object
44
-
45
- Returns:
46
- List of token IDs representing the board position
47
- """
48
- result = [0] * 64
49
- for sq, piece in board.piece_map().items():
50
- result[sq] = _PIECE_TOKEN[(piece.piece_type, piece.color)]
51
- return result
52
-
53
- def decode(self, token_ids: list[int]) -> chess.Board:
54
- """Decode a list of token IDs back into a chess board string.
55
- Args:
56
- token_ids: List of token IDs representing the board position
57
- Returns:
58
- A chess.Board object
59
- """
60
- chars = [self.inv_vocab[token_id] for token_id in token_ids]
61
- rows = ["".join(chars[i * 8 : (i + 1) * 8]) for i in range(8)][::-1] # Reverse to get the correct order
62
- board_str = "\n".join(rows)
63
- board = self._ascii2board(board_str)
64
- return board
65
-
66
- def _ascii2board(self, ascii_board: str) -> chess.Board:
67
- """Convert an ASCII board representation back to a chess.Board object.
68
- Args:
69
- ascii_board: String representation of the board (8 lines of 8 characters)
70
- Returns:
71
- A chess.Board object
72
- """
73
- board = chess.Board.empty()
74
- rows = ascii_board.split("\n")
75
- for rank in range(8):
76
- file = 0
77
- for char in rows[7 - rank]: # Reverse the order of ranks
78
- if char in self.vocab and char != ".":
79
- square = chess.square(file, rank)
80
- piece = chess.Piece.from_symbol(char)
81
- board.set_piece_at(square, piece)
82
- file += 1
83
- return board
84
-
85
- @property
86
- def vocab_size(self) -> int:
87
- """Return the size of the vocabulary."""
88
- return len(self.vocab)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
chesstransformer/models/tokenizer/train_bpe_tokenizer.py DELETED
@@ -1,94 +0,0 @@
1
- from pathlib import Path
2
- import argparse
3
-
4
- from chesstransformer.models.tokenizer import create_bpe_tokenizer
5
-
6
-
7
- def arg_parser():
8
- parser = argparse.ArgumentParser(description="Train a BPE tokenizer for chess moves.")
9
- parser.add_argument(
10
- "--vocab_size",
11
- type=int,
12
- default=2000,
13
- help="Size of the vocabulary (default: 2000)",
14
- )
15
- parser.add_argument(
16
- "--output_dir",
17
- type=str,
18
- help="Directory to save the trained tokenizer",
19
- required=True,
20
- )
21
- parser.add_argument(
22
- "--dataset_path",
23
- type=str,
24
- help="Path to the lichess dataset file (PGN format)",
25
- required=True,
26
- )
27
-
28
- parser.add_argument(
29
- "--num_samples",
30
- type=int,
31
- default=100000,
32
- help="Number of games to sample for training (default: 100000)",
33
- )
34
-
35
- parser.add_argument(
36
- "--notation",
37
- type=str,
38
- choices=["uci", "san"],
39
- default="uci",
40
- help="Notation format for moves (default: uci)",
41
- )
42
-
43
- args = parser.parse_args()
44
- return args
45
-
46
-
47
- def main():
48
- args = arg_parser()
49
- vocab_size = args.vocab_size
50
- output_dir = Path(args.output_dir)
51
- dataset_path = Path(args.dataset_path)
52
- num_samples = args.num_samples
53
- notation = args.notation
54
-
55
- output_dir.mkdir(parents=True, exist_ok=True)
56
-
57
- print(f"Creating BPE tokenizer with vocab size {vocab_size}...")
58
- tokenizer, trainer = create_bpe_tokenizer(vocab_size=vocab_size)
59
-
60
- print(f"Loading dataset from {dataset_path}...")
61
- from chesstransformer.datasets.lichess import extract_games_optimized
62
-
63
- games = extract_games_optimized(dataset_path, num_games=num_samples)
64
-
65
- print(f"Extracting moves in {notation} notation...")
66
- print("Using <STEP> token to separate moves...")
67
- if notation == "uci":
68
- move_sequences = ("<STEP>".join([move.uci() for move in game.mainline_moves()]) for game in games)
69
- elif notation == "san":
70
- move_sequences = (
71
- "<STEP>".join(
72
- [
73
- (lambda board, move: (board.san(move), board.push(move))[0])(board, move)
74
- for board in [game.board()]
75
- for move in game.mainline_moves()
76
- ]
77
- )
78
- for game in games
79
- )
80
- else:
81
- raise ValueError(f"Unsupported notation: {notation}")
82
-
83
- print(f"Training tokenizer on {num_samples} games...")
84
- tokenizer.train_from_iterator(move_sequences, trainer=trainer)
85
-
86
- tokenizer_path = output_dir / f"bpe_tokenizer_vocab{vocab_size}.json"
87
- print(f"Saving tokenizer to {tokenizer_path}...")
88
- tokenizer.save(str(tokenizer_path))
89
-
90
- print("Tokenizer training complete.")
91
-
92
-
93
- if __name__ == "__main__":
94
- main()
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
chesstransformer/models/transformer/__init__.py DELETED
File without changes
chesstransformer/models/transformer/attention.py DELETED
@@ -1,228 +0,0 @@
1
- """This module implements attention mechanisms for LLMs."""
2
-
3
- import torch
4
- import torch.nn as nn
5
- import torch.nn.functional as F
6
-
7
- from chesstransformer.models.transformer.rope import RoPEEmbedding
8
-
9
-
10
- class MultiHeadAttention(nn.Module):
11
- def __init__(
12
- self, d_in, d_out, num_heads, context_length, qkv_bias=False, dropout=0.0, mask_future=True, apply_rope=False
13
- ):
14
- super().__init__()
15
- assert d_out % num_heads == 0, "d_out must be divisible by num_heads"
16
- self.num_heads = num_heads
17
- self.d_out = d_out
18
- self.d_head = d_out // num_heads
19
- self.mask_future = mask_future
20
- self.W_query = nn.Linear(d_in, d_out, bias=qkv_bias)
21
- self.W_key = nn.Linear(d_in, d_out, bias=qkv_bias)
22
- self.W_value = nn.Linear(d_in, d_out, bias=qkv_bias)
23
- self.out_proj = nn.Linear(d_out, d_out)
24
- self.dropout = nn.Dropout(dropout)
25
- self.apply_rope = apply_rope
26
-
27
- if self.apply_rope:
28
- self.rope = RoPEEmbedding(dim=self.d_head, max_position_embeddings=context_length)
29
-
30
- self.register_buffer("mask", torch.triu(torch.ones(context_length, context_length), diagonal=1))
31
-
32
- def forward(self, x):
33
- """
34
- Computes multi-head self-attention.
35
-
36
- Args:
37
- x: Input tensor of shape (batch_size, seq_len, d_in)
38
-
39
- Returns:
40
- Output tensor of shape (batch_size, seq_len, d_out)
41
- """
42
- bz, nb_tockens, _ = x.shape
43
-
44
- # Linear projections
45
- keys = self.W_key(x)
46
- values = self.W_value(x)
47
- queries = self.W_query(x)
48
-
49
- # Reshape for multi-head attention
50
- keys = keys.view(bz, nb_tockens, self.num_heads, self.d_head).transpose(1, 2)
51
- values = values.view(bz, nb_tockens, self.num_heads, self.d_head).transpose(1, 2)
52
- queries = queries.view(bz, nb_tockens, self.num_heads, self.d_head).transpose(1, 2)
53
-
54
- if self.apply_rope:
55
- keys = self.rope(keys)
56
- queries = self.rope(values)
57
-
58
- # Compute attention scores
59
- attention_scores = queries @ keys.transpose(2, 3) # (batch_size, num_heads, seq_len, seq_len)
60
-
61
- if self.mask_future:
62
- # Apply mask
63
- mask_bool = self.mask.bool()[:nb_tockens, :nb_tockens]
64
- attention_scores = attention_scores.masked_fill(mask_bool, -torch.inf)
65
-
66
- # Compute attention weights
67
- attention_weights = F.softmax(attention_scores / keys.shape[-1] ** 0.5, dim=-1)
68
- attention_weights = self.dropout(attention_weights)
69
-
70
- # apply attention weights to values
71
- context_vec = (attention_weights @ values).transpose(1, 2).contiguous()
72
-
73
- # Merge heads
74
- context_vec = context_vec.view(bz, nb_tockens, self.d_out)
75
- context_vec = self.out_proj(context_vec)
76
- return context_vec
77
-
78
-
79
- class PyTorchMultiHeadAttention(nn.Module):
80
- def __init__(self, d_in, d_out, num_heads, dropout=0.0, qkv_bias=False, mask_future=True, apply_rope=False):
81
- super().__init__()
82
-
83
- assert d_out % num_heads == 0, "d_out is indivisible by num_heads"
84
-
85
- self.num_heads = num_heads
86
- self.head_dim = d_out // num_heads
87
- self.d_out = d_out
88
- self.mask_future = mask_future
89
- self.apply_rope = apply_rope
90
-
91
- self.qkv = nn.Linear(d_in, 3 * d_out, bias=qkv_bias)
92
- self.proj = nn.Linear(d_out, d_out)
93
- self.dropout = dropout
94
- self.rope = None # Placeholder for RoPE, if needed
95
- if self.apply_rope:
96
- self.rope = RoPEEmbedding(dim=self.head_dim)
97
-
98
- def forward(self, x):
99
- batch_size, num_tokens, embed_dim = x.shape
100
-
101
- # (b, num_tokens, embed_dim) --> (b, num_tokens, 3 * embed_dim)
102
- qkv = self.qkv(x)
103
-
104
- # (b, num_tokens, 3 * embed_dim) --> (b, num_tokens, 3, num_heads, head_dim)
105
- qkv = qkv.view(batch_size, num_tokens, 3, self.num_heads, self.head_dim)
106
-
107
- # (b, num_tokens, 3, num_heads, head_dim) --> (3, b, num_heads, num_tokens, head_dim)
108
- qkv = qkv.permute(2, 0, 3, 1, 4)
109
-
110
- # (3, b, num_heads, num_tokens, head_dim) -> 3 times (b, num_heads, num_tokens, head_dim)
111
- queries, keys, values = qkv
112
-
113
- if self.apply_rope and self.rope is not None:
114
- keys = self.rope(keys)
115
- queries = self.rope(queries)
116
-
117
- use_dropout = 0.0 if not self.training else self.dropout
118
-
119
- context_vec = nn.functional.scaled_dot_product_attention(
120
- queries, keys, values, attn_mask=None, dropout_p=use_dropout, is_causal=self.mask_future
121
- )
122
-
123
- # Combine heads, where self.d_out = self.num_heads * self.head_dim
124
- context_vec = context_vec.transpose(1, 2).contiguous().view(batch_size, num_tokens, self.d_out)
125
-
126
- context_vec = self.proj(context_vec)
127
-
128
- return context_vec
129
-
130
-
131
- class GroupedQueryAttention(nn.Module):
132
- """
133
- Multi-head self-attention with grouped keys and values.
134
- This implementation allows multiple attention heads to share the same keys and values,
135
- reducing the number of parameters and computational cost.
136
-
137
- It works by dividing the attention heads into groups, where each group shares a set of keys and values.
138
-
139
- Example:
140
- If there are 8 attention heads and 4 key-value groups, then each group will be shared by 2 attention heads.
141
-
142
- Args:
143
- d_in: Input feature dimension
144
- d_out: Output feature dimension
145
- num_heads: Number of attention heads
146
- num_kv_groups: Number of key-value groups (must divide num_heads)
147
- dropout: Dropout probability
148
- qkv_bias: Whether to include bias terms in the linear projections
149
- mask_future: Whether to apply causal masking to prevent attention to future tokens
150
- apply_rope: Whether to apply Rotary Positional Embeddings (RoPE)
151
- Returns:
152
- Output tensor of shape (batch_size, seq_len, d_out)
153
- """
154
-
155
- def __init__(
156
- self, d_in, d_out, num_heads, num_kv_groups, dropout=0.0, qkv_bias=False, mask_future=True, apply_rope=False
157
- ):
158
- super().__init__()
159
-
160
- assert d_out % num_heads == 0, "d_out is indivisible by num_heads"
161
- assert num_heads % num_kv_groups == 0, "num_heads is indivisible by num_kv_groups"
162
-
163
- self.num_heads = num_heads
164
- self.num_kv_groups = num_kv_groups
165
- self.head_dim = d_out // num_heads
166
- self.d_out = d_out
167
-
168
- self.q_proj = nn.Linear(d_in, d_out, bias=qkv_bias)
169
- self.k_proj = nn.Linear(d_in, num_kv_groups * self.head_dim, bias=qkv_bias)
170
- self.v_proj = nn.Linear(d_in, num_kv_groups * self.head_dim, bias=qkv_bias)
171
- self.group_size = num_heads // num_kv_groups
172
- self.num_kv_groups = num_kv_groups
173
-
174
- self.proj = nn.Linear(d_out, d_out, bias=False)
175
- self.dropout = nn.Dropout(dropout)
176
-
177
- self.mask_future = mask_future
178
- self.apply_rope = apply_rope
179
- if self.apply_rope:
180
- self.rope = RoPEEmbedding(dim=self.head_dim)
181
-
182
- def forward(self, x):
183
- batch_size, num_tokens, embed_dim = x.shape
184
-
185
- queries = self.q_proj(x)
186
- keys = self.k_proj(x)
187
- values = self.v_proj(x)
188
-
189
- queries = queries.view(batch_size, num_tokens, self.num_heads, self.head_dim).transpose(1, 2)
190
- keys_new = keys.view(batch_size, num_tokens, self.num_kv_groups, self.head_dim).transpose(1, 2)
191
- values_new = values.view(batch_size, num_tokens, self.num_kv_groups, self.head_dim).transpose(1, 2)
192
-
193
- keys = keys_new.repeat_interleave(self.group_size, dim=1)
194
- values = values_new.repeat_interleave(self.group_size, dim=1)
195
-
196
- if self.apply_rope:
197
- keys = self.rope(keys)
198
- queries = self.rope(queries)
199
-
200
- use_dropout = 0.0 if not self.training else self.dropout.p
201
-
202
- context_vec = F.scaled_dot_product_attention(
203
- queries, keys, values, attn_mask=None, dropout_p=use_dropout, is_causal=self.mask_future
204
- )
205
-
206
- context_vec = context_vec.transpose(1, 2).contiguous()
207
- context_vec = context_vec.view(batch_size, num_tokens, self.d_out)
208
- context_vec = self.proj(context_vec)
209
-
210
- return context_vec
211
-
212
-
213
- if __name__ == "__main__":
214
- # Simple test
215
- batch_size = 2
216
- seq_len = 16
217
- d_in = 64
218
- d_out = 64
219
- num_heads = 8
220
- num_kv_groups = 4
221
-
222
- x = torch.randn(batch_size, seq_len, d_in)
223
-
224
- attn = GroupedQueryAttention(d_in, d_out, num_heads, num_kv_groups, apply_rope=True)
225
- output = attn(x)
226
-
227
- print("Input shape:", x.shape)
228
- print("Output shape:", output.shape)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
chesstransformer/models/transformer/gpt.py DELETED
@@ -1,79 +0,0 @@
1
- """This module implements GPT model architecture."""
2
-
3
- import torch
4
- import torch.nn as nn
5
- from ..transformer.attention import MultiHeadAttention
6
- from ..transformer.utils import LayerNorm, GELU
7
-
8
-
9
- class FeedForward(nn.Module):
10
- def __init__(self, cfg):
11
- super().__init__()
12
- self.layers = nn.Sequential(
13
- nn.Linear(cfg["embed_dim"], 4 * cfg["embed_dim"]),
14
- GELU(),
15
- nn.Linear(4 * cfg["embed_dim"], cfg["embed_dim"]),
16
- )
17
-
18
- def forward(self, x):
19
- return self.layers(x)
20
-
21
-
22
- class TransformerBlock(nn.Module):
23
- def __init__(self, cfg):
24
- super().__init__()
25
- self.att = MultiHeadAttention(
26
- d_in=cfg["embed_dim"],
27
- d_out=cfg["embed_dim"],
28
- num_heads=cfg["num_heads"],
29
- context_length=cfg["context_size"],
30
- dropout=cfg["dropout"],
31
- qkv_bias=cfg["qkv_bias"],
32
- )
33
- self.feed_forward = FeedForward(cfg)
34
- self.norm1 = LayerNorm(cfg["embed_dim"])
35
- self.norm2 = LayerNorm(cfg["embed_dim"])
36
- self.dropout_shortcut = nn.Dropout(cfg["dropout"])
37
-
38
- def forward(self, x):
39
- shortcut = x
40
- x = self.norm1(x)
41
- x = self.att(x)
42
- x = self.dropout_shortcut(x)
43
- x = x + shortcut
44
-
45
- shortcut = x
46
- x = self.norm2(x)
47
- x = self.feed_forward(x)
48
- x = self.dropout_shortcut(x)
49
- x = x + shortcut
50
-
51
- return x
52
-
53
-
54
- class GPT2Model(nn.Module):
55
- def __init__(self, cfg):
56
- super().__init__()
57
- self.token_embedding = nn.Embedding(cfg["vocab_size"], cfg["embed_dim"])
58
- self.position_embedding = nn.Embedding(cfg["context_size"], cfg["embed_dim"])
59
- self.dropout_embed = nn.Dropout(cfg["dropout"])
60
- self.transfomer_blocks = nn.Sequential(*[TransformerBlock(cfg) for _ in range(cfg["num_layers"])])
61
- self.final_norm = LayerNorm(cfg["embed_dim"])
62
- self.lm_head = nn.Linear(cfg["embed_dim"], cfg["vocab_size"], bias=False)
63
-
64
- def forward(self, x):
65
- bz, sq_len = x.size()
66
-
67
- token_emb = self.token_embedding(x) # (bz, sq_len, embed_dim)
68
-
69
- positions = torch.arange(sq_len, device=x.device)
70
- pos_emb = self.position_embedding(positions) # (sq_len, embed_dim)
71
-
72
- x = token_emb + pos_emb # (bz, sq_len, embed_dim)
73
- x = self.dropout_embed(x)
74
-
75
- x = self.transfomer_blocks(x)
76
- x = self.final_norm(x)
77
-
78
- logits = self.lm_head(x) # (bz, sq_len, vocab_size)
79
- return logits
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
chesstransformer/models/transformer/pos2move_v2.py DELETED
@@ -1,311 +0,0 @@
1
- import torch
2
- import torch.nn as nn
3
- import torch.nn.functional as F
4
-
5
- NUM_ACTION_PLANES = 73
6
- BOARD_SQUARES = 64
7
- EXTRA_TOKENS = 3 # castling, en_passant, player
8
- CONTEXT_SIZE = BOARD_SQUARES + EXTRA_TOKENS # 67
9
-
10
-
11
- def build_chess_relation_index(context_size: int = CONTEXT_SIZE) -> torch.Tensor:
12
- """Precompute (T, T) relation index between every pair of tokens.
13
-
14
- Categories (8 total):
15
- 0: same square / same token
16
- 1: same file
17
- 2: same rank
18
- 3: same diagonal
19
- 4: knight-reachable
20
- 5: king-adjacent (Chebyshev <= 1)
21
- 6: nearby (Manhattan <= 3)
22
- 7: far
23
- Game-state tokens (>= 64) get category 0 toward themselves and
24
- a dedicated "global" category 7 toward all board squares.
25
- """
26
- rel = torch.zeros(context_size, context_size, dtype=torch.long)
27
- for i in range(context_size):
28
- for j in range(context_size):
29
- if i == j:
30
- rel[i, j] = 0
31
- continue
32
- if i >= BOARD_SQUARES or j >= BOARD_SQUARES:
33
- # Any pair involving a game-state token: "global" bucket
34
- rel[i, j] = 7
35
- continue
36
- fi, ri = i % 8, i // 8
37
- fj, rj = j % 8, j // 8
38
- df, dr = abs(fi - fj), abs(ri - rj)
39
- if df == 0:
40
- rel[i, j] = 1
41
- elif dr == 0:
42
- rel[i, j] = 2
43
- elif df == dr:
44
- rel[i, j] = 3
45
- elif {df, dr} == {1, 2}:
46
- rel[i, j] = 4
47
- elif df <= 1 and dr <= 1:
48
- rel[i, j] = 5
49
- elif df + dr <= 3:
50
- rel[i, j] = 6
51
- else:
52
- rel[i, j] = 7
53
- return rel
54
-
55
-
56
- _RELATION_INDEX: torch.Tensor = build_chess_relation_index()
57
-
58
-
59
- class ReluSquared(nn.Module):
60
- def forward(self, x):
61
- return F.relu(x) ** 2
62
-
63
- class ChessGroupedQueryAttention(nn.Module):
64
- def __init__(
65
- self, d_in, d_out, num_heads, num_kv_groups, context_size, dropout=0.0, qkv_bias=False
66
- ):
67
- super().__init__()
68
-
69
- assert d_out % num_heads == 0, "d_out is indivisible by num_heads"
70
- assert num_heads % num_kv_groups == 0, "num_heads is indivisible by num_kv_groups"
71
-
72
- self.num_heads = num_heads
73
- self.num_kv_groups = num_kv_groups
74
- self.head_dim = d_out // num_heads
75
- self.d_out = d_out
76
-
77
- self.q_proj = nn.Linear(d_in, d_out, bias=qkv_bias)
78
- self.k_proj = nn.Linear(d_in, num_kv_groups * self.head_dim, bias=qkv_bias)
79
- self.v_proj = nn.Linear(d_in, num_kv_groups * self.head_dim, bias=qkv_bias)
80
- self.group_size = num_heads // num_kv_groups
81
-
82
- self.proj = nn.Linear(d_out, d_out, bias=False)
83
- self.dropout = dropout
84
-
85
- self.q_norm = nn.RMSNorm(self.head_dim)
86
- self.k_norm = nn.RMSNorm(self.head_dim)
87
-
88
- # Chess-geometry relative bias: 8 relation categories per head
89
- # Massively fewer params than fully-learned (H, T, T) bias.
90
- self.num_relations = 8
91
- self.bias_table = nn.Parameter(torch.zeros(num_heads, self.num_relations))
92
- self.register_buffer("rel_idx", _RELATION_INDEX, persistent=False)
93
-
94
-
95
- def forward(self, x):
96
- batch_size, num_tokens, embed_dim = x.shape
97
-
98
- queries = self.q_proj(x)
99
- keys = self.k_proj(x)
100
- values = self.v_proj(x)
101
-
102
- queries = queries.view(batch_size, num_tokens, self.num_heads, self.head_dim).transpose(1, 2)
103
- keys_new = keys.view(batch_size, num_tokens, self.num_kv_groups, self.head_dim).transpose(1, 2)
104
- values_new = values.view(batch_size, num_tokens, self.num_kv_groups, self.head_dim).transpose(1, 2)
105
-
106
- queries = self.q_norm(queries)
107
- keys_new = self.k_norm(keys_new)
108
-
109
- # Expand K/V to full head count. At this tiny sequence length (67) the
110
- # repeat_interleave is cheaper than SDPA's native enable_gqa path, which
111
- # falls back to a slower kernel here (measured ~20% slower).
112
- keys = keys_new.repeat_interleave(self.group_size, dim=1)
113
- values = values_new.repeat_interleave(self.group_size, dim=1)
114
-
115
- # Build additive chess-geometry bias from the compact (H, 8) table.
116
- # Shape: (1, H, T, T) so SDPA broadcasts it across the batch.
117
- attn_bias = self.bias_table[:, self.rel_idx].unsqueeze(0)
118
-
119
- context_vec = F.scaled_dot_product_attention(
120
- queries, keys, values,
121
- attn_mask=attn_bias,
122
- dropout_p=self.dropout if self.training else 0.0,
123
- is_causal=False,
124
- )
125
-
126
- context_vec = context_vec.transpose(1, 2).contiguous()
127
- context_vec = context_vec.view(batch_size, num_tokens, self.d_out)
128
- context_vec = self.proj(context_vec)
129
-
130
- return context_vec
131
-
132
- class MLP(nn.Module):
133
- def __init__(self, d_in, d_hidden, d_out, dropout=0.0):
134
- super().__init__()
135
- self.fc1 = nn.Linear(d_in, d_hidden)
136
- self.act = ReluSquared()
137
- self.fc2 = nn.Linear(d_hidden, d_out)
138
- self.dropout = nn.Dropout(dropout)
139
-
140
- def forward(self, x):
141
- x = self.fc1(x)
142
- x = self.act(x)
143
- x = self.dropout(x)
144
- x = self.fc2(x)
145
- return x
146
-
147
- class ChessTransformerLayer(nn.Module):
148
- def __init__(self, dim, nb_heads, qkv_bias, context_size, dropout=0.1, layer_drop=0.0):
149
- super().__init__()
150
- self.attn = ChessGroupedQueryAttention(
151
- d_in=dim,
152
- d_out=dim,
153
- num_heads=nb_heads,
154
- num_kv_groups=nb_heads // 2,
155
- context_size=context_size,
156
- dropout=dropout,
157
- qkv_bias=qkv_bias,
158
- )
159
- self.mlp = MLP(dim, dim * 4, dim, dropout=dropout)
160
- self.norm1 = nn.RMSNorm(dim)
161
- self.norm2 = nn.RMSNorm(dim)
162
- self.layer_drop = layer_drop
163
-
164
- def forward(self, x):
165
- # Stochastic depth: with prob `layer_drop`, skip the entire layer.
166
- # Applied independently to attn and mlp branches (sub-layer LayerDrop).
167
- if self.training and self.layer_drop > 0.0:
168
- # Per-sample keep mask would be more standard, but per-layer scalar matches your old behavior.
169
- keep_attn = torch.empty(1, device=x.device, dtype=x.dtype).bernoulli_(1.0 - self.layer_drop)
170
- keep_mlp = torch.empty(1, device=x.device, dtype=x.dtype).bernoulli_(1.0 - self.layer_drop)
171
- x = x + keep_attn * self.attn(self.norm1(x))
172
- x = x + keep_mlp * self.mlp(self.norm2(x))
173
- else:
174
- x = x + self.attn(self.norm1(x))
175
- x = x + self.mlp(self.norm2(x))
176
- return x
177
-
178
- class Pos2MoveV2(nn.Module):
179
- def __init__(
180
- self,
181
- vocab_size: int = 13,
182
- embed_dim: int = 512,
183
- nb_transformer_layers: int = 2,
184
- num_heads: int = 8,
185
- dropout: float = 0.1,
186
- kvq_bias: bool = False,
187
- layer_drop: float = 0.0,
188
- value_bounded: bool = False,
189
- value_clamp: bool = False,
190
- ):
191
- super().__init__()
192
-
193
- self.token_embedding = nn.Embedding(vocab_size, embed_dim)
194
- self.position_embedding = nn.Embedding(67, embed_dim) # 64 squares + 3 extra tokens
195
- # Static position indices, kept as a buffer so we don't rebuild the
196
- # arange (and a host->device copy) on every forward pass.
197
- self.register_buffer("pos_index", torch.arange(67), persistent=False)
198
- self.castling_embedding = nn.Embedding(16, embed_dim) # 4-bit bitmask: 0-15
199
- self.en_passant_embedding = nn.Embedding(9, embed_dim) # 8 files + no en passant (8)
200
- self.player_embedding = nn.Embedding(2, embed_dim) # White or Black
201
-
202
- # Linearly increasing layer drop rate: deeper layers drop more.
203
- # This is the "Deep Networks with Stochastic Depth" schedule.
204
- drop_rates = [layer_drop * i / max(1, nb_transformer_layers - 1)
205
- for i in range(nb_transformer_layers)]
206
-
207
- self.transformer_layers = nn.ModuleList(
208
- [
209
- ChessTransformerLayer(
210
- dim=embed_dim,
211
- nb_heads=num_heads,
212
- qkv_bias=kvq_bias,
213
- context_size=67, # Max sequence length (64 squares + 3 extra tokens)
214
- dropout=dropout,
215
- layer_drop=drop_rates[i],
216
- )
217
- for i in range(nb_transformer_layers)
218
- ]
219
- )
220
-
221
- self.final_norm = nn.RMSNorm(embed_dim)
222
-
223
- policy_hidden = embed_dim // 2
224
- self.move_head = nn.Sequential(
225
- nn.Linear(embed_dim, policy_hidden),
226
- nn.GELU(),
227
- nn.Linear(policy_hidden, NUM_ACTION_PLANES),
228
- )
229
-
230
- # Deeper value head: D → D//4 → GELU → 1
231
- # value_bounded appends a tanh so the output cannot leave the target's
232
- # [-1, 1] range. Kept off by default: the shipped v2.1 weights were
233
- # trained without it, and squashing them post-hoc changes their scale.
234
- # The Tanh lands at index 3, so the Linear state-dict keys (0.*, 2.*)
235
- # are identical either way and checkpoints stay interchangeable.
236
- value_hidden = embed_dim // 4
237
- value_layers = [
238
- nn.Linear(embed_dim, value_hidden),
239
- nn.GELU(),
240
- nn.Linear(value_hidden, 1),
241
- ]
242
- if value_bounded:
243
- value_layers.append(nn.Tanh())
244
- elif value_clamp:
245
- # Minimal alternative to tanh for weights already trained unbounded:
246
- # a hard clamp leaves every in-range output untouched and only edits
247
- # the semantically impossible ones (a leaf "better than won"). v2.1
248
- # emits those on 0.76% of positions, reaching 1.039 -- and since MCTS
249
- # maximises over leaf values, that tail is precisely what it chases.
250
- # MEASURED 2026-08-19: on v2.1 this is a NO-OP -- the chosen move was
251
- # identical on 25/25 opening positions at both 4000 and 8000 sims. A
252
- # 4% excursion is too small to steer the search; it mattered for the
253
- # cp-trained head only because that one reached 1.211. Don't re-run.
254
- # Kept for unbounded heads that overshoot more; prefer value_bounded
255
- # (tanh) when training a head from scratch.
256
- value_layers.append(nn.Hardtanh(-1.0, 1.0))
257
- self.value_head = nn.Sequential(*value_layers)
258
-
259
- self._init_weights(nb_transformer_layers)
260
-
261
- def _init_weights(self, nb_layers: int):
262
- """GPT-2 style init with depth-scaled residual projections.
263
-
264
- - All Linear/Embedding: N(0, 0.02)
265
- - Residual projections (attn out + mlp out): scaled by 1/sqrt(2*N)
266
- to keep residual stream variance bounded with depth.
267
- """
268
- for module in self.modules():
269
- if isinstance(module, nn.Linear):
270
- nn.init.normal_(module.weight, mean=0.0, std=0.02)
271
- if module.bias is not None:
272
- nn.init.zeros_(module.bias)
273
- elif isinstance(module, nn.Embedding):
274
- nn.init.normal_(module.weight, mean=0.0, std=0.02)
275
-
276
- # Scale down the projections that write into the residual stream
277
- scale = (2 * nb_layers) ** -0.5
278
- for layer in self.transformer_layers:
279
- with torch.no_grad():
280
- layer.attn.proj.weight.mul_(scale)
281
- layer.mlp.fc2.weight.mul_(scale)
282
-
283
- def forward(self, board_tokens, player_token, castling_token, en_passant_token):
284
- batch_size = board_tokens.size(0)
285
-
286
- token_emb = self.token_embedding(board_tokens) # (B, 64, D)
287
- pos_emb = self.position_embedding(self.pos_index) # (67, D)
288
- castling_emb = self.castling_embedding(castling_token).unsqueeze(1) # (B, 1, D)
289
- en_passant_emb = self.en_passant_embedding(en_passant_token).unsqueeze(1) # (B, 1, D)
290
- player_emb = self.player_embedding(player_token).unsqueeze(1) # (B, 1, D)
291
-
292
- x = token_emb + pos_emb[:64] # Add positional embedding to board tokens
293
- x = torch.cat([
294
- x,
295
- castling_emb + pos_emb[64],
296
- en_passant_emb + pos_emb[65],
297
- player_emb + pos_emb[66],
298
- ], dim=1) # (B, 67, D)
299
-
300
- for layer in self.transformer_layers:
301
- x = layer(x)
302
-
303
- x = self.final_norm(x)
304
-
305
- state_token = x[:, -3:, :].mean(dim=1) # Average the last 3 game-state tokens: (B, D)
306
-
307
- board_tokens_out = x[:, :64, :] # (B, 64, D)
308
- move_logits = self.move_head(board_tokens_out) # (B, 64, NUM_ACTION_PLANES)
309
- value_pred = self.value_head(state_token) # (B, 1)
310
-
311
- return move_logits, value_pred
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
chesstransformer/models/transformer/position2move.py DELETED
@@ -1,329 +0,0 @@
1
- import torch
2
- from torch import nn
3
- import torch.nn.functional as F
4
-
5
- from chesstransformer.models.transformer.attention import GroupedQueryAttention
6
-
7
-
8
- class RMSNorm(nn.Module):
9
- """Root Mean Square Layer Normalization.
10
-
11
- More efficient than LayerNorm as it doesn't compute mean.
12
- Used in Llama, Mistral, and other modern transformers.
13
- """
14
-
15
- def __init__(self, dim: int, eps: float = 1e-6):
16
- super().__init__()
17
- self.eps = eps
18
- self.weight = nn.Parameter(torch.ones(dim))
19
-
20
- def forward(self, x: torch.Tensor) -> torch.Tensor:
21
- # x: (batch, seq, dim)
22
- rms = torch.rsqrt(x.pow(2).mean(-1, keepdim=True) + self.eps)
23
- return x * rms * self.weight
24
-
25
-
26
- class SwiGLUFeedForward(nn.Module):
27
- """SwiGLU Feed-Forward Network.
28
-
29
- Combines Swish activation with Gated Linear Units.
30
- Better performance than standard GELU FFN.
31
- Used in Llama 2, PaLM, and other modern transformers.
32
- """
33
-
34
- def __init__(self, embed_dim: int, hidden_dim: int = None, dropout: float = 0.0):
35
- super().__init__()
36
- # Default hidden_dim is ~2.67x embed_dim (compensates for extra params from gate)
37
- hidden_dim = hidden_dim or int(embed_dim * 8 / 3)
38
- # Round to nearest multiple of 64 for efficiency
39
- hidden_dim = ((hidden_dim + 63) // 64) * 64
40
-
41
- self.w1 = nn.Linear(embed_dim, hidden_dim, bias=False) # Up projection
42
- self.w2 = nn.Linear(hidden_dim, embed_dim, bias=False) # Down projection
43
- self.w3 = nn.Linear(embed_dim, hidden_dim, bias=False) # Gate projection
44
- self.dropout = nn.Dropout(dropout)
45
-
46
- def forward(self, x: torch.Tensor) -> torch.Tensor:
47
- # SwiGLU: w2(SiLU(w1(x)) * w3(x))
48
- return self.dropout(self.w2(F.silu(self.w1(x)) * self.w3(x)))
49
-
50
-
51
- # Keep old FeedForward for backward compatibility
52
- class FeedForward(nn.Module):
53
- def __init__(self, embed_dim):
54
- super().__init__()
55
- self.layers = nn.Sequential(
56
- nn.Linear(embed_dim, 4 * embed_dim),
57
- nn.GELU(approximate="tanh"),
58
- nn.Linear(4 * embed_dim, embed_dim),
59
- )
60
-
61
- def forward(self, x):
62
- return self.layers(x)
63
-
64
-
65
- class TransformerBlock(nn.Module):
66
- def __init__(
67
- self,
68
- embed_dim,
69
- num_heads,
70
- num_kv_groups,
71
- context_length,
72
- dropout,
73
- qkv_bias,
74
- mask_future=False,
75
- apply_rope=True,
76
- use_swiglu=True,
77
- ):
78
- super().__init__()
79
- self.att = GroupedQueryAttention(
80
- d_in=embed_dim,
81
- d_out=embed_dim,
82
- num_heads=num_heads,
83
- num_kv_groups=num_kv_groups,
84
- dropout=dropout,
85
- qkv_bias=qkv_bias,
86
- mask_future=mask_future,
87
- apply_rope=apply_rope,
88
- )
89
- # Use SwiGLU by default, fallback to standard FFN
90
- if use_swiglu:
91
- self.feed_forward = SwiGLUFeedForward(embed_dim, dropout=dropout)
92
- else:
93
- self.feed_forward = FeedForward(embed_dim)
94
-
95
- # Use RMSNorm instead of LayerNorm
96
- self.norm1 = RMSNorm(embed_dim)
97
- self.norm2 = RMSNorm(embed_dim)
98
- self.dropout_shortcut = nn.Dropout(dropout)
99
-
100
- def forward(self, x):
101
- shortcut = x
102
- x = self.norm1(x)
103
- x = self.att(x)
104
- x = self.dropout_shortcut(x)
105
- x = x + shortcut
106
-
107
- shortcut = x
108
- x = self.norm2(x)
109
- x = self.feed_forward(x)
110
- x = self.dropout_shortcut(x)
111
- x = x + shortcut
112
-
113
- return x
114
-
115
-
116
- class Position2MoveModel(nn.Module):
117
- def __init__(
118
- self,
119
- vocab_size: int = 13,
120
- move_vocab_size: int = 1856,
121
- embed_dim: int = 512,
122
- nb_transformer_layers: int = 8,
123
- num_heads: int = 8,
124
- num_kv_groups: int = 4,
125
- dropout: float = 0.1,
126
- kvq_bias: bool = False,
127
- mask_future: bool = False,
128
- rope: bool = True,
129
- use_swiglu: bool = True,
130
- use_col_row_emb: bool = False,
131
- ):
132
- super().__init__()
133
- # Learnable CLS token for classification (instead of mean pooling)
134
- self.cls_token = nn.Parameter(torch.randn(1, 1, embed_dim) * 0.02)
135
-
136
- # Board piece embeddings
137
- self.token_embedding = nn.Embedding(vocab_size, embed_dim)
138
- if not rope:
139
- self.position_embedding = nn.Embedding(64, embed_dim) # 64 squares
140
- self.player_embedding = nn.Embedding(2, embed_dim) # 2 players: white and black
141
-
142
- self.use_col_row_emb = use_col_row_emb
143
-
144
- if use_col_row_emb:
145
- # Optional column and row embeddings for board squares
146
- self.col_embedding = nn.Embedding(8, embed_dim)
147
- self.row_embedding = nn.Embedding(8, embed_dim)
148
-
149
- # Additional game state embeddings
150
- # Castling rights: 4 binary flags (white_king, white_queen, black_king, black_queen)
151
- self.castling_embedding = nn.Embedding(16, embed_dim) # 2^4 = 16 combinations
152
-
153
- # En passant: 9 values (files a-h = 0-7, none = 8)
154
- self.en_passant_embedding = nn.Embedding(9, embed_dim)
155
-
156
- # Halfmove clock: bucketed into ranges for 50-move rule awareness
157
- # Buckets: 0, 1-5, 6-10, 11-20, 21-30, 31-40, 41-50, 50+
158
- self.halfmove_embedding = nn.Embedding(8, embed_dim)
159
-
160
- self.dropout = nn.Dropout(dropout)
161
- self.transformer_blocks = nn.Sequential(
162
- *[
163
- TransformerBlock(
164
- embed_dim=embed_dim,
165
- num_heads=num_heads,
166
- num_kv_groups=num_kv_groups,
167
- context_length=64 + 3 + 1, # 64 squares + 3 game state tokens + 1 CLS
168
- dropout=dropout,
169
- qkv_bias=kvq_bias,
170
- mask_future=mask_future,
171
- apply_rope=rope,
172
- use_swiglu=use_swiglu,
173
- )
174
- for _ in range(nb_transformer_layers)
175
- ]
176
- )
177
- # Final RMSNorm
178
- self.norm = RMSNorm(embed_dim)
179
-
180
- self.lm_head = nn.Linear(embed_dim, move_vocab_size, bias=False)
181
-
182
- self._init_weights()
183
-
184
- def _init_weights(self):
185
- torch.nn.init.normal_(self.token_embedding.weight, mean=0.0, std=0.02)
186
- if hasattr(self, "position_embedding"):
187
- torch.nn.init.normal_(self.position_embedding.weight, mean=0.0, std=0.02)
188
- if self.use_col_row_emb:
189
- torch.nn.init.normal_(self.col_embedding.weight, mean=0.0, std=0.02)
190
- torch.nn.init.normal_(self.row_embedding.weight, mean=0.0, std=0.02)
191
-
192
- torch.nn.init.normal_(self.player_embedding.weight, mean=0.0, std=0.02)
193
- torch.nn.init.normal_(self.castling_embedding.weight, mean=0.0, std=0.02)
194
- torch.nn.init.normal_(self.en_passant_embedding.weight, mean=0.0, std=0.02)
195
- torch.nn.init.normal_(self.halfmove_embedding.weight, mean=0.0, std=0.02)
196
-
197
- # init the q, k, v, and output projection layers using xavier initialization
198
- for layer in self.transformer_blocks:
199
- if isinstance(layer.att, GroupedQueryAttention):
200
- torch.nn.init.xavier_uniform_(layer.att.q_proj.weight)
201
- if layer.att.q_proj.bias is not None:
202
- torch.nn.init.zeros_(layer.att.q_proj.bias)
203
- torch.nn.init.xavier_uniform_(layer.att.k_proj.weight)
204
- if layer.att.k_proj.bias is not None:
205
- torch.nn.init.zeros_(layer.att.k_proj.bias)
206
- torch.nn.init.xavier_uniform_(layer.att.v_proj.weight)
207
- if layer.att.v_proj.bias is not None:
208
- torch.nn.init.zeros_(layer.att.v_proj.bias)
209
- torch.nn.init.xavier_uniform_(layer.att.proj.weight)
210
-
211
- # Initialize SwiGLU layers
212
- if isinstance(layer.feed_forward, SwiGLUFeedForward):
213
- torch.nn.init.xavier_uniform_(layer.feed_forward.w1.weight)
214
- torch.nn.init.xavier_uniform_(layer.feed_forward.w2.weight)
215
- torch.nn.init.xavier_uniform_(layer.feed_forward.w3.weight)
216
-
217
- torch.nn.init.xavier_uniform_(self.lm_head.weight)
218
-
219
- def _bucket_halfmove(self, halfmove: torch.Tensor) -> torch.Tensor:
220
- """Convert halfmove clock values to bucket indices.
221
-
222
- Buckets: 0, 1-5, 6-10, 11-20, 21-30, 31-40, 41-50, 50+
223
- Uses searchsorted to avoid torch.where (better for MPS compilation).
224
- """
225
- # Define bucket boundaries
226
- boundaries = torch.tensor([1, 6, 11, 21, 31, 41, 51], device=halfmove.device)
227
- # searchsorted gives us the bucket index directly
228
- # Values less than 1 -> bucket 0
229
- # Values in [1, 6) -> bucket 1
230
- # Values in [6, 11) -> bucket 2, etc.
231
- buckets = torch.searchsorted(boundaries, halfmove.float(), right=False)
232
- buckets = torch.clamp(buckets, 0, 7)
233
- return buckets.long()
234
-
235
- def forward(self, x, is_white, castling_rights=None, en_passant_file=None, halfmove_clock=None):
236
- """
237
- Args:
238
- x: Tensor of shape (batch_size, 64) containing token IDs for each square.
239
- is_white: Tensor of shape (batch_size,) indicating if the player to move is white (True) or black (False).
240
- castling_rights: Tensor of shape (batch_size,) with values 0-15 encoding 4 binary flags.
241
- Bit 0: white kingside, Bit 1: white queenside,
242
- Bit 2: black kingside, Bit 3: black queenside.
243
- If None, defaults to all castling allowed (15).
244
- en_passant_file: Tensor of shape (batch_size,) with values 0-7 for files a-h, or 8 for none.
245
- If None, defaults to no en passant (8).
246
- halfmove_clock: Tensor of shape (batch_size,) with halfmove clock values (0-100+).
247
- If None, defaults to 0.
248
- Returns:
249
- logits: Tensor of shape (batch_size, move_vocab_size) containing
250
- the logits for the next move prediction at each position.
251
- """
252
- batch_size, seq_len = x.size()
253
- assert seq_len == 64, "Input sequence length must be 64 (8x8 chess board)."
254
-
255
- device = x.device
256
-
257
- # Default values if not provided
258
- if castling_rights is None:
259
- castling_rights = torch.full((batch_size,), 15, dtype=torch.long, device=device)
260
- if en_passant_file is None:
261
- en_passant_file = torch.full((batch_size,), 8, dtype=torch.long, device=device)
262
- if halfmove_clock is None:
263
- halfmove_clock = torch.zeros(batch_size, dtype=torch.long, device=device)
264
-
265
- positions = torch.arange(seq_len, device=device).unsqueeze(0).expand(batch_size, -1)
266
- token_emb = self.token_embedding(x) # (batch_size, 64, embed_dim)
267
- if hasattr(self, "position_embedding"):
268
- pos_emb = self.position_embedding(positions) # (batch_size, 64, embed_dim)
269
- else:
270
- pos_emb = torch.zeros_like(token_emb) # Fallback if RoPE is used
271
-
272
- player_ids = is_white.long().unsqueeze(1).expand(-1, seq_len) # (batch_size, 64)
273
-
274
- player_emb = self.player_embedding(player_ids) # (batch_size, 64, embed_dim)
275
-
276
- board_emb = token_emb + pos_emb + player_emb # (batch_size, 64, embed_dim)
277
-
278
- if self.use_col_row_emb:
279
- cols = (positions % 8).long() # (batch_size, 64)
280
- rows = (positions // 8).long() # (batch_size, 64)
281
- col_emb = self.col_embedding(cols) # (batch_size, 64, embed_dim)
282
- row_emb = self.row_embedding(rows) # (batch_size, 64, embed_dim)
283
- board_emb = board_emb + col_emb + row_emb # (batch_size, 64, embed_dim)
284
-
285
- # Create game state tokens
286
- castling_emb = self.castling_embedding(castling_rights.long()) # (batch_size, embed_dim)
287
- en_passant_emb = self.en_passant_embedding(en_passant_file.long()) # (batch_size, embed_dim)
288
- halfmove_bucket = self._bucket_halfmove(halfmove_clock.long())
289
- halfmove_emb = self.halfmove_embedding(halfmove_bucket) # (batch_size, embed_dim)
290
-
291
- # Stack game state tokens: (batch_size, 3, embed_dim)
292
- game_state_tokens = torch.stack([castling_emb, en_passant_emb, halfmove_emb], dim=1)
293
-
294
- # Expand CLS token for batch: (batch_size, 1, embed_dim)
295
- cls_tokens = self.cls_token.expand(batch_size, -1, -1)
296
-
297
- # Concatenate: [CLS] + board + game_state: (batch_size, 68, embed_dim)
298
- x = torch.cat([cls_tokens, board_emb, game_state_tokens], dim=1)
299
- x = self.dropout(x)
300
-
301
- x = self.transformer_blocks(x) # (batch_size, 68, embed_dim)
302
- x = self.norm(x) # (batch_size, 68, embed_dim)
303
-
304
- # Use CLS token output for classification (instead of mean pooling)
305
- x = x[:, 0, :] # (batch_size, embed_dim)
306
-
307
- logits = self.lm_head(x) # (batch_size, move_vocab_size)
308
- return logits
309
-
310
-
311
- if __name__ == "__main__":
312
- # Example usage
313
- model = Position2MoveModel()
314
- batch_size = 2
315
- dummy_input = torch.randint(0, 13, (batch_size, 64)) # batch_size=2, seq_len=64
316
- is_white = torch.tensor([1, 0]) # First sample is white to move, second is black
317
-
318
- # With game state info
319
- castling_rights = torch.tensor([15, 3]) # First: all castling, Second: only white can castle
320
- en_passant_file = torch.tensor([8, 4]) # First: no EP, Second: EP on file e (4)
321
- halfmove_clock = torch.tensor([0, 25]) # Halfmove counters
322
-
323
- # Forward pass with all info
324
- logits = model(dummy_input, is_white, castling_rights, en_passant_file, halfmove_clock)
325
- print("Output shape:", logits.shape) # Should be (2, move_vocab_size)
326
-
327
- # Also works without game state (backward compatible)
328
- logits_simple = model(dummy_input, is_white)
329
- print("Output shape (simple):", logits_simple.shape)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
chesstransformer/models/transformer/rope.py DELETED
@@ -1,104 +0,0 @@
1
- """This module implements Rotary Positional Embeddings (RoPE) as described in
2
- the paper "RoFormer: Enhanced Transformer with Rotary Position Embedding" (Su et al., 2021).
3
- """
4
-
5
- import torch
6
- import torch.nn as nn
7
-
8
-
9
- class RoPEEmbedding(nn.Module):
10
- """Rotary Positional Embedding module.
11
-
12
- This module applies rotary positional embeddings to the input tensor.
13
- Works with input shape (batch, num_heads, seq_len, head_dim).
14
- """
15
-
16
- def __init__(self, dim: int, base: int = 10_000, max_position_embeddings: int = 2048):
17
- """Initialize the RoPE embedding module.
18
-
19
- Args:
20
- dim (int): Dimension of each attention head (head_dim).
21
- base (int): Base for the frequency computation. Lower values = faster rotation.
22
- Default 10,000 works well for long sequences. For short sequences
23
- like chess (67 tokens), consider lower values like 1,000.
24
- max_position_embeddings (int): Maximum number of position embeddings.
25
- """
26
- super(RoPEEmbedding, self).__init__()
27
- self.dim = dim
28
- self.base = base
29
-
30
- # Compute inverse frequencies
31
- inv_freq = 1.0 / (base ** (torch.arange(0, dim, 2).float() / dim))
32
-
33
- # Compute position encodings for all positions
34
- position = torch.arange(max_position_embeddings, dtype=torch.float)
35
- # (max_pos, dim/2)
36
- idx_theta = torch.einsum("i, j -> i j", position, inv_freq)
37
-
38
- # Duplicate for both halves: (max_pos, dim)
39
- idx_theta2 = torch.cat([idx_theta, idx_theta], dim=-1)
40
-
41
- # Shape: (1, 1, max_pos, dim) for broadcasting with (batch, heads, seq, dim)
42
- self.register_buffer("cos_cached", idx_theta2.cos()[None, None, :, :], persistent=False)
43
- self.register_buffer("sin_cached", idx_theta2.sin()[None, None, :, :], persistent=False)
44
-
45
- def _neg_half(self, x: torch.Tensor):
46
- """Rotate tensor by swapping and negating halves.
47
-
48
- For RoPE, we need to compute: [-x2, x1] where x = [x1, x2]
49
- This implements the rotation matrix multiplication efficiently.
50
-
51
- Args:
52
- x (torch.Tensor): Input tensor of shape (..., dim).
53
-
54
- Returns:
55
- torch.Tensor: Rotated tensor.
56
- """
57
- d_2 = self.dim // 2
58
- x1 = x[..., :d_2]
59
- x2 = x[..., d_2:]
60
- return torch.cat([-x2, x1], dim=-1)
61
-
62
- def forward(self, x: torch.Tensor):
63
- """Apply RoPE to the input tensor.
64
-
65
- Args:
66
- x (torch.Tensor): Input tensor of shape (batch, num_heads, seq_len, head_dim).
67
-
68
- Returns:
69
- torch.Tensor: Tensor with RoPE applied, same shape as input.
70
- """
71
- # x shape: (batch, num_heads, seq_len, head_dim)
72
- seq_len = x.shape[2] # Get seq_len from the correct dimension
73
-
74
- # Slice cached values to sequence length: (1, 1, seq_len, dim)
75
- cos = self.cos_cached[:, :, :seq_len, :].to(x.device)
76
- sin = self.sin_cached[:, :, :seq_len, :].to(x.device)
77
-
78
- # Apply rotary embedding: x * cos + rotate(x) * sin
79
- x_rotated = self._neg_half(x)
80
- return x * cos + x_rotated * sin
81
-
82
-
83
- if __name__ == "__main__":
84
- # Test with chess-like dimensions
85
- batch_size = 2
86
- num_heads = 8
87
- seq_len = 67 # 64 squares + 3 game state tokens
88
- head_dim = 64
89
-
90
- rope = RoPEEmbedding(dim=head_dim, base=10_000, max_position_embeddings=128)
91
- x = torch.randn(batch_size, num_heads, seq_len, head_dim)
92
-
93
- output = rope(x)
94
- print(f"Input shape: {x.shape}")
95
- print(f"Output shape: {output.shape}")
96
-
97
- # Verify positions are encoded differently
98
- # Two different positions should produce different outputs
99
- x_same = torch.ones(1, 1, 10, head_dim)
100
- out = rope(x_same)
101
- print(f"\nPosition encoding test (should be different):")
102
- print(f" Position 0 norm: {out[0, 0, 0].norm():.4f}")
103
- print(f" Position 5 norm: {out[0, 0, 5].norm():.4f}")
104
- print(f" Position 0 vs 5 cosine sim: {torch.cosine_similarity(out[0, 0, 0], out[0, 0, 5], dim=0):.4f}")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
chesstransformer/models/transformer/utils.py DELETED
@@ -1,46 +0,0 @@
1
- """Utility functions for models."""
2
-
3
- import torch
4
- import torch.nn as nn
5
-
6
-
7
- class LayerNorm(nn.Module):
8
- """Implements Layer Normalization"""
9
-
10
- def __init__(self, dim, eps=1e-5):
11
- super().__init__()
12
- self.eps = eps
13
- self.gamma = nn.Parameter(torch.ones(dim))
14
- self.beta = nn.Parameter(torch.zeros(dim))
15
-
16
- def forward(self, x):
17
- """
18
- Forward pass of layer normalization
19
- Args:
20
- x: Input tensor of shape (batch_size, seq_length, dim)
21
- Returns:
22
- out: Layer-normalized tensor of the same shape as input
23
- """
24
-
25
- mean = x.mean(-1, keepdim=True)
26
- var = x.var(-1, keepdim=True, unbiased=False)
27
- x_hat = (x - mean) / torch.sqrt(var + self.eps)
28
- out = self.gamma * x_hat + self.beta
29
- return out
30
-
31
-
32
- class GELU(nn.Module):
33
- """Implements the GELU activation function"""
34
-
35
- def __init__(self):
36
- super().__init__()
37
-
38
- def forward(self, x):
39
- """
40
- Forward pass of GELU activation
41
- Args:
42
- x: Input tensor
43
- Returns:
44
- out: Tensor after applying GELU activation
45
- """
46
- return 0.5 * x * (1 + torch.tanh(torch.sqrt(torch.tensor(2.0 / torch.pi)) * (x + 0.044715 * torch.pow(x, 3))))
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
chesstransformer/optimizer.py DELETED
@@ -1,206 +0,0 @@
1
- """Combined AdamW + Muon optimizer.
2
-
3
- The two underlying optimizers are kept as separate ``torch.optim`` instances
4
- internally; this class is a thin wrapper that exposes the standard
5
- ``torch.optim.Optimizer`` interface (``step``, ``zero_grad``, ``param_groups``,
6
- ``state_dict`` / ``load_state_dict``) so that LR schedulers, gradient
7
- accumulation, and HuggingFace ``Accelerator`` treat it as a single optimizer.
8
-
9
- Routing rule (when constructed from an ``nn.Module``):
10
-
11
- * parameters whose name contains an ``embedding_keyword`` → AdamW (low LR)
12
- * parameters whose name contains a ``head_keyword`` → AdamW (no decay)
13
- * remaining parameters with ``ndim >= 2`` → Muon
14
- * remaining parameters with ``ndim < 2`` (biases/norms) → AdamW (no decay)
15
- """
16
-
17
- from __future__ import annotations
18
-
19
- from typing import Iterable
20
-
21
- import torch
22
- from torch import nn
23
- from torch.optim import AdamW, Muon, Optimizer
24
-
25
-
26
- class AdamWMuon(Optimizer):
27
- """Wrapper that drives an inner ``AdamW`` and ``Muon`` in lock-step.
28
-
29
- Parameters
30
- ----------
31
- model:
32
- ``nn.Module`` whose parameters will be auto-routed by name.
33
- lr:
34
- Base AdamW learning rate (used for the ``other`` 1D parameter group).
35
- lr_embedding, lr_head:
36
- Per-group AdamW LR overrides. Default to ``lr``.
37
- lr_muon:
38
- Muon LR for ≥2D body weights. Default ``= 3 * lr`` (Muon tolerates a
39
- higher LR than AdamW because updates are orthogonalised).
40
- weight_decay:
41
- AdamW weight decay applied to the embedding group. Heads and 1D
42
- params get ``0.0`` by convention.
43
- weight_decay_muon:
44
- Muon weight decay. Default = ``weight_decay``.
45
- embedding_keywords / head_keywords:
46
- Substrings used to classify parameters by ``named_parameters`` key.
47
- """
48
-
49
- def __init__(
50
- self,
51
- model: nn.Module,
52
- *,
53
- lr: float = 3e-4,
54
- lr_embedding: float | None = None,
55
- lr_head: float | None = None,
56
- lr_muon: float | None = None,
57
- weight_decay: float = 0.1,
58
- weight_decay_muon: float | None = None,
59
- betas: tuple[float, float] = (0.9, 0.95),
60
- eps: float = 1e-8,
61
- muon_momentum: float = 0.95,
62
- muon_nesterov: bool = True,
63
- muon_ns_steps: int = 5,
64
- embedding_keywords: Iterable[str] = ("embedding", "cls_token"),
65
- head_keywords: Iterable[str] = ("lm_head", "value_head"),
66
- ) -> None:
67
- if not isinstance(model, nn.Module):
68
- raise TypeError(
69
- f"AdamWMuon expects an nn.Module to auto-route parameters by name, "
70
- f"got {type(model).__name__}."
71
- )
72
-
73
- lr_embedding = lr if lr_embedding is None else lr_embedding
74
- lr_head = lr if lr_head is None else lr_head
75
- lr_muon = 3.0 * lr if lr_muon is None else lr_muon
76
- weight_decay_muon = weight_decay if weight_decay_muon is None else weight_decay_muon
77
-
78
- embedding_keywords = tuple(embedding_keywords)
79
- head_keywords = tuple(head_keywords)
80
-
81
- emb_params: list[torch.Tensor] = []
82
- head_params: list[torch.Tensor] = []
83
- other_params: list[torch.Tensor] = []
84
- muon_params: list[torch.Tensor] = []
85
-
86
- for name, param in model.named_parameters():
87
- if not param.requires_grad:
88
- continue
89
- if any(kw in name for kw in embedding_keywords):
90
- emb_params.append(param)
91
- elif any(kw in name for kw in head_keywords):
92
- head_params.append(param)
93
- elif param.ndim >= 2:
94
- muon_params.append(param)
95
- else:
96
- other_params.append(param)
97
-
98
- adamw_groups: list[dict] = []
99
- if emb_params:
100
- adamw_groups.append({
101
- "params": emb_params, "lr": lr_embedding,
102
- "weight_decay": weight_decay, "name": "adamw_embedding",
103
- })
104
- if other_params:
105
- adamw_groups.append({
106
- "params": other_params, "lr": lr,
107
- "weight_decay": 0.0, "name": "adamw_other",
108
- })
109
- if head_params:
110
- adamw_groups.append({
111
- "params": head_params, "lr": lr_head,
112
- "weight_decay": 0.0, "name": "adamw_head",
113
- })
114
-
115
- if not adamw_groups:
116
- raise ValueError("No AdamW-eligible parameters found in model.")
117
-
118
- self.adamw = AdamW(adamw_groups, lr=lr, betas=betas, eps=eps,
119
- weight_decay=weight_decay)
120
-
121
- if muon_params:
122
- self.muon: Muon | None = Muon(
123
- [{"params": muon_params, "lr": lr_muon,
124
- "weight_decay": weight_decay_muon, "name": "muon_body"}],
125
- lr=lr_muon, weight_decay=weight_decay_muon,
126
- momentum=muon_momentum, nesterov=muon_nesterov,
127
- ns_steps=muon_ns_steps,
128
- )
129
- else:
130
- self.muon = None
131
-
132
- # Initialise the ``Optimizer`` base with a placeholder group so
133
- # ``isinstance(opt, Optimizer)`` checks (Accelerate, schedulers) pass
134
- # and ``self.state`` / ``self.defaults`` exist. We then expose the
135
- # *inner* groups via ``param_groups`` so schedulers mutate them in
136
- # place and both sub-optimizers see the new LR.
137
- all_params = emb_params + other_params + head_params + muon_params
138
- self._init_done = False
139
- super().__init__([{"params": all_params}], defaults={"lr": lr})
140
- self.param_groups = self._combined_param_groups()
141
- self._init_done = True
142
-
143
- # ── helpers ─────────────────────────────────────────────────────────
144
- def _combined_param_groups(self) -> list[dict]:
145
- groups = list(self.adamw.param_groups)
146
- if self.muon is not None:
147
- groups += list(self.muon.param_groups)
148
- return groups
149
-
150
- # ── Optimizer API ───────────────────────────────────────────────────
151
- @torch.no_grad()
152
- def step(self, closure=None):
153
- loss = None
154
- if closure is not None:
155
- with torch.enable_grad():
156
- loss = closure()
157
- self.adamw.step()
158
- if self.muon is not None:
159
- self.muon.step()
160
- return loss
161
-
162
- def zero_grad(self, set_to_none: bool = True) -> None:
163
- self.adamw.zero_grad(set_to_none=set_to_none)
164
- if self.muon is not None:
165
- self.muon.zero_grad(set_to_none=set_to_none)
166
-
167
- def state_dict(self) -> dict:
168
- return {
169
- "adamw": self.adamw.state_dict(),
170
- "muon": self.muon.state_dict() if self.muon is not None else None,
171
- }
172
-
173
- def load_state_dict(self, state_dict: dict) -> None:
174
- self.adamw.load_state_dict(state_dict["adamw"])
175
- if self.muon is not None and state_dict.get("muon") is not None:
176
- self.muon.load_state_dict(state_dict["muon"])
177
- # Re-link in case sub-optimizers rebuilt their group dicts.
178
- self.param_groups = self._combined_param_groups()
179
-
180
- def add_param_group(self, param_group) -> None:
181
- if not getattr(self, "_init_done", False):
182
- # Called from ``Optimizer.__init__`` with the placeholder group.
183
- super().add_param_group(param_group)
184
- return
185
- raise NotImplementedError(
186
- "add_param_group is not supported on AdamWMuon after construction; "
187
- "build a new optimizer with the full parameter set instead."
188
- )
189
-
190
- def __repr__(self) -> str:
191
- n_emb = sum(p.numel() for g in self.adamw.param_groups
192
- if g.get("name") == "adamw_embedding" for p in g["params"])
193
- n_head = sum(p.numel() for g in self.adamw.param_groups
194
- if g.get("name") == "adamw_head" for p in g["params"])
195
- n_other = sum(p.numel() for g in self.adamw.param_groups
196
- if g.get("name") == "adamw_other" for p in g["params"])
197
- n_muon = (sum(p.numel() for g in self.muon.param_groups for p in g["params"])
198
- if self.muon is not None else 0)
199
- return (
200
- f"AdamWMuon(emb={n_emb:,}, other_1d={n_other:,}, "
201
- f"head={n_head:,}, muon_2d={n_muon:,})"
202
- )
203
-
204
-
205
- # Backwards-compat alias for the previous skeleton class.
206
- AdamWMuonOptim = AdamWMuon
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
chesstransformer/trainers/__init__.py DELETED
@@ -1,11 +0,0 @@
1
-
2
-
3
- __all__ = [
4
- "SelfPlayTrainer",
5
- "SelfPlayEngine",
6
- "GameRecord",
7
- "MoveRecord",
8
- "OpeningPositionSampler",
9
- "compute_rewards",
10
- "COMMON_OPENINGS",
11
- ]
 
 
 
 
 
 
 
 
 
 
 
 
chesstransformer/trainers/moveseq2moveseq_trainer.py DELETED
@@ -1,342 +0,0 @@
1
- from pathlib import Path
2
- from datetime import datetime
3
-
4
- from tqdm.auto import tqdm
5
- import torch
6
- import accelerate
7
- from torch.utils.data import DataLoader
8
- from torch.utils.data import random_split
9
- from chesstransformer.datasets import LichessSimpleUciDataset
10
- from chesstransformer.models.transformer.gpt import GPT2Model
11
-
12
-
13
- ROOT_PATH = Path(__file__).parents[1]
14
-
15
-
16
- def get_next_run_number(log_dir="logs"):
17
- """Get the next run number by checking existing run directories."""
18
- log_path = Path(log_dir)
19
- if not log_path.exists():
20
- return 1
21
-
22
- # Find all directories that match the pattern "run_XXX"
23
- existing_runs = [d.name for d in log_path.iterdir() if d.is_dir() and d.name.startswith("run_")]
24
-
25
- if not existing_runs:
26
- return 1
27
-
28
- # Extract run numbers and find the maximum
29
- run_numbers = []
30
- for run_name in existing_runs:
31
- try:
32
- num = int(run_name.split("_")[1])
33
- run_numbers.append(num)
34
- except (IndexError, ValueError):
35
- continue
36
-
37
- return max(run_numbers) + 1 if run_numbers else 1
38
-
39
-
40
- def main():
41
- # Load the full dataset with LichessSimpleUciDataset
42
- print("Loading dataset...")
43
- full_dataset = LichessSimpleUciDataset(
44
- dataset_path=ROOT_PATH.parents[1] / "data/Lichess Rated Games 2017.pgn.zst",
45
- vocab_path=ROOT_PATH / "data/tokenizer_models/uci_vocab.json",
46
- num_samples=1_000_000, # Load all available games
47
- )
48
-
49
- vocab_size = len(full_dataset.vocab)
50
- print(f"Vocabulary size: {vocab_size}")
51
-
52
- # Split the dataset into training, validation, and test sets
53
- val = 20_000
54
- test = 20_000
55
- train_size = len(full_dataset) - val - test
56
- train_dataset, val_dataset, test_dataset = random_split(
57
- full_dataset,
58
- [train_size, val, test],
59
- generator=torch.Generator().manual_seed(42),
60
- )
61
-
62
- print(f"Train size: {len(train_dataset)}")
63
- print(f"Validation size: {len(val_dataset)}")
64
- print(f"Test size: {len(test_dataset)}")
65
-
66
- def collate_fn(batch):
67
- max_length = max(item["input_ids"].shape[0] for item in batch)
68
- input_ids = torch.full((len(batch), max_length), fill_value=-100, dtype=torch.long)
69
- target_ids = torch.full((len(batch), max_length), fill_value=-100, dtype=torch.long)
70
- legal_moves_mask = torch.zeros((len(batch), max_length, vocab_size), dtype=torch.float32)
71
-
72
- for i, item in enumerate(batch):
73
- length = item["input_ids"].shape[0]
74
- input_ids[i, :length] = item["input_ids"]
75
- target_ids[i, :length] = item["target_ids"]
76
- legal_moves_mask[i, :length, :] = item["legal_moves_mask"]
77
-
78
- if max_length > config["context_size"]:
79
- input_ids = input_ids[:, : config["context_size"]]
80
- target_ids = target_ids[:, : config["context_size"]]
81
- legal_moves_mask = legal_moves_mask[:, : config["context_size"], :]
82
-
83
- return {
84
- "input_ids": input_ids,
85
- "target_ids": target_ids,
86
- "legal_moves_mask": legal_moves_mask,
87
- }
88
-
89
- train_dl = DataLoader(train_dataset, batch_size=8, shuffle=True, collate_fn=collate_fn, drop_last=True)
90
- val_dl = DataLoader(val_dataset, batch_size=8, shuffle=False, collate_fn=collate_fn)
91
- test_dl = DataLoader(test_dataset, batch_size=8, shuffle=False, collate_fn=collate_fn)
92
-
93
- config = {
94
- "vocab_size": vocab_size,
95
- "context_size": 256,
96
- "embed_dim": 512,
97
- "num_heads": 8,
98
- "num_layers": 8,
99
- "dropout": 0.1,
100
- "qkv_bias": False,
101
- }
102
-
103
- model = GPT2Model(config)
104
- print(model)
105
-
106
- # Generate unique run identifier
107
- run_number = get_next_run_number("logs")
108
- timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
109
- run_name = f"run_{run_number:03d}_{timestamp}"
110
-
111
- print(f"Starting training run: {run_name}")
112
-
113
- # Gradient accumulation settings
114
- gradient_accumulation_steps = 8 # Effective batch size = 16 * 4 = 64
115
-
116
- # Initialize accelerator with logging and gradient accumulation
117
- accelerator = accelerate.Accelerator(
118
- log_with="tensorboard",
119
- project_dir=f"logs/{run_name}",
120
- mixed_precision="fp16" if torch.cuda.is_available() else "no",
121
- gradient_accumulation_steps=gradient_accumulation_steps,
122
- )
123
- device = accelerator.device
124
- print(f"Using device: {device}")
125
- print(f"Gradient accumulation steps: {gradient_accumulation_steps}")
126
- print(f"Effective batch size: {16 * gradient_accumulation_steps}")
127
- model.to(device)
128
-
129
- # Initialize TensorBoard tracker
130
- accelerator.init_trackers(
131
- project_name="chess_transformer",
132
- config={
133
- **config,
134
- "run_name": run_name,
135
- "run_number": run_number,
136
- "timestamp": timestamp,
137
- "learning_rate": 1e-3,
138
- "weight_decay": 1e-1,
139
- "num_epochs": 10,
140
- "batch_size": 16,
141
- "gradient_accumulation_steps": 8,
142
- "effective_batch_size": 128,
143
- "warmup_steps": 1000,
144
- },
145
- )
146
-
147
- optimizer = torch.optim.AdamW(model.parameters(), lr=5e-4, weight_decay=1e-1)
148
-
149
- # Calculate total training steps (accounting for gradient accumulation)
150
- num_epochs = 10
151
- total_steps = (len(train_dl) // gradient_accumulation_steps) * num_epochs
152
- warmup_steps = 1000
153
-
154
- print(f"Total training steps: {total_steps}")
155
- print(f"Warmup steps: {warmup_steps}")
156
-
157
- # Create linear scheduler with warmup
158
- def get_linear_schedule_with_warmup(optimizer, num_warmup_steps, num_training_steps):
159
- def lr_lambda(current_step):
160
- if current_step < num_warmup_steps:
161
- # Linear warmup
162
- return float(current_step) / float(max(1, num_warmup_steps))
163
- # Linear decay
164
- return max(
165
- 0.0, float(num_training_steps - current_step) / float(max(1, num_training_steps - num_warmup_steps))
166
- )
167
-
168
- return torch.optim.lr_scheduler.LambdaLR(optimizer, lr_lambda)
169
-
170
- scheduler = get_linear_schedule_with_warmup(optimizer, warmup_steps, total_steps)
171
-
172
- model, optimizer, train_dl, val_dl, scheduler = accelerator.prepare(model, optimizer, train_dl, val_dl, scheduler)
173
- criterion = torch.nn.CrossEntropyLoss(ignore_index=-100)
174
-
175
- best_val_loss = float("inf")
176
- global_step = 0
177
-
178
- for epoch in range(num_epochs):
179
- model.train()
180
- total_loss = 0.0
181
- total_llm_loss = 0.0
182
- total_l2_loss = 0.0
183
-
184
- loader = tqdm(train_dl, desc=f"Epoch {epoch + 1}/{num_epochs}")
185
- for batch_idx, batch in enumerate(loader):
186
- # Accelerate automatically handles gradient accumulation context
187
- with accelerator.accumulate(model):
188
- inputs = batch["input_ids"].to(device)
189
- targets = batch["target_ids"].to(device)
190
- legal_moves_mask = batch["legal_moves_mask"].to(device)
191
-
192
- logits = model(inputs)
193
-
194
- llm_loss = criterion(logits.flatten(0, 1), targets.flatten())
195
-
196
- probas = torch.nn.functional.softmax(logits, dim=-1)
197
-
198
- ilegal_moves_probas = probas * (1 - legal_moves_mask)
199
- l2_loss = torch.sum(ilegal_moves_probas**2, dim=-1).mean()
200
- loss = llm_loss + 10 * l2_loss
201
-
202
- loader.set_postfix({"llm_loss": llm_loss.item(), "l2_loss": l2_loss.item(), "total_loss": loss.item()})
203
-
204
- accelerator.backward(loss)
205
- optimizer.step()
206
- scheduler.step() # Update learning rate
207
- optimizer.zero_grad()
208
-
209
- total_loss += loss.item()
210
- total_llm_loss += llm_loss.item()
211
- total_l2_loss += l2_loss.item()
212
-
213
- # Log step metrics only on sync steps (when gradients are actually applied)
214
- if accelerator.sync_gradients:
215
- # Get current learning rate
216
- current_lr = scheduler.get_last_lr()[0]
217
-
218
- accelerator.log(
219
- {
220
- "train/step_loss": loss.item(),
221
- "train/step_llm_loss": llm_loss.item(),
222
- "train/step_l2_loss": l2_loss.item(),
223
- "train/learning_rate": current_lr,
224
- },
225
- step=global_step,
226
- )
227
-
228
- global_step += 1
229
-
230
- avg_train_loss = total_loss / len(train_dl)
231
- avg_train_llm_loss = total_llm_loss / len(train_dl)
232
- avg_train_l2_loss = total_l2_loss / len(train_dl)
233
- print(f"Epoch {epoch + 1}/{num_epochs}, Training Loss: {avg_train_loss:.4f}")
234
-
235
- # Validation
236
- model.eval()
237
- total_val_loss = 0.0
238
- total_val_llm_loss = 0.0
239
- total_val_l2_loss = 0.0
240
- with torch.no_grad():
241
- for batch in val_dl:
242
- inputs = batch["input_ids"].to(device)
243
- targets = batch["target_ids"].to(device)
244
- legal_moves_mask = batch["legal_moves_mask"].to(device)
245
-
246
- logits = model(inputs)
247
-
248
- llm_loss = criterion(logits.flatten(0, 1), targets.flatten())
249
-
250
- probas = torch.nn.functional.softmax(logits, dim=-1)
251
- ilegal_moves_probas = probas * (1 - legal_moves_mask)
252
- l2_loss = torch.sum(ilegal_moves_probas**2, dim=-1).mean()
253
- loss = llm_loss + l2_loss
254
-
255
- total_val_loss += loss.item()
256
- total_val_llm_loss += llm_loss.item()
257
- total_val_l2_loss += l2_loss.item()
258
-
259
- avg_val_loss = total_val_loss / len(val_dl)
260
- avg_val_llm_loss = total_val_llm_loss / len(val_dl)
261
- avg_val_l2_loss = total_val_l2_loss / len(val_dl)
262
- print(f"Epoch {epoch + 1}/{num_epochs}, Validation Loss: {avg_val_loss:.4f}")
263
-
264
- # Log epoch metrics
265
- accelerator.log(
266
- {
267
- "train/epoch_loss": avg_train_loss,
268
- "train/epoch_llm_loss": avg_train_llm_loss,
269
- "train/epoch_l2_loss": avg_train_l2_loss,
270
- "val/epoch_loss": avg_val_loss,
271
- "val/epoch_llm_loss": avg_val_llm_loss,
272
- "val/epoch_l2_loss": avg_val_l2_loss,
273
- "epoch": epoch + 1,
274
- },
275
- step=global_step,
276
- )
277
-
278
- # Save the best model
279
- if avg_val_loss < best_val_loss:
280
- best_val_loss = avg_val_loss
281
- unwrapped_model = accelerator.unwrap_model(model)
282
- torch.save(unwrapped_model.state_dict(), ROOT_PATH / "models/best_model.pth")
283
- print(f"Best model saved with validation loss: {best_val_loss:.4f}")
284
-
285
- # Log best metrics
286
- accelerator.log(
287
- {
288
- "best/val_loss": best_val_loss,
289
- "best/epoch": epoch + 1,
290
- },
291
- step=global_step,
292
- )
293
-
294
- # Testing
295
- model.eval()
296
- total_test_loss = 0.0
297
- total_test_llm_loss = 0.0
298
- total_test_l2_loss = 0.0
299
- with torch.no_grad():
300
- for batch in test_dl:
301
- inputs = batch["input_ids"].to(device)
302
- targets = batch["target_ids"].to(device)
303
- legal_moves_mask = batch["legal_moves_mask"].to(device)
304
-
305
- logits = model(inputs)
306
-
307
- llm_loss = criterion(logits.flatten(0, 1), targets.flatten())
308
-
309
- probas = torch.nn.functional.softmax(logits, dim=-1)
310
- ilegal_moves_probas = probas * (1 - legal_moves_mask)
311
- l2_loss = torch.sum(ilegal_moves_probas**2, dim=-1).mean()
312
- loss = llm_loss + l2_loss
313
-
314
- total_test_loss += loss.item()
315
- total_test_llm_loss += llm_loss.item()
316
- total_test_l2_loss += l2_loss.item()
317
-
318
- avg_test_loss = total_test_loss / len(test_dl)
319
- avg_test_llm_loss = total_test_llm_loss / len(test_dl)
320
- avg_test_l2_loss = total_test_l2_loss / len(test_dl)
321
- print(f"Test Loss: {avg_test_loss:.4f}")
322
-
323
- # Log test metrics
324
- accelerator.log(
325
- {
326
- "test/loss": avg_test_loss,
327
- "test/llm_loss": avg_test_llm_loss,
328
- "test/l2_loss": avg_test_l2_loss,
329
- },
330
- step=global_step,
331
- )
332
-
333
- # End tracking
334
- accelerator.end_training()
335
- print(f"\nTraining complete! Run: {run_name}")
336
- print(f"TensorBoard logs saved to 'logs/{run_name}/'")
337
- print("To view logs, run: tensorboard --logdir=logs")
338
- print(f"To view this run only: tensorboard --logdir=logs/{run_name}")
339
-
340
-
341
- if __name__ == "__main__":
342
- main()
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
chesstransformer/trainers/pos2move_v2_trainer.py DELETED
@@ -1,1128 +0,0 @@
1
- #!/usr/bin/env python3
2
- """Training script for Pos2MoveV2.
3
-
4
- Trains the v2 transformer with GQA + QK-norm + chess-geometry attention bias,
5
- AlphaZero-style 64×73 action output and value head on HDF5 chess datasets.
6
-
7
- Usage:
8
- python src/chesstransformer/trainers/pos2move_v2_trainer.py --data data/elite_db.h5
9
- """
10
-
11
- from pathlib import Path
12
- from datetime import datetime
13
- import argparse
14
- import json
15
- import os
16
- import random
17
- import shutil
18
- import sys
19
-
20
- # Make the checkout runnable without installing the package. `uv run` installs it into the
21
- # venv, so this never fires locally -- but a bare `git clone` on Kaggle or a rented box has
22
- # no chesstransformer on sys.path and every import below fails. scripts/build_shards.py,
23
- # scripts/verify_shards.py and tests/conftest.py already do this; the trainer did not, which
24
- # is exactly where it was first hit.
25
- sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
26
-
27
- import numpy as np
28
- import torch
29
- from torch.nn import functional as F
30
- from torch.utils.data import DataLoader, Subset, random_split
31
- from torch.optim.lr_scheduler import LambdaLR
32
- from tqdm.auto import tqdm
33
- import accelerate
34
- from accelerate.utils import broadcast_object_list, set_seed
35
-
36
- from chesstransformer.datasets.flat_shard_dataset import (
37
- FlatShardDataset,
38
- make_dataloader as make_shard_dataloader,
39
- scatter_legal_planes,
40
- worker_init_fn as shard_worker_init_fn,
41
- )
42
- from chesstransformer.datasets.h5_lichess_dataset import HDF5ChessDataset
43
- from chesstransformer.distributed_muon import (
44
- DistributedMuon,
45
- muon_momentum,
46
- muon_weight_decay,
47
- )
48
- from chesstransformer.models.tokenizer.position_tokenizer import PostionTokenizer
49
- from chesstransformer.models.transformer.pos2move_v2 import Pos2MoveV2, NUM_ACTION_PLANES
50
-
51
-
52
- #: Model sizes. DDP replicates weights *and* optimizer state on every rank.
53
- #:
54
- #: The static column is **measured**, not derived: one optimizer step then
55
- #: torch.cuda.memory_allocated(), on a 16 GB card. It comes out at ~8.4 bytes/param, not
56
- #: the ~18 a generic "fp32 master + two Adam moments + grads" estimate gives, because the
57
- #: optimizer is split -- 2D params go to Muon, which keeps a single momentum buffer, and
58
- #: the AdamW params (norms, biases, embeddings' non-2D parts) are a small remainder.
59
- #: An earlier version of this table used the 18 B/param estimate and overstated xl by 2.3x.
60
- #:
61
- #: Activations dominate and scale with micro-batch: measured ~76 MB/sample for `large`
62
- #: under torch.compile (~90 MB uncompiled). That is what actually decides whether a config
63
- #: fits, so the static column alone is not a fit test.
64
- #:
65
- #: ===== ============ ======== ============= =============================================
66
- #: name dim x layers params static/rank notes
67
- #: ===== ============ ======== ============= =============================================
68
- #: base 256 x 16 11.7 M 0.10 GB anything
69
- #: 46m 512 x 16 46.5 M 0.35 GB anything
70
- #: large 768 x 24 156.4 M 1.23 GB measured 8.5 GB/rank total at micro-batch 96
71
- #: on a T4; micro-batch 128 fits, 160 OOMs on 16 GB
72
- #: xl 1536 x 32 832.8 M ~6.4 GB extrapolated. Static alone would fit a 40 GB
73
- #: card under DDP; whether activations do is
74
- #: unmeasured -- measure before assuming FSDP.
75
- #: ===== ============ ======== ============= =============================================
76
- #:
77
- #: `xl` is deliberately oversized as a sharding/scaling testbed. It is not a bid for
78
- #: playing strength -- 46M already lost its head-to-head against v2.1 at 44.8%.
79
- PRESETS = {
80
- "base": {"embed_dim": 256, "num_layers": 16, "num_heads": 8},
81
- "46m": {"embed_dim": 512, "num_layers": 16, "num_heads": 8},
82
- "large": {"embed_dim": 768, "num_layers": 24, "num_heads": 12},
83
- "xl": {"embed_dim": 1536, "num_layers": 32, "num_heads": 16},
84
- }
85
-
86
- torch.set_float32_matmul_precision('high')
87
-
88
-
89
- def rank0_print(*a, **kw):
90
- """print() on the main process only.
91
-
92
- Keyed off the RANK env var rather than accelerator.is_main_process so it also works
93
- before the Accelerator is constructed -- dataset loading logs before that point. N
94
- ranks echoing the same line is noise; N ranks echoing *different* lines is a
95
- debugging trap, and the dataset banners are exactly where they would differ.
96
- """
97
- if int(os.environ.get("RANK", 0)) == 0:
98
- print(*a, **kw)
99
-
100
-
101
- def resolve_precision(requested: str) -> str:
102
- """Pick a mixed-precision mode this GPU can actually run.
103
-
104
- bf16 needs compute capability 8.0 (Ampere). Kaggle's T4 is Turing, 7.5 -- so the
105
- trainer's bf16 default silently does not apply there, and a run that looks fine is
106
- quietly not using tensor cores as intended. Fall back to fp16 (which Turing does have
107
- tensor cores for) and say so, rather than let the first multi-GPU session discover it.
108
- """
109
- # Accelerate spells "no mixed precision" as "no"; "fp32" is not a value it accepts.
110
- # Passing it through raises ValueError, which means --precision fp32 has never worked
111
- # and a CPU-only box (where the old code forced "fp32") could never start at all.
112
- if not torch.cuda.is_available() or os.environ.get("ACCELERATE_USE_CPU"):
113
- return "no"
114
- if requested == "fp32":
115
- return "no"
116
- if requested == "bf16" and not torch.cuda.is_bf16_supported():
117
- cap = torch.cuda.get_device_capability(0)
118
- rank0_print(f"WARNING: {torch.cuda.get_device_name(0)} is compute capability "
119
- f"{cap[0]}.{cap[1]}; bf16 needs 8.0+. Falling back to fp16 "
120
- f"(Accelerate adds a GradScaler). Pass --precision fp32 to opt out.")
121
- return "fp16"
122
- return requested
123
-
124
-
125
- def unpack_batch(batch, device):
126
- """Normalise a batch from either data path into the tensors the loss expects.
127
-
128
- The shard loader ships the legal mask as a ``(B, 64)`` uint16 index list — 64 KB per
129
- batch of 512 against 2.4 MB for the dense ``(64, 73)`` bool mask — so the mask is
130
- rebuilt here, on device, after the H2D copy. Doing it in the dataset would put the
131
- 2.4 MB straight back on the wire and undo the entire point of the shard format.
132
- """
133
- if "legal_idx" in batch:
134
- legal_planes = scatter_legal_planes(batch["legal_idx"].to(device, non_blocking=True))
135
- else:
136
- legal_planes = batch["legal_moves_planes"].to(device, non_blocking=True)
137
- return {
138
- "board": batch["position"].to(device, non_blocking=True).long(),
139
- "player": batch["is_white"].to(device, non_blocking=True).long(),
140
- "castling": batch["castling_rights"].to(device, non_blocking=True).long(),
141
- "en_passant": batch["en_passant_file"].to(device, non_blocking=True).long(),
142
- "from_sq": batch["from_square"].to(device, non_blocking=True).long(),
143
- "action_plane": batch["action_plane"].to(device, non_blocking=True).long(),
144
- "legal_planes": legal_planes,
145
- # Narrow on the wire (int8/int16 out of the shard), widened here so both data
146
- # paths hand compute_loss exactly the same dtypes.
147
- "result": batch["result"].to(device, non_blocking=True).long(),
148
- # Must stay bool: compute_loss uses it as a *mask* (`target_value[white_win &
149
- # is_white]`). An int tensor there would silently become fancy indexing and
150
- # scatter the value targets onto the wrong rows.
151
- "is_white": batch["is_white"].to(device, non_blocking=True).bool(),
152
- "move_number": batch["move_number"].to(device, non_blocking=True).long(),
153
- }
154
-
155
-
156
- def get_next_run_number(log_dir: str) -> int:
157
- log_path = Path(log_dir)
158
- if not log_path.exists():
159
- return 1
160
- run_numbers = []
161
- for d in log_path.iterdir():
162
- if d.is_dir() and d.name.startswith("run_"):
163
- try:
164
- run_numbers.append(int(d.name.split("_")[1]))
165
- except (IndexError, ValueError):
166
- continue
167
- return max(run_numbers, default=0) + 1
168
-
169
-
170
- def point_project_config_at(project_config, run_dir) -> None:
171
- """Move both project_dir and logging_dir onto the resolved run directory.
172
-
173
- Not ``set_directories(run_dir)``: that fills logging_dir only when it is None, and it is
174
- not. ``ProjectConfiguration.__post_init__`` already defaulted it to whatever project_dir
175
- the Accelerator was built with -- here the *parent*, logs/pos2move_v2, because the run
176
- name is not known until after the process group exists (see main()). Moving project_dir
177
- alone leaves the TensorBoard writer on the parent, where it appends the project name, so
178
- every run's events pile into one shared logs/pos2move_v2/pos2move_v2 instead of landing
179
- in the run directory beside that run's checkpoints.
180
- """
181
- project_config.project_dir = str(run_dir)
182
- project_config.logging_dir = str(run_dir)
183
-
184
-
185
- def compute_loss(
186
- move_logits,
187
- value,
188
- from_sq,
189
- action_plane,
190
- legal_moves_planes,
191
- result,
192
- is_white,
193
- move_number,
194
- value_loss_weight,
195
- value_rampup_halfmoves=40,
196
- label_smoothing: float = 0.0,
197
- ):
198
- """Total loss = (label-smoothed) policy CE + value MSE.
199
-
200
- Returns
201
- -------
202
- total_loss : scalar tensor (the only thing that gets backprop'd)
203
- metrics : dict of detached scalar tensors for logging
204
- - ce : standard cross-entropy on the full action space
205
- - legal_ce : cross-entropy after masking to legal moves only
206
- - value_loss : weighted MSE on the value head
207
- - acc : top-1 accuracy on the full action space
208
- - legal_acc : top-1 accuracy after masking to legal moves only
209
- """
210
- B = move_logits.size(0)
211
-
212
- # Flatten (B, 64, 73) -> (B, 64*73). Upcast to fp32 for loss stability under bf16.
213
- flat_logits = move_logits.reshape(B, -1).float()
214
- flat_target = from_sq * NUM_ACTION_PLANES + action_plane
215
- flat_mask = legal_moves_planes.reshape(B, -1).float()
216
-
217
- # ── Policy loss: label-smoothed CE over LEGAL moves only ────────────
218
- if label_smoothing > 0:
219
- eps = label_smoothing
220
- legal_count = flat_mask.sum(dim=-1, keepdim=True).clamp(min=1.0)
221
- smooth = eps / legal_count
222
- target_dist = flat_mask * smooth
223
- idx = flat_target.unsqueeze(1)
224
- target_dist.scatter_add_(1, idx, torch.full_like(idx, 1.0 - eps, dtype=target_dist.dtype))
225
- log_pred = F.log_softmax(flat_logits, dim=-1)
226
- policy_loss = F.kl_div(log_pred, target_dist, reduction="batchmean")
227
- else:
228
- policy_loss = F.cross_entropy(flat_logits, flat_target)
229
-
230
- # ── Value loss: per-sample MSE, ramped up by move number ────────────
231
- target_value = torch.zeros_like(value)
232
- white_win = result == 1
233
- black_win = result == 2
234
- target_value[white_win & is_white] = 1.0
235
- target_value[white_win & ~is_white] = -1.0
236
- target_value[black_win & is_white] = -1.0
237
- target_value[black_win & ~is_white] = 1.0
238
-
239
- progress = (move_number.float() / value_rampup_halfmoves).clamp(max=1.0)
240
- per_sample_value_loss = (value.float() - target_value) ** 2
241
- value_loss = (progress * per_sample_value_loss).mean()
242
-
243
- total_loss = policy_loss + value_loss_weight * value_loss
244
-
245
- # ── Metrics (no grad) ───────────────────────────────────────────────
246
- with torch.no_grad():
247
- ce_metric = F.cross_entropy(flat_logits, flat_target)
248
-
249
- masked_logits = flat_logits.masked_fill(flat_mask == 0, float("-inf"))
250
- legal_ce_metric = F.cross_entropy(masked_logits, flat_target)
251
-
252
- preds = flat_logits.argmax(dim=-1)
253
- acc = (preds == flat_target).float().mean()
254
-
255
- legal_preds = masked_logits.argmax(dim=-1)
256
- legal_acc = (legal_preds == flat_target).float().mean()
257
-
258
- metrics = {
259
- "ce": ce_metric,
260
- "legal_ce": legal_ce_metric,
261
- "value_loss": value_loss.detach(),
262
- "acc": acc,
263
- "legal_acc": legal_acc,
264
- }
265
- return total_loss, metrics
266
-
267
-
268
- def create_lr_scheduler(optimizer, warmup_steps, total_steps, final_lr_ratio,
269
- schedule="linear", warmdown_ratio=0.2):
270
- """Build the LR multiplier schedule.
271
-
272
- ``wsd`` -- warmup, then **stable at peak**, then a linear warmdown over the final
273
- ``warmdown_ratio`` of the run (nanochat's ``get_lr_multiplier``). ``linear`` -- warmup
274
- then decay across the entire remainder, which is what this trainer used to do.
275
-
276
- WSD is the default for two reasons. It decouples the horizon from the schedule: peak LR
277
- is held flat, so a run can be stopped or extended at any point in the stable phase and
278
- only the warmdown re-run, where a decay-from-peak schedule bakes ``total_steps`` into
279
- every step it takes. And it is the phase structure Muon's momentum schedule was designed
280
- against -- momentum warms down *during the LR warmdown*, which only exists here.
281
-
282
- Note the asymmetry in defaults, which is deliberate: the CLI defaults to ``wsd`` for new
283
- runs, but this function defaults to ``linear``. scheduler_config dicts persisted in
284
- checkpoints from before this change carry no ``schedule`` key, and are replayed through
285
- here on resume -- so the function default is what those runs get. Defaulting it to
286
- ``wsd`` would silently change the LR shape of an in-flight run at the point it resumed.
287
- """
288
- warmup_steps = max(1, warmup_steps)
289
- total_steps = max(warmup_steps + 1, total_steps)
290
- warmdown = max(1, round(warmdown_ratio * total_steps))
291
-
292
- def lr_lambda(step):
293
- if step < warmup_steps:
294
- return float(step + 1) / float(warmup_steps)
295
- if schedule == "wsd":
296
- if step <= total_steps - warmdown:
297
- return 1.0
298
- progress = max(0.0, min(1.0, (total_steps - step) / warmdown))
299
- return progress * 1.0 + (1 - progress) * final_lr_ratio
300
- progress = (step - warmup_steps + 1) / float(max(1, total_steps - warmup_steps))
301
- return 1.0 - (1.0 - final_lr_ratio) * min(1.0, progress)
302
-
303
- return LambdaLR(optimizer, lr_lambda)
304
-
305
-
306
- def save_trainer_state(checkpoint_dir, data_pass, global_step, best_val_loss, scheduler_config=None):
307
- # "epoch" is written alongside "data_pass" only so a checkpoint from this trainer still
308
- # loads in anything that predates the step paradigm. Nothing here reads it back.
309
- state = {"data_pass": data_pass, "epoch": data_pass,
310
- "global_step": global_step, "best_val_loss": best_val_loss}
311
- if scheduler_config:
312
- state["scheduler_config"] = scheduler_config
313
- with (Path(checkpoint_dir) / "trainer_state.json").open("w") as f:
314
- json.dump(state, f)
315
-
316
-
317
- def load_trainer_state(checkpoint_dir):
318
- path = Path(checkpoint_dir) / "trainer_state.json"
319
- if not path.exists():
320
- return None
321
- with path.open() as f:
322
- return json.load(f)
323
-
324
-
325
- # ── EMA helpers ──────────────────��───────────────────────────────────────
326
- # EMA state is kept in fp32 to avoid bf16 precision loss on the
327
- # (1 - decay) * param update term (which is ~0.001 * param at decay=0.999
328
- # and would round to zero for many small values in bf16).
329
-
330
- def create_ema_state(model):
331
- return {name: p.data.detach().float().clone() for name, p in model.named_parameters()}
332
-
333
-
334
- @torch.no_grad()
335
- def update_ema(model, ema_state, decay):
336
- for name, param in model.named_parameters():
337
- # Cast live param to fp32 for the lerp; ema_state stays fp32.
338
- ema_state[name].lerp_(param.data.float(), 1.0 - decay)
339
-
340
-
341
- def swap_ema_weights(model, ema_state):
342
- # Backup current (compute-dtype) weights into ema_state, load EMA weights
343
- # into the model — cast to the model's dtype on the way in.
344
- for name, param in model.named_parameters():
345
- tmp = param.data.detach().float().clone()
346
- param.data.copy_(ema_state[name].to(param.dtype))
347
- ema_state[name].copy_(tmp)
348
-
349
-
350
- def save_ema_state(ema_state, path):
351
- torch.save(ema_state, path)
352
-
353
-
354
- def load_ema_state(path, device):
355
- return torch.load(path, map_location=device, weights_only=True)
356
-
357
-
358
- def get_raw_model(model, accelerator):
359
- m = accelerator.unwrap_model(model)
360
- if hasattr(m, "_orig_mod"):
361
- m = m._orig_mod
362
- return m
363
-
364
-
365
- def cleanup_old_checkpoints(checkpoint_dir, max_checkpoints):
366
- checkpoints = sorted(
367
- [d for d in Path(checkpoint_dir).iterdir() if d.is_dir() and d.name.startswith("checkpoint_step_")],
368
- key=lambda x: int(x.name.split("_")[-1]),
369
- )
370
- while len(checkpoints) > max_checkpoints:
371
- shutil.rmtree(checkpoints.pop(0))
372
-
373
- def main():
374
- default_data = Path(__file__).parents[3] / "data" / "elite_db.h5"
375
-
376
- parser = argparse.ArgumentParser(description="Train Pos2MoveV2 (clean)")
377
- # Data
378
- parser.add_argument("--data", type=str, default=str(default_data))
379
- parser.add_argument("--shards", type=str, default=None,
380
- help="Directory of flat shards from scripts/build_shards.py. Bypasses "
381
- "--data/--min-elo/--sample-weighting entirely: the sampling "
382
- "distribution is frozen into the shard at build time.")
383
- parser.add_argument("--min-elo", type=int, default=None)
384
- parser.add_argument("--max-elo", type=int, default=None)
385
- parser.add_argument("--sample-weighting", type=str, default="middlegame", choices=["uniform", "middlegame"])
386
- parser.add_argument("--skip-opening-plies", type=int, default=0)
387
- parser.add_argument("--max-val-samples", type=int, default=50_000,
388
- help="Cap on val/test size. The HDF5 path has always applied this; "
389
- "the shard path needs it too since a reserved shard is much larger.")
390
- # Training
391
- parser.add_argument("--batch-size", type=int, default=1024)
392
- parser.add_argument("--grad-accum", type=int, default=4)
393
- parser.add_argument("--epochs", type=int, default=None,
394
- help="DEPRECATED and ignored. Training length is --max-steps. Kept so "
395
- "existing invocations do not hard-fail; it warns and does nothing.")
396
- parser.add_argument("--lr", type=float, default=1e-3)
397
- parser.add_argument("--lr-muon", type=float, default=2e-3)
398
- parser.add_argument("--lr-embedding", type=float, default=4e-4)
399
- parser.add_argument("--lr-head", type=float, default=None)
400
- parser.add_argument("--warmup-steps", type=int, default=2000)
401
- parser.add_argument("--final-lr-ratio", type=float, default=0.05)
402
- parser.add_argument("--lr-schedule", type=str, default="wsd", choices=["wsd", "linear"],
403
- help="'wsd' = warmup / stable at peak / linear warmdown over the last "
404
- "--warmdown-ratio of the run. 'linear' = the previous behaviour, "
405
- "decay from peak across the whole remainder.")
406
- parser.add_argument("--warmdown-ratio", type=float, default=0.2,
407
- help="Fraction of the run spent decaying, for --lr-schedule wsd. Muon's "
408
- "momentum warmdown is aligned to the same window.")
409
- parser.add_argument("--max-grad-norm", type=float, default=1.0)
410
- parser.add_argument("--weight-decay", type=float, default=0.1)
411
- # Loss weights
412
- parser.add_argument("--value-loss-weight", type=float, default=5.0)
413
- parser.add_argument("--label-smoothing", type=float, default=0.1)
414
- # Model
415
- parser.add_argument("--preset", type=str, default="base", choices=list(PRESETS),
416
- help="Model size. See PRESETS for per-GPU DDP memory. Individual "
417
- "--embed-dim/--num-layers/--num-heads override the preset.")
418
- parser.add_argument("--embed-dim", type=int, default=None)
419
- parser.add_argument("--num-layers", type=int, default=None)
420
- parser.add_argument("--num-heads", type=int, default=None)
421
- parser.add_argument("--dropout", type=float, default=0.05)
422
- parser.add_argument("--layer-drop", type=float, default=0.1,
423
- help="Stochastic depth rate. Linearly scales from 0 (first layer) to this value (last layer). Try 0.1.") # Checkpointing
424
- parser.add_argument("--max-steps", type=int, default=50_000,
425
- help="Total optimizer steps to train for. This is THE training-length "
426
- "knob: it bounds the loop and sets the LR decay horizon, and it "
427
- "means the same thing on both data paths and at any world size.")
428
- parser.add_argument("--save-every", type=int, default=None,
429
- help="DEPRECATED and ignored (was epoch-based checkpointing). "
430
- "Use --save-steps.")
431
- parser.add_argument("--save-steps", type=int, default=5000)
432
- parser.add_argument("--eval-steps", type=int, default=1000,
433
- help="Validate every N optimizer steps. Val loss selects the best "
434
- "checkpoint, so this sets the resolution of that choice. 0 "
435
- "disables periodic validation (a final one still runs).")
436
- parser.add_argument("--max-checkpoints", type=int, default=5)
437
- parser.add_argument("--resume-from", type=str, default=None)
438
- # EMA
439
- parser.add_argument("--ema-decay", type=float, default=0.999)
440
- # Muon
441
- parser.add_argument("--distributed-muon", action=argparse.BooleanOptionalAction, default=True,
442
- help="Partition Newton-Schulz across ranks by whole parameter, so the "
443
- "NS work and momentum state are divided instead of every rank "
444
- "redundantly computing the same thing. No-op at world size 1.")
445
- parser.add_argument("--muon-schedules", action=argparse.BooleanOptionalAction, default=True,
446
- help="Schedule Muon momentum (0.85->0.97, down to 0.90 during warmdown) "
447
- "and cosine-decay its weight decay to zero, as nanochat does.")
448
- # Misc
449
- parser.add_argument("--seed", type=int, default=42)
450
- parser.add_argument("--precision", type=str, default="bf16", choices=["bf16", "fp16", "fp32"])
451
- parser.add_argument("--num-workers", type=int, default=12)
452
- # store_true with default=True could never be turned off. BooleanOptionalAction
453
- # gives a real --no-compile, which CPU/gloo testing and debugging both need.
454
- parser.add_argument("--compile", action=argparse.BooleanOptionalAction, default=True)
455
- parser.add_argument("--compile-mode", type=str, default="default",
456
- choices=["default", "reduce-overhead", "max-autotune"])
457
- parser.add_argument("--compile-order", type=str, default="auto",
458
- choices=["auto", "before-prepare", "after-prepare"],
459
- help="Whether torch.compile runs before or after accelerator.prepare(). "
460
- "'auto' = before on a single process (the ordering the 2.2x "
461
- "single-GPU speedup was measured with), after under DDP so "
462
- "Dynamo's DDPOptimizer engages. See the note at the call site.")
463
- args = parser.parse_args()
464
-
465
- # Epochs are gone as a unit of training length. They were never comparable between the
466
- # two data paths — the HDF5 dataset is indexed by *game* (738 optimizer steps/epoch) and
467
- # the shard dataset by *sample* (11,495 for elite_k16, 50,505 for full_k4) — so the same
468
- # --epochs N bought a 15-68x different amount of training, LR decay horizon and
469
- # validation cadence depending only on which --data/--shards flag you passed.
470
- for dead, replacement in (("epochs", "--max-steps"), ("save_every", "--save-steps")):
471
- if getattr(args, dead) is not None:
472
- rank0_print(f"WARNING: --{dead.replace('_', '-')} is deprecated and ignored; "
473
- f"use {replacement}.")
474
- if args.max_steps <= 0:
475
- parser.error("--max-steps must be > 0")
476
-
477
- # Preset supplies the geometry; an explicit flag always wins over it.
478
- preset = PRESETS[args.preset]
479
- for flag, key in (("embed_dim", "embed_dim"), ("num_layers", "num_layers"),
480
- ("num_heads", "num_heads")):
481
- if getattr(args, flag) is None:
482
- setattr(args, flag, preset[key])
483
-
484
- set_seed(args.seed)
485
- random.seed(args.seed)
486
- np.random.seed(args.seed)
487
- torch.manual_seed(args.seed)
488
- torch.cuda.manual_seed_all(args.seed)
489
-
490
- # ── Logging ──────────────────────────────────────────────────────────
491
- # The run directory has to be decided by ONE rank and told to the others. Every rank
492
- # scanning logs/ for the next free run number is a race: they can pick the same number
493
- # (two ranks writing one run) or different ones (N runs, N-1 of them silently empty),
494
- # and mkdir(exist_ok=False) then crashes whichever rank loses. The timestamp in the
495
- # name makes independent derivation impossible too -- ranks start milliseconds apart.
496
- #
497
- # Ordering note: the Accelerator must exist before the broadcast (it is what sets up
498
- # the process group), but init_trackers needs the run dir. Hence build it against the
499
- # parent, then point it at the resolved run dir with set_directories().
500
- log_dir = Path("logs") / "pos2move_v2"
501
- log_dir.mkdir(parents=True, exist_ok=True)
502
-
503
- precision = resolve_precision(args.precision)
504
- accelerator = accelerate.Accelerator(
505
- log_with="tensorboard",
506
- project_dir=str(log_dir),
507
- mixed_precision=precision,
508
- gradient_accumulation_steps=args.grad_accum,
509
- )
510
-
511
- if accelerator.is_main_process:
512
- run_number = get_next_run_number(str(log_dir))
513
- run_name = f"run_{run_number:03d}_{datetime.now().strftime('%Y%m%d_%H%M%S')}"
514
- else:
515
- run_name = None
516
- run_name = broadcast_object_list([run_name], from_process=0)[0]
517
-
518
- log_path = log_dir / run_name
519
- checkpoint_dir = log_path / "checkpoints"
520
- if accelerator.is_main_process:
521
- checkpoint_dir.mkdir(parents=True, exist_ok=False)
522
- # Every rank must see the directory before anyone writes a checkpoint into it.
523
- accelerator.wait_for_everyone()
524
- point_project_config_at(accelerator.project_configuration, log_path)
525
-
526
- # ── Dataset ──────────────────────────────────────────────────────────
527
- use_shards = args.shards is not None
528
- if use_shards:
529
- # Flat memmap shards: the replay, tokenize and legal-move enumeration were all
530
- # paid once at build time, and the val/test split is by whole shard (== whole
531
- # game range) rather than by sample, so no game straddles the split.
532
- train_set = FlatShardDataset(args.shards, "train")
533
- val_set = FlatShardDataset(args.shards, "val")
534
- test_set = FlatShardDataset(args.shards, "test")
535
-
536
- # Whole reserved shards hold far more than validation needs (478k samples at
537
- # K=16). Cap them the way the HDF5 path does, with a fixed seed so val loss stays
538
- # comparable across runs — it is what selects the best checkpoint.
539
- def cap(ds, name):
540
- if len(ds) <= args.max_val_samples:
541
- return ds
542
- g = np.random.default_rng(0xC0FFEE)
543
- keep = np.sort(g.choice(len(ds), size=args.max_val_samples, replace=False))
544
- rank0_print(f" {name}: capped {len(ds):,} -> {args.max_val_samples:,} samples")
545
- return Subset(ds, keep.tolist())
546
-
547
- val_set = cap(val_set, "val")
548
- test_set = cap(test_set, "test")
549
-
550
- # The shard loaders bypass accelerator.prepare() (see the note at prepare()), so
551
- # rank splitting is NOT automatic here the way it is for the HDF5 path -- it has to
552
- # be asked for. Without this every rank draws the identical batches from the same
553
- # seeded RandomSampler, and DDP degenerates into N ranks computing one gradient N
554
- # times: the loss curve looks healthy, the effective batch never grows, and the
555
- # extra GPUs buy nothing.
556
- # This is exactly why the Accelerator is constructed above the dataset section:
557
- # DistributedSampler requires an initialised process group, and the Accelerator is
558
- # what initialises it. Building the loaders first raises "Default process group has
559
- # not been initialized".
560
- world_size = accelerator.num_processes
561
- train_loader = make_shard_dataloader(train_set, batch_size=args.batch_size, shuffle=True,
562
- num_workers=args.num_workers, drop_last=True,
563
- seed=args.seed, distributed=world_size > 1)
564
- # Val/test are deliberately NOT sharded. Each rank evaluates the whole (capped)
565
- # set and therefore computes an identical val loss, so best-model selection agrees
566
- # on every rank by construction and needs no gather or padding-trim. The cost is
567
- # N x redundant work on <=10k samples -- about two seconds -- against a class of
568
- # bug where ranks disagree about which checkpoint is best and race to write it.
569
- val_loader = make_shard_dataloader(val_set, batch_size=args.batch_size, shuffle=False,
570
- num_workers=args.num_workers)
571
- test_loader = make_shard_dataloader(test_set, batch_size=args.batch_size, shuffle=False,
572
- num_workers=args.num_workers)
573
- vocab_size = PostionTokenizer().vocab_size
574
- rank0_print(f"Shards: {args.shards} (K={train_set.meta['k']}, "
575
- f"mode={train_set.meta['sample_mode']}, seed={train_set.meta['seed']})")
576
- else:
577
- dataset = HDF5ChessDataset(
578
- hdf5_path=args.data,
579
- min_elo=args.min_elo,
580
- max_elo=args.max_elo,
581
- sample_weighting=args.sample_weighting,
582
- skip_opening_plies=args.skip_opening_plies,
583
- )
584
-
585
- val_size = min(int(0.1 * len(dataset)), args.max_val_samples)
586
- test_size = min(int(0.1 * len(dataset)), args.max_val_samples)
587
- train_size = len(dataset) - val_size - test_size
588
- train_set, val_set, test_set = random_split(dataset, [train_size, val_size, test_size])
589
-
590
- # pin_memory lets the H2D copy run on the DMA engine and overlap with compute;
591
- # without it `.to(device, non_blocking=True)` is silently synchronous.
592
- train_loader = DataLoader(train_set, batch_size=args.batch_size, shuffle=True,
593
- num_workers=args.num_workers, drop_last=True,
594
- persistent_workers=True, prefetch_factor=4, pin_memory=True,
595
- worker_init_fn=shard_worker_init_fn)
596
- val_loader = DataLoader(val_set, batch_size=args.batch_size, shuffle=False,
597
- num_workers=args.num_workers, pin_memory=True,
598
- worker_init_fn=shard_worker_init_fn)
599
- test_loader = DataLoader(test_set, batch_size=args.batch_size, shuffle=False,
600
- num_workers=args.num_workers, pin_memory=True,
601
- worker_init_fn=shard_worker_init_fn)
602
- vocab_size = dataset.position_tokenizer.vocab_size
603
-
604
- rank0_print(f"Train: {len(train_set):,} | Val: {len(val_set):,} | Test: {len(test_set):,}")
605
-
606
- # ── Model ────────────────────────────────────────────────────────────
607
- model_config = {
608
- "vocab_size": vocab_size,
609
- "embed_dim": args.embed_dim,
610
- "nb_transformer_layers": args.num_layers,
611
- "num_heads": args.num_heads,
612
- "dropout": args.dropout,
613
- "kvq_bias": False,
614
- "layer_drop": args.layer_drop,
615
- }
616
- if accelerator.is_main_process:
617
- with (log_path / "model_config.json").open("w") as f:
618
- json.dump(model_config, f, indent=2)
619
-
620
- # Effective batch scales with world size: every rank contributes its own micro-batch
621
- # to the same all-reduced gradient. Reporting it without num_processes understates the
622
- # real batch by N, which is exactly the number linear LR scaling is derived from.
623
- effective_bs = args.batch_size * args.grad_accum * accelerator.num_processes
624
- rank0_print(f"Effective batch size: {effective_bs} (micro={args.batch_size} × "
625
- f"accum={args.grad_accum} × ranks={accelerator.num_processes})")
626
- accelerator.init_trackers(
627
- project_name="pos2move_v2",
628
- config={
629
- "batch_size": args.batch_size,
630
- "grad_accum": args.grad_accum,
631
- "effective_batch_size": effective_bs,
632
- "max_steps": args.max_steps,
633
- "lr": args.lr,
634
- "weight_decay": args.weight_decay,
635
- "value_loss_weight": args.value_loss_weight,
636
- **model_config,
637
- },
638
- )
639
-
640
- device = accelerator.device
641
- rank0_print(f"Device: {device} | Precision: {precision} | Logging to: {log_path}")
642
-
643
- model = Pos2MoveV2(**model_config)
644
- total_params = sum(p.numel() for p in model.parameters())
645
- rank0_print(f"Model parameters: {total_params:,}")
646
- model.to(device)
647
-
648
- # torch.compile vs accelerator.prepare() ordering is not cosmetic under DDP.
649
- #
650
- # Dynamo's DDPOptimizer (torch._dynamo.backends.distributed, on by default) splits the
651
- # graph at DDP's gradient all-reduce bucket boundaries so the collectives can overlap
652
- # with backward compute. Its own docs: "DDPOptimizer applies when dynamo compiles
653
- # models wrapped in DistributedDataParallel". Compiling first gives DDP(OptimizedModule)
654
- # -- the wrap happens outside the compiled graph, DDPOptimizer never sees a DDP module,
655
- # and the overlap is lost. Compiling after prepare() gives compile(DDP(model)), which is
656
- # the shape it wants.
657
- #
658
- # Single process has no buckets to align to, so 'before' is kept there: it is the
659
- # ordering the measured 2.2x speedup and the compile-time numbers come from.
660
- compile_order = args.compile_order
661
- if compile_order == "auto":
662
- compile_order = "after-prepare" if accelerator.num_processes > 1 else "before-prepare"
663
-
664
- def maybe_compile(m, when):
665
- if args.compile and compile_order == when:
666
- rank0_print(f"torch.compile (mode={args.compile_mode}, {when})")
667
- return torch.compile(m, mode=args.compile_mode, dynamic=False) # shapes are static
668
- return m
669
-
670
- model = maybe_compile(model, "before-prepare")
671
-
672
- # ── Optimizer (Muon for 2D weights, AdamW for embeddings/heads/1D) ──
673
- lr_emb = args.lr_embedding or args.lr
674
- lr_head = args.lr_head or args.lr
675
-
676
- muon_params = []
677
- adamw_emb_params = []
678
- adamw_head_params = []
679
- adamw_other_params = []
680
-
681
- for name, param in model.named_parameters():
682
- if "embedding" in name:
683
- adamw_emb_params.append(param)
684
- elif "move_head" in name or "value_head" in name:
685
- adamw_head_params.append(param)
686
- elif param.ndim == 2:
687
- muon_params.append(param)
688
- else:
689
- adamw_other_params.append(param)
690
-
691
- rank0_print(f"LR: emb={lr_emb:.2e} | muon={args.lr_muon:.2e} | head={lr_head:.2e}")
692
- rank0_print(
693
- f"Params: emb={sum(p.numel() for p in adamw_emb_params):,} | "
694
- f"muon={sum(p.numel() for p in muon_params):,} | "
695
- f"adamw_other={sum(p.numel() for p in adamw_other_params):,} | "
696
- f"head={sum(p.numel() for p in adamw_head_params):,}"
697
- )
698
-
699
- # DistributedMuon degenerates to exactly torch.optim.Muon at world size 1 (verified in
700
- # tests/test_distributed_muon.py), so there is one code path rather than two.
701
- muon_cls = DistributedMuon if args.distributed_muon else torch.optim.Muon
702
- muon_optimizer = muon_cls(
703
- muon_params,
704
- lr=args.lr_muon,
705
- momentum=0.95,
706
- weight_decay=args.weight_decay,
707
- )
708
- if args.distributed_muon and accelerator.num_processes > 1:
709
- rank0_print(muon_optimizer.sharding_summary().split(" | ")[0])
710
- adamw_optimizer = torch.optim.AdamW([
711
- {"params": adamw_emb_params, "lr": lr_emb, "weight_decay": args.weight_decay},
712
- {"params": adamw_other_params, "lr": args.lr, "weight_decay": 0.0},
713
- {"params": adamw_head_params, "lr": lr_head, "weight_decay": 0.0},
714
- ])
715
-
716
- if use_shards:
717
- # The shard loaders are deliberately kept out of prepare(). They run with
718
- # batch_size=None and a BatchSampler so that a whole index list reaches
719
- # __getitem__ in one call; Accelerate reads batch_size=None as "this iterable
720
- # already yields batches" and would re-shard at the wrong granularity. Rank
721
- # splitting is done by the DistributedSampler inside make_shard_dataloader, and
722
- # the H2D copy by unpack_batch.
723
- model, muon_optimizer, adamw_optimizer = accelerator.prepare(
724
- model, muon_optimizer, adamw_optimizer
725
- )
726
- else:
727
- model, muon_optimizer, adamw_optimizer, train_loader, val_loader, test_loader = accelerator.prepare(
728
- model, muon_optimizer, adamw_optimizer, train_loader, val_loader, test_loader
729
- )
730
-
731
- model = maybe_compile(model, "after-prepare")
732
-
733
- # ── EMA ──────────────────────────────────────────────────────────────
734
- use_ema = args.ema_decay > 0
735
- ema_state = None
736
- if use_ema:
737
- ema_state = create_ema_state(get_raw_model(model, accelerator))
738
- rank0_print(f"EMA enabled (decay={args.ema_decay})")
739
-
740
- # ── Resume ───────────────────────────────────────────────────────────
741
- best_val_loss = float("inf")
742
- start_pass = 0
743
- global_step = 0
744
- trainer_state = None
745
-
746
- if args.resume_from:
747
- trainer_state = load_trainer_state(args.resume_from)
748
- if trainer_state:
749
- # "epoch" is the pre-step-paradigm key; read it so old checkpoints still resume.
750
- start_pass = trainer_state.get("data_pass", trainer_state.get("epoch", 0)) + 1
751
- global_step = trainer_state["global_step"]
752
- best_val_loss = trainer_state["best_val_loss"]
753
-
754
- # ── Scheduler ────────────────────────────────────────────────────────
755
- total_steps = args.max_steps
756
- scheduler_config = {
757
- "warmup_steps": args.warmup_steps,
758
- "total_steps": total_steps,
759
- "final_lr_ratio": args.final_lr_ratio,
760
- "schedule": args.lr_schedule,
761
- "warmdown_ratio": args.warmdown_ratio,
762
- }
763
- if trainer_state and "scheduler_config" in trainer_state:
764
- # A resume keeps the schedule it started with, so the LR curve stays continuous.
765
- # That means --max-steps still bounds the *loop* but no longer sets the LR horizon,
766
- # and the two can silently disagree. Say so rather than let it be discovered later.
767
- scheduler_config = trainer_state["scheduler_config"]
768
- resumed_total = scheduler_config.get("total_steps")
769
- if resumed_total != args.max_steps:
770
- rank0_print(f"WARNING: resuming with the checkpoint's LR schedule "
771
- f"(total_steps={resumed_total:,}) while --max-steps={args.max_steps:,} "
772
- f"bounds the loop. The LR decays over {resumed_total:,} steps regardless.")
773
- total_steps = resumed_total
774
-
775
- # An "epoch" is not comparable between the two data paths: the HDF5 dataset is indexed
776
- # by *game* (738 optimizer steps/epoch) and the shard dataset by *sample* (11.5k for
777
- # elite_k16, 50.5k for full_k4). So --epochs N silently buys a 15-68x longer LR decay
778
- # on shards, and epoch-end validation fires that much less often. Print both so the
779
- # schedule is never a surprise, and say so out loud when --max-steps is not pinning it.
780
- steps_per_pass = len(train_loader) // max(1, args.grad_accum)
781
- samples_seen = args.max_steps * effective_bs
782
- warmdown_note = (f"warmdown {round(args.warmdown_ratio * total_steps):,} | "
783
- if args.lr_schedule == "wsd" else "")
784
- rank0_print(f"Schedule: {total_steps:,} optimizer steps | {args.lr_schedule} | "
785
- f"warmup {args.warmup_steps:,} | {warmdown_note}"
786
- f"eval every {args.eval_steps:,} | save every {args.save_steps:,}")
787
- rank0_print(f" one pass over the training data = {steps_per_pass:,} steps/rank "
788
- f"({args.max_steps / max(1, steps_per_pass):.2f} passes, "
789
- f"{samples_seen:,} samples consumed)")
790
-
791
- muon_scheduler = create_lr_scheduler(muon_optimizer, **scheduler_config)
792
- adamw_scheduler = create_lr_scheduler(adamw_optimizer, **scheduler_config)
793
- accelerator.register_for_checkpointing(muon_scheduler, adamw_scheduler)
794
-
795
- if args.resume_from:
796
- accelerator.load_state(args.resume_from)
797
- if use_ema:
798
- ema_path = Path(args.resume_from) / "ema_state.pt"
799
- if ema_path.exists():
800
- ema_state = load_ema_state(ema_path, device)
801
- rank0_print(f"Restored EMA state from {ema_path}")
802
- else:
803
- ema_state = create_ema_state(get_raw_model(model, accelerator))
804
- rank0_print("EMA state not found in checkpoint, re-initialized from model")
805
- rank0_print(f"Resumed from {args.resume_from} (step {global_step:,}/{args.max_steps:,}, "
806
- f"data pass {start_pass})")
807
-
808
- # ── Validation ───────────────────────────────────────────────────────
809
- # Runs on the --eval-steps counter. It used to run only at epoch end, which was a fine
810
- # cadence on the HDF5 path (738 steps/epoch) and a bad one on shards (11,495 steps for
811
- # elite_k16, 50,505 for full_k4) — the dataset index is a sample there, not a game. Val
812
- # loss selects the best checkpoint, so that gap mattered. Hence the step paradigm.
813
-
814
- def run_validation():
815
- """Evaluate on val_loader (under EMA weights if enabled). Restores train mode."""
816
- model.eval()
817
- unwrapped = get_raw_model(model, accelerator)
818
- if use_ema:
819
- swap_ema_weights(unwrapped, ema_state)
820
-
821
- # No cross-rank reduction here on purpose: val_loader is replicated, not sharded,
822
- # so every rank walks the identical set and arrives at the identical number. That
823
- # is what keeps best-model selection consistent across ranks without a gather.
824
- sums = dict(loss=0.0, ce=0.0, legal_ce=0.0, value=0.0, correct=0, legal_correct=0, total=0)
825
- with torch.no_grad():
826
- for batch in tqdm(val_loader, desc="Validation", leave=False,
827
- disable=not accelerator.is_main_process):
828
- b = unpack_batch(batch, device)
829
- move_logits, value = model(b["board"], b["player"], b["castling"], b["en_passant"])
830
- loss, metrics = compute_loss(
831
- move_logits, value,
832
- b["from_sq"], b["action_plane"],
833
- b["legal_planes"], b["result"], b["is_white"], b["move_number"],
834
- args.value_loss_weight,
835
- )
836
- B = move_logits.size(0)
837
- sums["loss"] += loss.item() * B
838
- sums["ce"] += metrics["ce"].item() * B
839
- sums["legal_ce"] += metrics["legal_ce"].item() * B
840
- sums["value"] += metrics["value_loss"].item() * B
841
- sums["correct"] += int(metrics["acc"].item() * B)
842
- sums["legal_correct"] += int(metrics["legal_acc"].item() * B)
843
- sums["total"] += B
844
-
845
- if use_ema:
846
- swap_ema_weights(unwrapped, ema_state)
847
- model.train()
848
-
849
- n = max(1, sums["total"])
850
- return {
851
- "val/loss": sums["loss"] / n,
852
- "val/ce": sums["ce"] / n,
853
- "val/legal_ce": sums["legal_ce"] / n,
854
- "val/value_loss": sums["value"] / n,
855
- "val/accuracy": sums["correct"] / n,
856
- "val/legal_accuracy": sums["legal_correct"] / n,
857
- }
858
-
859
- def record_validation(vals, data_pass, extra_log=None, label=""):
860
- """Log, print, and update best-model selection. Returns True if it was a new best."""
861
- nonlocal best_val_loss
862
- log_dict = dict(extra_log or {})
863
- log_dict.update(vals)
864
- if use_ema:
865
- log_dict["val/ema_loss"] = vals["val/loss"]
866
- log_dict["val/ema_accuracy"] = vals["val/accuracy"]
867
- accelerator.log(log_dict, step=global_step)
868
-
869
- ema_tag = " (EMA)" if use_ema else ""
870
- rank0_print(f"{label}val_loss={vals['val/loss']:.4f}{ema_tag} "
871
- f"| val_acc={vals['val/accuracy']:.4f} "
872
- f"| val_legal_acc={vals['val/legal_accuracy']:.4f}")
873
-
874
- if vals["val/loss"] < best_val_loss:
875
- best_val_loss = vals["val/loss"]
876
- best_path = checkpoint_dir / "best_model"
877
- accelerator.save_state(str(best_path))
878
- save_trainer_state(best_path, data_pass, global_step, best_val_loss, scheduler_config)
879
- if use_ema and accelerator.is_main_process:
880
- save_ema_state(ema_state, best_path / "ema_state.pt")
881
- rank0_print(f" -> New best model (val_loss={vals['val/loss']:.4f})")
882
- return True
883
- return False
884
-
885
- # ── Training loop (step-driven) ──────────────────────────────────────
886
- # The loop is bounded by optimizer steps and cycles the dataloader as many times as
887
- # that takes. Nothing downstream keys off "epoch" any more: LR decay, validation and
888
- # checkpointing are all on step counters, which mean the same thing whichever dataset
889
- # is mounted and at any world size.
890
-
891
- def start_pass_shuffle(loader, n):
892
- """Reshuffle for a fresh pass over the data.
893
-
894
- DistributedSampler needs set_epoch or every rank replays the identical order on
895
- every pass. RandomSampler carries its generator across iterators and reshuffles on
896
- its own, so this is a no-op there.
897
- """
898
- for obj in (loader,
899
- getattr(loader, "sampler", None),
900
- getattr(getattr(loader, "sampler", None), "sampler", None)):
901
- if obj is not None and hasattr(obj, "set_epoch"):
902
- obj.set_epoch(n)
903
- return
904
-
905
- def new_window():
906
- """Train metrics accumulated since the last validation, sample-weighted."""
907
- return dict(loss=0.0, ce=0.0, legal_ce=0.0, value=0.0, acc=0.0, legal_acc=0.0, n=0)
908
-
909
- WINDOW_KEYS = ("loss", "ce", "legal_ce", "value", "acc", "legal_acc")
910
-
911
- def flush_window(w):
912
- """Average the window across ranks, not just within one.
913
-
914
- Each rank trains on a different slice, so these sums are rank-local: rank 0's
915
- numbers alone are a 1/N sample of the step. Reducing sum-of-(metric x batch) and
916
- sum-of-batch separately, then dividing, is exact regardless of how unevenly the
917
- last batches fall -- which is why this does not need gather_for_metrics and its
918
- padding-trim bookkeeping (that machinery also would not work here, since the shard
919
- loaders never went through prepare()).
920
- """
921
- vals = torch.tensor([w[k] for k in WINDOW_KEYS] + [float(w["n"])],
922
- dtype=torch.float64, device=accelerator.device)
923
- if accelerator.num_processes > 1:
924
- vals = accelerator.reduce(vals, reduction="sum")
925
- total = max(1.0, vals[-1].item())
926
- return {f"train/{'value_loss' if k == 'value' else k}": vals[i].item() / total
927
- for i, k in enumerate(WINDOW_KEYS)}
928
-
929
- last_val_step = -1
930
-
931
- def validate_now(window, data_pass, label_prefix):
932
- """Flush the train window, validate, log both against the same step.
933
-
934
- No-ops if this step has already been validated, which happens whenever max_steps is
935
- an exact multiple of eval_steps and the final validation lands on the same step as
936
- the last periodic one. Worth guarding rather than tolerating: on the HDF5 path the
937
- two calls do not even agree, because ``HDF5ChessDataset.__getitem__`` samples a
938
- fresh ply per call, so its val set re-rolls on every pass. (The shard val set is
939
- fixed, so shard val loss is comparable across evaluations and HDF5 val loss is not
940
- — worth remembering when reading either curve.)
941
- """
942
- nonlocal last_val_step
943
- if global_step == last_val_step:
944
- return
945
- last_val_step = global_step
946
- extra = flush_window(window) if window["n"] else {}
947
- extra["data_pass"] = data_pass
948
- train_tag = f"train_loss={extra['train/loss']:.4f} | " if window["n"] else ""
949
- record_validation(run_validation(), data_pass, extra_log=extra,
950
- label=f"{label_prefix}{train_tag}")
951
-
952
- model.train()
953
- window = new_window()
954
- data_pass = start_pass
955
- pbar = tqdm(total=args.max_steps, initial=global_step, desc="train", unit="step",
956
- disable=not accelerator.is_main_process)
957
-
958
- while global_step < args.max_steps:
959
- start_pass_shuffle(train_loader, data_pass)
960
- for batch in train_loader:
961
- with accelerator.accumulate(model):
962
- b = unpack_batch(batch, device)
963
- board, player = b["board"], b["player"]
964
- castling, en_passant = b["castling"], b["en_passant"]
965
- from_sq, action_plane = b["from_sq"], b["action_plane"]
966
- legal_planes, result = b["legal_planes"], b["result"]
967
- is_white, move_number = b["is_white"], b["move_number"]
968
-
969
- move_logits, value = model(board, player, castling, en_passant)
970
- loss, metrics = compute_loss(
971
- move_logits, value,
972
- from_sq, action_plane,
973
- legal_planes, result, is_white, move_number,
974
- args.value_loss_weight,
975
- label_smoothing=args.label_smoothing,
976
- )
977
-
978
- accelerator.backward(loss)
979
- if accelerator.sync_gradients and args.max_grad_norm > 0:
980
- accelerator.clip_grad_norm_(model.parameters(), args.max_grad_norm)
981
- if args.muon_schedules:
982
- # Momentum is not a constant in a well-tuned Muon run: a long memory
983
- # early averages over a fast-moving gradient, and a slightly shorter one
984
- # late helps the run settle. Weight decay cosine-decays to zero.
985
- mom = muon_momentum(global_step, args.max_steps,
986
- warmdown_ratio=args.warmdown_ratio)
987
- wdk = muon_weight_decay(global_step, args.max_steps, args.weight_decay)
988
- for g in muon_optimizer.param_groups:
989
- g["momentum"], g["weight_decay"] = mom, wdk
990
- muon_optimizer.step()
991
- adamw_optimizer.step()
992
- if accelerator.sync_gradients:
993
- muon_scheduler.step()
994
- adamw_scheduler.step()
995
- muon_optimizer.zero_grad()
996
- adamw_optimizer.zero_grad()
997
-
998
- # EMA is redundant under DDP but not wrong, and that is worth stating.
999
- # Gradients are all-reduced before the step, so every rank holds identical
1000
- # params and computes an identical EMA from an identical decay -- no drift.
1001
- # swap_ema_weights in run_validation is likewise symmetric because every
1002
- # rank validates (val_loader is replicated, not sharded). The failure mode
1003
- # to avoid is any rank taking a different branch here; sync_gradients is
1004
- # world-uniform, so none does. tests/test_distributed.py asserts it.
1005
- if use_ema and accelerator.sync_gradients:
1006
- update_ema(get_raw_model(model, accelerator), ema_state, args.ema_decay)
1007
-
1008
- bs = board.size(0)
1009
- window["loss"] += loss.item() * bs
1010
- window["ce"] += metrics["ce"].item() * bs
1011
- window["legal_ce"] += metrics["legal_ce"].item() * bs
1012
- window["value"] += metrics["value_loss"].item() * bs
1013
- window["acc"] += metrics["acc"].item() * bs
1014
- window["legal_acc"] += metrics["legal_acc"].item() * bs
1015
- window["n"] += bs
1016
-
1017
- pbar.set_postfix(
1018
- loss=f"{loss.item():.4f}",
1019
- ce=f"{metrics['ce'].item():.4f}",
1020
- acc=f"{metrics['acc'].item():.3f}",
1021
- val_l=f"{metrics['value_loss'].item():.4f}",
1022
- )
1023
-
1024
- if not accelerator.sync_gradients:
1025
- continue
1026
-
1027
- muon_lrs = muon_scheduler.get_last_lr()
1028
- adamw_lrs = adamw_scheduler.get_last_lr()
1029
- accelerator.log(
1030
- {
1031
- "train/step_loss": loss.item(),
1032
- "train/step_ce": metrics["ce"].item(),
1033
- "train/step_legal_ce": metrics["legal_ce"].item(),
1034
- "train/step_value_loss": metrics["value_loss"].item(),
1035
- "train/step_acc": metrics["acc"].item(),
1036
- "train/step_legal_acc": metrics["legal_acc"].item(),
1037
- "train/lr_muon": muon_lrs[0],
1038
- "train/lr_emb": adamw_lrs[0],
1039
- "train/lr_head": adamw_lrs[2],
1040
- **({"train/muon_momentum": muon_optimizer.param_groups[0]["momentum"],
1041
- "train/muon_wd": muon_optimizer.param_groups[0]["weight_decay"]}
1042
- if args.muon_schedules else {}),
1043
- },
1044
- step=global_step,
1045
- )
1046
-
1047
- global_step += 1
1048
- pbar.update(1)
1049
-
1050
- if args.save_steps > 0 and global_step % args.save_steps == 0:
1051
- ckpt = checkpoint_dir / f"checkpoint_step_{global_step:07d}"
1052
- accelerator.save_state(str(ckpt))
1053
- save_trainer_state(ckpt, data_pass, global_step, best_val_loss, scheduler_config)
1054
- if use_ema and accelerator.is_main_process:
1055
- save_ema_state(ema_state, ckpt / "ema_state.pt")
1056
- if accelerator.is_main_process:
1057
- rank0_print(f"\n Saved step checkpoint at step {global_step}")
1058
- cleanup_old_checkpoints(checkpoint_dir, args.max_checkpoints)
1059
-
1060
- # Validation is on a step counter, and global_step is identical on every rank,
1061
- # so all ranks enter the collective forward together.
1062
- if args.eval_steps > 0 and global_step % args.eval_steps == 0:
1063
- validate_now(window, data_pass,
1064
- f"\nStep {global_step:,}/{args.max_steps:,}: ")
1065
- window = new_window()
1066
-
1067
- if global_step >= args.max_steps:
1068
- break
1069
-
1070
- data_pass += 1
1071
-
1072
- pbar.close()
1073
-
1074
- # Always finish on a validation, so the final steps can still win best_model even when
1075
- # max_steps is not a multiple of eval_steps.
1076
- rank0_print(f"\nReached max-steps ({args.max_steps:,}) after {data_pass} pass(es) over the data.")
1077
- validate_now(window, data_pass, f"Final (step {global_step:,}): ")
1078
-
1079
- # ── Final test (using EMA weights) ───────────────────────────────────
1080
- rank0_print("\nFinal test evaluation...")
1081
- model.eval()
1082
- if use_ema:
1083
- swap_ema_weights(get_raw_model(model, accelerator), ema_state)
1084
- test_loss_sum = 0.0
1085
- test_correct = 0
1086
- test_legal_correct = 0
1087
- test_total = 0
1088
-
1089
- with torch.no_grad():
1090
- for batch in tqdm(test_loader, desc="Testing", leave=False,
1091
- disable=not accelerator.is_main_process):
1092
- b = unpack_batch(batch, device)
1093
- board, player = b["board"], b["player"]
1094
- castling, en_passant = b["castling"], b["en_passant"]
1095
- from_sq, action_plane = b["from_sq"], b["action_plane"]
1096
- legal_planes, result = b["legal_planes"], b["result"]
1097
- is_white, move_number = b["is_white"], b["move_number"]
1098
-
1099
- move_logits, value = model(board, player, castling, en_passant)
1100
- loss, metrics = compute_loss(
1101
- move_logits, value,
1102
- from_sq, action_plane,
1103
- legal_planes, result, is_white, move_number,
1104
- args.value_loss_weight,
1105
- )
1106
-
1107
- B = move_logits.size(0)
1108
- test_loss_sum += loss.item() * B
1109
- test_correct += int(metrics["acc"].item() * B)
1110
- test_legal_correct += int(metrics["legal_acc"].item() * B)
1111
- test_total += B
1112
-
1113
- test_loss = test_loss_sum / test_total
1114
- test_acc = test_correct / test_total
1115
- test_legal_acc = test_legal_correct / test_total
1116
-
1117
- accelerator.log(
1118
- {"test/loss": test_loss, "test/accuracy": test_acc, "test/legal_accuracy": test_legal_acc},
1119
- step=global_step,
1120
- )
1121
- rank0_print(f"Test: loss={test_loss:.4f} | accuracy={test_acc:.4f} | legal_accuracy={test_legal_acc:.4f}")
1122
-
1123
- accelerator.end_training()
1124
- rank0_print(f"\nTraining complete. Logs: {log_path}")
1125
-
1126
-
1127
- if __name__ == "__main__":
1128
- main()
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
chesstransformer/trainers/position2move_trainer.py DELETED
@@ -1,682 +0,0 @@
1
- #!/usr/bin/env python3
2
- """
3
- Training script for Position2Move transformer model.
4
-
5
- This script trains a transformer to predict chess moves from board positions.
6
- Uses the new efficient HDF5 dataset format with game-based storage and random sampling.
7
-
8
- Usage:
9
- python src/chesstransformer/trainers/position2move_trainer.py --data data/Lichess2017.h5
10
-
11
- Options:
12
- --data: Path to HDF5 dataset
13
- --min-elo: Minimum average ELO for filtering games
14
- --max-elo: Maximum average ELO for filtering games
15
- --batch-size: Batch size (default: 32)
16
- --epochs: Number of epochs (default: 10)
17
- --lr: Learning rate (default: 3e-4)
18
- --save-every: Save checkpoint every N epochs (default: 5)
19
- """
20
-
21
- from pathlib import Path
22
- from datetime import datetime
23
- import argparse
24
- import copy
25
- import random
26
- import numpy as np
27
- import json
28
-
29
- from tqdm.auto import tqdm
30
- import torch
31
- from torch.nn import functional as F
32
- from torch.utils.data import DataLoader, random_split
33
- import accelerate
34
- from accelerate.utils import set_seed
35
- from torch.optim.lr_scheduler import LambdaLR
36
-
37
- from chesstransformer.datasets.h5_lichess_dataset import HDF5ChessDataset
38
- from chesstransformer.models.transformer.position2move import Position2MoveModel
39
-
40
-
41
- def get_next_run_number(log_dir="logs"):
42
- """Get the next run number by checking existing run directories."""
43
- log_path = Path(log_dir)
44
- if not log_path.exists():
45
- return 1
46
-
47
- existing_runs = [d.name for d in log_path.iterdir() if d.is_dir() and d.name.startswith("run_")]
48
-
49
- if not existing_runs:
50
- return 1
51
-
52
- run_numbers = []
53
- for run_name in existing_runs:
54
- try:
55
- num = int(run_name.split("_")[1])
56
- run_numbers.append(num)
57
- except (IndexError, ValueError):
58
- continue
59
-
60
- return max(run_numbers) + 1 if run_numbers else 1
61
-
62
-
63
- def compute_loss(logits, moves, legal_moves_mask, beta, legal_move_loss_weight):
64
- llm_loss = F.cross_entropy(logits, moves)
65
- illegal_prob = F.softmax(logits, dim=-1) * (1.0 - legal_moves_mask.float())
66
- legal_move_loss = torch.sum(illegal_prob**2, dim=-1).mean()
67
- filtered_logits = torch.where(legal_moves_mask.bool(), logits, -torch.inf)
68
- legal_move_cross_entropy = F.cross_entropy(filtered_logits, moves)
69
- loss = beta * llm_loss + legal_move_loss_weight * legal_move_loss + (1 - beta) * legal_move_cross_entropy
70
- return loss, llm_loss, legal_move_loss, legal_move_cross_entropy
71
-
72
-
73
- def save_trainer_state(checkpoint_dir, epoch, global_step, best_val_loss, scheduler_config=None):
74
- state = {
75
- "epoch": epoch,
76
- "global_step": global_step,
77
- "best_val_loss": best_val_loss,
78
- }
79
- if scheduler_config:
80
- state["scheduler_config"] = scheduler_config
81
- state_path = Path(checkpoint_dir) / "trainer_state.json"
82
- with state_path.open("w", encoding="utf-8") as fp:
83
- json.dump(state, fp)
84
-
85
-
86
- def load_trainer_state(checkpoint_dir):
87
- state_path = Path(checkpoint_dir) / "trainer_state.json"
88
- if not state_path.exists():
89
- return None
90
- with state_path.open("r", encoding="utf-8") as fp:
91
- return json.load(fp)
92
-
93
-
94
- def cleanup_old_step_checkpoints(checkpoint_dir, max_checkpoints):
95
- """Remove old step checkpoints, keeping only the most recent ones."""
96
- checkpoint_path = Path(checkpoint_dir)
97
- step_checkpoints = sorted(
98
- [d for d in checkpoint_path.iterdir() if d.is_dir() and d.name.startswith("checkpoint_step_")],
99
- key=lambda x: int(x.name.split("_")[-1]),
100
- )
101
-
102
- # Remove oldest checkpoints if we exceed max_checkpoints
103
- while len(step_checkpoints) > max_checkpoints:
104
- oldest = step_checkpoints.pop(0)
105
- import shutil
106
-
107
- shutil.rmtree(oldest)
108
- print(f" Removed old checkpoint: {oldest.name}")
109
-
110
-
111
- def create_linear_warmup_decay_scheduler(optimizer, warmup_steps, total_steps, final_lr_ratio):
112
- final_lr_ratio = max(0.0, final_lr_ratio)
113
- warmup_steps = max(1, warmup_steps)
114
- total_steps = max(warmup_steps + 1, total_steps)
115
-
116
- def lr_lambda(current_step):
117
- if current_step < warmup_steps:
118
- return float(current_step + 1) / float(warmup_steps)
119
- progress = (current_step - warmup_steps + 1) / float(max(1, total_steps - warmup_steps))
120
- return 1.0 - (1.0 - final_lr_ratio) * min(1.0, progress)
121
-
122
- return LambdaLR(optimizer, lr_lambda)
123
-
124
-
125
- def find_best_learning_rate(
126
- model, train_loader, accelerator, lr_min=1e-6, lr_max=1e-2, num_steps=100, beta=0.7, legal_move_loss_weight=10.0
127
- ):
128
- model.train()
129
- device = accelerator.device
130
- initial_state = copy.deepcopy(model.state_dict())
131
- # Use base learning rate for LR finder
132
- optimizer = torch.optim.AdamW(model.parameters(), lr=lr_min)
133
- multiplier = (lr_max / lr_min) ** (1 / max(num_steps - 1, 1))
134
- best_lr = lr_min
135
- best_loss = float("inf")
136
- lr = lr_min
137
- data_iter = iter(train_loader)
138
-
139
- for step in range(num_steps):
140
- try:
141
- batch = next(data_iter)
142
- except StopIteration:
143
- data_iter = iter(train_loader)
144
- batch = next(data_iter)
145
-
146
- optimizer.zero_grad()
147
- positions = batch["position"].to(device)
148
- moves = batch["move"].to(device)
149
- is_white = batch["is_white"].to(device)
150
- legal_moves_mask = batch["legal_moves_mask"].to(device)
151
- castling_rights = batch["castling_rights"].to(device)
152
- en_passant_file = batch["en_passant_file"].to(device)
153
- halfmove_clock = batch["halfmove_clock"].to(device)
154
-
155
- logits = model(positions, is_white, castling_rights, en_passant_file, halfmove_clock)
156
- loss, llm_loss, legal_move_loss, legal_move_cross_entropy = compute_loss(
157
- logits, moves, legal_moves_mask, beta, legal_move_loss_weight
158
- )
159
-
160
- accelerator.backward(loss)
161
- optimizer.step()
162
-
163
- loss_value = accelerator.gather_for_metrics(loss.detach()).mean().item()
164
- if loss_value < best_loss:
165
- best_loss = loss_value
166
- best_lr = lr
167
-
168
- lr *= multiplier
169
- for param_group in optimizer.param_groups:
170
- param_group["lr"] = lr
171
-
172
- model.load_state_dict(initial_state)
173
- model.zero_grad(set_to_none=True)
174
- accelerator.free_memory()
175
- return best_lr, best_loss
176
-
177
-
178
- def main():
179
- default_data_path = Path(__file__).parents[2] / "data" / "Lichess2017.h5"
180
-
181
- parser = argparse.ArgumentParser(description="Train Position2Move Transformer")
182
- parser.add_argument("--data", type=str, default=default_data_path, help="Path to HDF5 dataset")
183
- parser.add_argument("--min-elo", type=int, default=None, help="Minimum average ELO for filtering games")
184
- parser.add_argument("--max-elo", type=int, default=None, help="Maximum average ELO for filtering games")
185
- parser.add_argument("--batch-size", type=int, default=32, help="Batch size")
186
- parser.add_argument("--epochs", type=int, default=10, help="Number of epochs")
187
- parser.add_argument("--lr", type=float, default=3e-4, help="Learning rate for transformers (base LR)")
188
- parser.add_argument(
189
- "--lr-embedding", type=float, default=None, help="Learning rate for embeddings (default: same as --lr)"
190
- )
191
- parser.add_argument(
192
- "--lr-head", type=float, default=None, help="Learning rate for output head (default: same as --lr)"
193
- )
194
- parser.add_argument("--save-every", type=int, default=5, help="Save checkpoint every N epochs")
195
- parser.add_argument("--lr-find", action="store_true", help="Run learning rate finder before training")
196
- parser.add_argument("--lr-min", type=float, default=1e-6, help="Minimum LR for finder sweep")
197
- parser.add_argument("--lr-max", type=float, default=1e-2, help="Maximum LR for finder sweep")
198
- parser.add_argument("--lr-steps", type=int, default=100, help="Number of steps for LR finder sweep")
199
- parser.add_argument("--beta", type=float, default=0.7, help="Weight for LLM loss in combined loss function")
200
- parser.add_argument(
201
- "--legal-move-loss-weight",
202
- type=float,
203
- default=10.0,
204
- help="Weight for legal move loss in combined loss function",
205
- )
206
- parser.add_argument("--random-seed", type=int, default=42, help="Random seed for reproducibility")
207
- parser.add_argument("--warmup-steps", type=int, default=1000, help="Number of warmup steps for LR scheduler")
208
- parser.add_argument("--final-lr-ratio", type=float, default=0.01, help="Final LR multiplier after linear decay")
209
- parser.add_argument("--resume-from", type=str, default=None, help="Path to checkpoint directory to resume from")
210
- parser.add_argument(
211
- "--max-grad-norm", type=float, default=1.0, help="Maximum gradient norm for clipping (0 to disable)"
212
- )
213
- parser.add_argument(
214
- "--max_examples", type=int, default=None, help="Maximum number of examples to use from the dataset"
215
- )
216
- parser.add_argument("--save-steps", type=int, default=5000, help="Save checkpoint every N steps")
217
- parser.add_argument(
218
- "--max-checkpoints",
219
- type=int,
220
- default=5,
221
- help="Maximum number of step checkpoints to keep (excluding best model)",
222
- )
223
- parser.add_argument(
224
- "--precision", type=str, default="fp16", choices=["bf16", "fp16", "fp32"], help="Precision for training"
225
- )
226
- args = parser.parse_args()
227
-
228
- set_seed(args.random_seed)
229
- random.seed(args.random_seed)
230
- np.random.seed(args.random_seed)
231
- torch.manual_seed(args.random_seed)
232
- torch.cuda.manual_seed_all(args.random_seed)
233
-
234
- # Setup dataset and dataloaders
235
- dataset = HDF5ChessDataset(
236
- hdf5_path=args.data,
237
- min_elo=args.min_elo,
238
- max_elo=args.max_elo,
239
- )
240
-
241
- MAX_VALIDATION_SAMPLES = 10000
242
-
243
- val_size = min(int(0.1 * len(dataset)), MAX_VALIDATION_SAMPLES)
244
- test_size = min(int(0.1 * len(dataset)), MAX_VALIDATION_SAMPLES)
245
- train_size = len(dataset) - val_size - test_size
246
- train_set, val_set, test_set = random_split(dataset, [train_size, val_size, test_size])
247
-
248
- train_loader = DataLoader(train_set, batch_size=args.batch_size, shuffle=True, num_workers=12, drop_last=True)
249
- val_loader = DataLoader(val_set, batch_size=args.batch_size, shuffle=False, num_workers=12)
250
- test_loader = DataLoader(test_set, batch_size=args.batch_size, shuffle=False, num_workers=12)
251
-
252
- print(f"Train samples: {len(train_set)}, Val samples: {len(val_set)}, Test samples: {len(test_set)}")
253
-
254
- # Model configuration
255
- model_config = {
256
- "vocab_size": dataset.position_tokenizer.vocab_size,
257
- "move_vocab_size": dataset.move_tokenizer.vocab_size,
258
- "embed_dim": 512,
259
- "nb_transformer_layers": 8,
260
- "num_heads": 8,
261
- "num_kv_groups": 4, # GQA: 8 heads / 4 groups = 2 heads per KV group
262
- "dropout": 0.1,
263
- "kvq_bias": False,
264
- "mask_future": False,
265
- "rope": True, # Use RoPE for positional encoding
266
- "use_swiglu": True, # Use SwiGLU activation (Llama-style)
267
- "use_col_row_emb": True, # Use separate column and row embeddings
268
- }
269
-
270
- # Setup logging
271
- log_dir = Path("logs") / "position2move"
272
- log_dir.mkdir(parents=True, exist_ok=True)
273
- run_number = get_next_run_number(str(log_dir))
274
- log_dir_full = f"run_{run_number:03d}_{datetime.now().strftime('%Y%m%d_%H%M%S')}"
275
- log_path = log_dir / log_dir_full
276
- log_path.mkdir(parents=True, exist_ok=False)
277
- print(f"Logging to: {log_path}")
278
-
279
- # Create checkpoints directory
280
- checkpoint_dir = log_path / "checkpoints"
281
- checkpoint_dir.mkdir(exist_ok=True)
282
-
283
- # Save model config to JSON
284
- config_path = log_path / "model_config.json"
285
- with config_path.open("w", encoding="utf-8") as f:
286
- json.dump(model_config, f, indent=2)
287
- print(f"Model config saved to: {config_path}")
288
-
289
- precision = args.precision if torch.cuda.is_available() or torch.backends.mps.is_available() else "fp32"
290
- print(f"Using precision: {precision}")
291
-
292
- accelerator = accelerate.Accelerator(log_with="tensorboard", project_dir=str(log_path), mixed_precision=precision)
293
-
294
- # Initialize trackers
295
- accelerator.init_trackers(
296
- project_name="position2move",
297
- config={
298
- "batch_size": args.batch_size,
299
- "epochs": args.epochs,
300
- "learning_rate": args.lr,
301
- "lr_embedding": args.lr_embedding if args.lr_embedding is not None else args.lr,
302
- "lr_head": args.lr_head if args.lr_head is not None else args.lr,
303
- "min_elo": args.min_elo,
304
- "max_elo": args.max_elo,
305
- "max_grad_norm": args.max_grad_norm,
306
- **model_config,
307
- "warmup_steps": args.warmup_steps,
308
- "final_lr_ratio": args.final_lr_ratio,
309
- "precision": args.precision,
310
- },
311
- )
312
-
313
- device = accelerator.device
314
- print(f"Using device: {device}")
315
-
316
- model = Position2MoveModel(**model_config)
317
-
318
- # torch.compile has limited support on MPS backend - disable if compilation fails
319
- if device.type == "mps":
320
- print("Warning: Skipping torch.compile on MPS device (Mac GPU) due to backend limitations")
321
- print("Training will proceed without compilation optimizations")
322
- else:
323
- try:
324
- model = torch.compile(
325
- model,
326
- )
327
- print("Model compiled successfully")
328
- except Exception as e:
329
- print(f"Warning: torch.compile failed ({e}), proceeding without compilation")
330
-
331
- model.to(device)
332
-
333
- if args.lr_find:
334
- if accelerator.is_main_process:
335
- print("Running learning rate finder...")
336
- best_lr, best_loss = find_best_learning_rate(
337
- model,
338
- train_loader,
339
- accelerator,
340
- lr_min=args.lr_min,
341
- lr_max=args.lr_max,
342
- num_steps=args.lr_steps,
343
- beta=args.beta,
344
- legal_move_loss_weight=args.legal_move_loss_weight,
345
- )
346
- accelerator.log({"lr_finder/best_lr": best_lr, "lr_finder/best_loss": best_loss}, step=0)
347
- args.lr = best_lr
348
- if accelerator.is_main_process:
349
- print(f"Best LR found: {best_lr:.6g} (loss {best_loss:.4f})")
350
-
351
- # Set up discriminative learning rates
352
- lr_embedding = args.lr_embedding if args.lr_embedding is not None else args.lr
353
- lr_head = args.lr_head if args.lr_head is not None else args.lr
354
- lr_transformer = args.lr
355
-
356
- # Create parameter groups
357
- embedding_params = []
358
- transformer_params = []
359
- head_params = []
360
-
361
- # Get underlying model (unwrap from accelerator if needed)
362
- unwrapped_model = accelerator.unwrap_model(model)
363
-
364
- # Categorize parameters
365
- for name, param in unwrapped_model.named_parameters():
366
- if "embedding" in name:
367
- embedding_params.append(param)
368
- elif "lm_head" in name:
369
- head_params.append(param)
370
- else:
371
- transformer_params.append(param)
372
-
373
- param_groups = [
374
- {"params": embedding_params, "lr": lr_embedding, "name": "embeddings"},
375
- {"params": transformer_params, "lr": lr_transformer, "name": "transformers"},
376
- {"params": head_params, "lr": lr_head, "name": "head"},
377
- ]
378
-
379
- if accelerator.is_main_process:
380
- emb_count = sum(p.numel() for p in embedding_params)
381
- trans_count = sum(p.numel() for p in transformer_params)
382
- head_count = sum(p.numel() for p in head_params)
383
- print(f"\nLearning rates:")
384
- print(f" Embeddings: {lr_embedding:.6g}")
385
- print(f" Transformers: {lr_transformer:.6g}")
386
- print(f" Head: {lr_head:.6g}")
387
- print(f" Param counts: Emb={emb_count:,}, Trans={trans_count:,}, Head={head_count:,}\n")
388
-
389
- optimizer = torch.optim.AdamW(param_groups, lr=args.lr)
390
- model, optimizer, train_loader, val_loader, test_loader = accelerator.prepare(
391
- model, optimizer, train_loader, val_loader, test_loader
392
- )
393
-
394
- # Check if resuming to load scheduler config from checkpoint
395
- best_val_loss = float("inf")
396
- start_epoch = 1
397
- global_step = 0
398
- resume_path = Path(args.resume_from) if args.resume_from else None
399
- trainer_state = None
400
-
401
- if resume_path:
402
- trainer_state = load_trainer_state(resume_path)
403
- if trainer_state:
404
- start_epoch = trainer_state.get("epoch", 0) + 1
405
- global_step = trainer_state.get("global_step", 0)
406
- best_val_loss = trainer_state.get("best_val_loss", float("inf"))
407
-
408
- # Determine scheduler parameters - use saved config when resuming for consistency
409
- has_scheduler_config = trainer_state and "scheduler_config" in trainer_state
410
-
411
- if has_scheduler_config:
412
- saved_config = trainer_state["scheduler_config"]
413
- scheduler_warmup_steps = saved_config["warmup_steps"]
414
- scheduler_total_steps = saved_config["total_steps"]
415
- scheduler_final_lr_ratio = saved_config["final_lr_ratio"]
416
- if accelerator.is_main_process:
417
- print(
418
- f"Using scheduler config from checkpoint: warmup={scheduler_warmup_steps}, total={scheduler_total_steps}, final_lr_ratio={scheduler_final_lr_ratio}"
419
- )
420
- else:
421
- scheduler_warmup_steps = args.warmup_steps
422
- scheduler_total_steps = max(1, len(train_loader)) * max(1, args.epochs)
423
- scheduler_final_lr_ratio = args.final_lr_ratio
424
- if resume_path and accelerator.is_main_process:
425
- print(
426
- f"Old checkpoint without scheduler_config - using new scheduler params: warmup={scheduler_warmup_steps}, total={scheduler_total_steps}"
427
- )
428
-
429
- # Store scheduler config for saving in checkpoints
430
- scheduler_config = {
431
- "warmup_steps": scheduler_warmup_steps,
432
- "total_steps": scheduler_total_steps,
433
- "final_lr_ratio": scheduler_final_lr_ratio,
434
- }
435
-
436
- scheduler = create_linear_warmup_decay_scheduler(
437
- optimizer,
438
- warmup_steps=scheduler_warmup_steps,
439
- total_steps=scheduler_total_steps,
440
- final_lr_ratio=scheduler_final_lr_ratio,
441
- )
442
-
443
- # Always register scheduler for checkpointing (required for load_state to work)
444
- accelerator.register_for_checkpointing(scheduler)
445
-
446
- # Load the full accelerator state (model, optimizer, scheduler) if resuming
447
- if resume_path:
448
- accelerator.load_state(str(resume_path))
449
-
450
- # For old checkpoints without scheduler_config, the loaded scheduler state
451
- # has a mismatched lr_lambda (old total_steps). We need to recreate it.
452
- if not has_scheduler_config:
453
- # Recreate scheduler with correct parameters
454
- scheduler = create_linear_warmup_decay_scheduler(
455
- optimizer,
456
- warmup_steps=scheduler_warmup_steps,
457
- total_steps=scheduler_total_steps,
458
- final_lr_ratio=scheduler_final_lr_ratio,
459
- )
460
- # Manually advance to the correct step
461
- scheduler.last_epoch = global_step - 1 # -1 because step() will increment it
462
- scheduler.step() # This sets it to global_step and computes correct LR
463
- if accelerator.is_main_process:
464
- current_lrs = scheduler.get_last_lr()
465
- print(f" Recreated scheduler and advanced to step {global_step}")
466
- print(f" Current LRs: emb={current_lrs[0]:.6g}, trans={current_lrs[1]:.6g}, head={current_lrs[2]:.6g}")
467
-
468
- if accelerator.is_main_process:
469
- print(f"Resumed from checkpoint: {resume_path}")
470
- print(f" Starting from epoch {start_epoch}, global_step {global_step}, best_val_loss {best_val_loss:.4f}")
471
-
472
- for epoch in range(start_epoch, args.epochs + 1):
473
- # Training phase
474
- model.train()
475
- train_loss = 0.0
476
- train_llm_loss = 0.0
477
- train_legal_move_loss = 0.0
478
- train_legal_move_ce = 0.0
479
-
480
- progress_bar = tqdm(train_loader, desc=f"Epoch {epoch}/{args.epochs} - Training", leave=False)
481
- for batch in progress_bar:
482
- optimizer.zero_grad()
483
- positions = batch["position"]
484
- moves = batch["move"]
485
- is_white = batch["is_white"]
486
- legal_moves_mask = batch["legal_moves_mask"]
487
- castling_rights = batch["castling_rights"]
488
- en_passant_file = batch["en_passant_file"]
489
- halfmove_clock = batch["halfmove_clock"]
490
-
491
- logits = model(positions, is_white, castling_rights, en_passant_file, halfmove_clock)
492
- loss, llm_loss, legal_move_loss, legal_move_cross_entropy = compute_loss(
493
- logits, moves, legal_moves_mask, args.beta, args.legal_move_loss_weight
494
- )
495
-
496
- accelerator.backward(loss)
497
-
498
- # Gradient clipping
499
- if args.max_grad_norm > 0:
500
- accelerator.clip_grad_norm_(model.parameters(), args.max_grad_norm)
501
-
502
- optimizer.step()
503
- scheduler.step()
504
-
505
- batch_size = positions.size(0)
506
- train_loss += loss.item() * batch_size
507
- train_llm_loss += llm_loss.item() * batch_size
508
- train_legal_move_loss += legal_move_loss.item() * batch_size
509
- train_legal_move_ce += legal_move_cross_entropy.item() * batch_size
510
-
511
- progress_bar.set_postfix(
512
- {
513
- "loss": loss.item(),
514
- "llm_loss": llm_loss.item(),
515
- "legal_move_loss": legal_move_loss.item(),
516
- "legal_move_ce": legal_move_cross_entropy.item(),
517
- }
518
- )
519
-
520
- # Log step-level metrics
521
- current_lrs = scheduler.get_last_lr()
522
- log_dict = {
523
- "train/step_loss": loss.item(),
524
- "train/step_llm_loss": llm_loss.item(),
525
- "train/step_legal_move_loss": legal_move_loss.item(),
526
- "train/step_legal_move_ce": legal_move_cross_entropy.item(),
527
- "train/lr_embeddings": current_lrs[0],
528
- "train/lr_transformers": current_lrs[1],
529
- "train/lr_head": current_lrs[2],
530
- }
531
- accelerator.log(log_dict, step=global_step)
532
-
533
- global_step += 1
534
-
535
- # Save step checkpoint
536
- if args.save_steps > 0 and global_step % args.save_steps == 0:
537
- step_checkpoint_path = checkpoint_dir / f"checkpoint_step_{global_step:07d}"
538
- accelerator.save_state(str(step_checkpoint_path))
539
- save_trainer_state(step_checkpoint_path, epoch, global_step, best_val_loss, scheduler_config)
540
- if accelerator.is_main_process:
541
- print(f"\n Saved step checkpoint at step {global_step}")
542
- cleanup_old_step_checkpoints(checkpoint_dir, args.max_checkpoints)
543
-
544
- # Calculate average training metrics
545
- train_loss /= len(train_set)
546
- train_llm_loss /= len(train_set)
547
- train_legal_move_loss /= len(train_set)
548
- train_legal_move_ce /= len(train_set)
549
-
550
- # Validation phase
551
- model.eval()
552
- val_loss = 0.0
553
- val_llm_loss = 0.0
554
- val_legal_move_loss = 0.0
555
- val_legal_move_ce = 0.0
556
- val_correct = 0
557
- val_total = 0
558
-
559
- with torch.no_grad():
560
- progress_bar = tqdm(val_loader, desc=f"Epoch {epoch}/{args.epochs} - Validation", leave=False)
561
- for batch in progress_bar:
562
- positions = batch["position"]
563
- moves = batch["move"]
564
- is_white = batch["is_white"]
565
- legal_moves_mask = batch["legal_moves_mask"]
566
- castling_rights = batch["castling_rights"]
567
- en_passant_file = batch["en_passant_file"]
568
- halfmove_clock = batch["halfmove_clock"]
569
-
570
- logits = model(positions, is_white, castling_rights, en_passant_file, halfmove_clock)
571
- loss, llm_loss, legal_move_loss, legal_move_cross_entropy = compute_loss(
572
- logits, moves, legal_moves_mask, args.beta, args.legal_move_loss_weight
573
- )
574
-
575
- batch_size = positions.size(0)
576
- val_loss += loss.item() * batch_size
577
- val_llm_loss += llm_loss.item() * batch_size
578
- val_legal_move_loss += legal_move_loss.item() * batch_size
579
- val_legal_move_ce += legal_move_cross_entropy.item() * batch_size
580
-
581
- # Calculate accuracy
582
- predictions = torch.argmax(logits, dim=-1)
583
- val_correct += (predictions == moves).sum().item()
584
- val_total += batch_size
585
-
586
- progress_bar.set_postfix({"val_loss": loss.item()})
587
-
588
- val_loss /= len(val_set)
589
- val_llm_loss /= len(val_set)
590
- val_legal_move_loss /= len(val_set)
591
- val_legal_move_ce /= len(val_set)
592
- val_accuracy = val_correct / val_total if val_total > 0 else 0.0
593
-
594
- # Log epoch-level metrics
595
- accelerator.log(
596
- {
597
- "train/epoch_loss": train_loss,
598
- "train/epoch_llm_loss": train_llm_loss,
599
- "train/epoch_legal_move_loss": train_legal_move_loss,
600
- "train/epoch_legal_move_ce": train_legal_move_ce,
601
- "val/loss": val_loss,
602
- "val/llm_loss": val_llm_loss,
603
- "val/legal_move_loss": val_legal_move_loss,
604
- "val/legal_move_ce": val_legal_move_ce,
605
- "val/accuracy": val_accuracy,
606
- "epoch": epoch,
607
- "train/lr_embeddings_epoch_end": scheduler.get_last_lr()[0],
608
- "train/lr_transformers_epoch_end": scheduler.get_last_lr()[1],
609
- "train/lr_head_epoch_end": scheduler.get_last_lr()[2],
610
- },
611
- step=global_step,
612
- )
613
-
614
- print(f"\nEpoch {epoch}/{args.epochs}:")
615
- print(f" Train Loss: {train_loss:.4f} | Val Loss: {val_loss:.4f} | Val Accuracy: {val_accuracy:.4f}")
616
-
617
- # Save best model
618
- if val_loss < best_val_loss:
619
- best_val_loss = val_loss
620
- best_checkpoint_path = checkpoint_dir / "best_model"
621
- accelerator.save_state(str(best_checkpoint_path))
622
- save_trainer_state(best_checkpoint_path, epoch, global_step, best_val_loss, scheduler_config)
623
- print(f" Saved best model (val_loss: {val_loss:.4f})")
624
-
625
- # Periodic checkpoint saving
626
- if epoch % args.save_every == 0:
627
- checkpoint_path = checkpoint_dir / f"checkpoint_epoch_{epoch:03d}"
628
- accelerator.save_state(str(checkpoint_path))
629
- save_trainer_state(checkpoint_path, epoch, global_step, best_val_loss, scheduler_config)
630
- print(f" Saved checkpoint at epoch {epoch}")
631
-
632
- # Final test evaluation
633
- print("\nRunning final test evaluation...")
634
- model.eval()
635
- test_loss = 0.0
636
- test_correct = 0
637
- test_total = 0
638
-
639
- with torch.no_grad():
640
- progress_bar = tqdm(test_loader, desc="Testing", leave=False)
641
- for batch in progress_bar:
642
- positions = batch["position"]
643
- moves = batch["move"]
644
- is_white = batch["is_white"]
645
- legal_moves_mask = batch["legal_moves_mask"]
646
- castling_rights = batch["castling_rights"]
647
- en_passant_file = batch["en_passant_file"]
648
- halfmove_clock = batch["halfmove_clock"]
649
-
650
- logits = model(positions, is_white, castling_rights, en_passant_file, halfmove_clock)
651
- loss, llm_loss, legal_move_loss, legal_move_cross_entropy = compute_loss(
652
- logits, moves, legal_moves_mask, args.beta, args.legal_move_loss_weight
653
- )
654
-
655
- batch_size = positions.size(0)
656
- test_loss += loss.item() * batch_size
657
-
658
- predictions = torch.argmax(logits, dim=-1)
659
- test_correct += (predictions == moves).sum().item()
660
- test_total += batch_size
661
-
662
- test_loss /= len(test_set)
663
- test_accuracy = test_correct / test_total if test_total > 0 else 0.0
664
-
665
- accelerator.log(
666
- {
667
- "test/loss": test_loss,
668
- "test/accuracy": test_accuracy,
669
- },
670
- step=global_step,
671
- )
672
-
673
- print(f"\nTest Results:")
674
- print(f" Test Loss: {test_loss:.4f} | Test Accuracy: {test_accuracy:.4f}")
675
-
676
- # End tracking
677
- accelerator.end_training()
678
- print(f"\nTraining complete! Logs saved to: {log_path}")
679
-
680
-
681
- if __name__ == "__main__":
682
- main()
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
chesstransformer/utils/__init__.py DELETED
@@ -1,3 +0,0 @@
1
- from .chess_utils import generate_board_image, generate_board_ascii, generate_game_gif
2
-
3
- __all__ = ["generate_board_image", "generate_board_ascii", "generate_game_gif"]
 
 
 
 
chesstransformer/utils/chess_utils.py DELETED
@@ -1,123 +0,0 @@
1
- import chess
2
- from PIL import Image
3
- from io import BytesIO
4
- import cairosvg
5
-
6
-
7
- def _create_board_from_moves(moves: list[str]) -> chess.Board:
8
- """
9
- Create a chess board and apply a sequence of moves.
10
-
11
- Args:
12
- moves: List of moves in UCI format (e.g., ['e2e4', 'd7d5'])
13
-
14
- Returns:
15
- Chess board with moves applied
16
- """
17
- board = chess.Board()
18
- for move_str in moves:
19
- try:
20
- move = chess.Move.from_uci(move_str)
21
- board.push(move)
22
- except ValueError:
23
- pass # Skip invalid moves
24
- return board
25
-
26
-
27
- def generate_board_image(moves: list[str], size: int = 400) -> Image.Image:
28
- """
29
- Generate a PIL image of a chess board after applying a sequence of moves.
30
-
31
- Args:
32
- moves: List of moves in UCI format (e.g., ['e2e4', 'd7d5'])
33
- size: Size of the output image in pixels (default: 400)
34
-
35
- Returns:
36
- PIL Image of the chess board
37
- """
38
- import chess.svg
39
-
40
- board = _create_board_from_moves(moves)
41
-
42
- # Generate SVG using chess library
43
- svg_data = chess.svg.board(board, size=size)
44
-
45
- # Convert SVG to PNG using cairosvg
46
- png_data = cairosvg.svg2png(bytestring=svg_data.encode("utf-8"))
47
-
48
- # Load PNG into PIL Image
49
- img = Image.open(BytesIO(png_data))
50
-
51
- return img
52
-
53
-
54
- def generate_board_ascii(moves: list[str]) -> str:
55
- """
56
- Generate an ASCII representation of a chess board after applying a sequence of moves.
57
-
58
- Args:
59
- moves: List of moves in UCI format (e.g., ['e2e4', 'd7d5'])
60
-
61
- Returns:
62
- String representation of the chess board
63
- """
64
- board = _create_board_from_moves(moves)
65
- return str(board)
66
-
67
-
68
- def generate_game_gif(moves: list[str], size: int = 400, duration: int = 500) -> bytes:
69
- """
70
- Generate an animated GIF of a chess game from a sequence of moves.
71
-
72
- Args:
73
- moves: List of moves in UCI format (e.g., ['e2e4', 'd7d5'])
74
- size: Size of each frame in pixels (default: 400)
75
- duration: Duration of each frame in milliseconds (default: 500)
76
-
77
- Returns:
78
- Bytes of the animated GIF
79
- """
80
- frames = []
81
-
82
- # Initial position
83
- img = generate_board_image([], size)
84
- frames.append(img)
85
-
86
- # Apply moves and capture frames
87
- moves_so_far = []
88
- for move_str in moves:
89
- try:
90
- move = chess.Move.from_uci(move_str)
91
- moves_so_far.append(move_str)
92
- img = generate_board_image(moves_so_far, size)
93
- frames.append(img)
94
- except ValueError:
95
- pass # Skip invalid moves
96
-
97
- # Save frames as GIF in memory
98
- gif_bytes = BytesIO()
99
- frames[0].save(
100
- gif_bytes,
101
- format="GIF",
102
- save_all=True,
103
- append_images=frames[1:],
104
- duration=duration,
105
- loop=0,
106
- )
107
- gif_bytes.seek(0)
108
-
109
- return gif_bytes.getvalue()
110
-
111
-
112
- if __name__ == "__main__":
113
- moves = ["e2e4"]
114
- img = generate_board_image(moves)
115
- img.show()
116
-
117
- ascii_board = generate_board_ascii(moves)
118
- print(ascii_board)
119
-
120
- coup_du_berget = ["e2e4", "e7e5", "f1c4", "b8c6", "d1h5", "g8f6", "h5f7"]
121
- gif = generate_game_gif(coup_du_berget)
122
- with open("coup_du_berget.gif", "wb") as f:
123
- f.write(gif)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
chesstransformer/utils/evaluate_puzzles.py DELETED
@@ -1,426 +0,0 @@
1
- #!/usr/bin/env python3
2
- """
3
- Evaluation script for chess puzzle solving with Position2Move model.
4
-
5
- This script evaluates how well a trained model can solve chess puzzles by:
6
- 1. Loading a trained model checkpoint
7
- 2. Testing it on puzzles from the Lichess puzzle database
8
- 3. Measuring accuracy for first move, full solution, and by rating/theme
9
- 4. Generating detailed reports
10
-
11
- Usage:
12
- python src/chesstransformer/utils/evaluate_puzzles.py --model data/models/puzzle_training/run_001/best_model.pth
13
-
14
- Options:
15
- --model: Path to model checkpoint (required)
16
- --puzzle-data: Path to puzzle .csv.zst file (default: data/lichess_db_puzzle.csv.zst)
17
- --num-puzzles: Number of puzzles to evaluate (default: 1000)
18
- --min-rating: Minimum puzzle rating (default: None)
19
- --max-rating: Maximum puzzle rating (default: None)
20
- --themes: Comma-separated puzzle themes to filter by (default: None)
21
- --output: Output JSON file for results (default: results/puzzle_eval_{timestamp}.json)
22
- """
23
-
24
- from pathlib import Path
25
- from datetime import datetime
26
- import argparse
27
- import json
28
-
29
- import torch
30
- import chess
31
- from tqdm.auto import tqdm
32
-
33
- from chesstransformer.datasets.puzzle_dataset import LichessPuzzleFullSolutionDataset
34
- from chesstransformer.models.transformer.position2move import Position2MoveModel
35
- from chesstransformer.models.tokenizer.position_tokenizer import PostionTokenizer
36
- from chesstransformer.models.tokenizer.move_tokenizer import MoveTokenizer
37
-
38
-
39
- class PuzzleEvaluator:
40
- """Evaluates a Position2Move model on chess puzzles."""
41
-
42
- def __init__(self, model, device="cpu"):
43
- self.model = model
44
- self.device = device
45
- self.model.to(device)
46
- self.model.eval()
47
-
48
- self.position_tokenizer = PostionTokenizer()
49
- self.move_tokenizer = MoveTokenizer()
50
-
51
- def predict_move(self, board: chess.Board, top_k=5):
52
- """
53
- Predict the best move for a given board position.
54
-
55
- Args:
56
- board: chess.Board object
57
- top_k: Return top-k predictions
58
-
59
- Returns:
60
- List of (move_uci, probability, is_legal) tuples
61
- """
62
- # Encode position
63
- position_tokens = self.position_tokenizer.encode(board)
64
- position_tensor = torch.tensor(position_tokens, dtype=torch.long).unsqueeze(0).to(self.device)
65
- is_white = torch.tensor([board.turn == chess.WHITE], dtype=torch.long).to(self.device)
66
-
67
- # Get predictions
68
- with torch.no_grad():
69
- logits = self.model(position_tensor, is_white)
70
-
71
- # Create legal moves mask
72
- legal_moves = [m.uci() for m in board.legal_moves]
73
- legal_move_indices = []
74
- for move_uci in legal_moves:
75
- try:
76
- idx = self.move_tokenizer.encode(move_uci)
77
- legal_move_indices.append(idx)
78
- except ValueError:
79
- continue
80
-
81
- # Apply mask to logits (set illegal moves to -inf before softmax)
82
- mask = torch.full_like(logits, float("-inf"))
83
- if legal_move_indices:
84
- mask[0, legal_move_indices] = 0
85
- masked_logits = logits + mask
86
-
87
- probs = torch.softmax(masked_logits, dim=-1)
88
-
89
- # Get top-k predictions
90
- top_probs, top_indices = torch.topk(probs[0], k=top_k)
91
-
92
- predictions = []
93
-
94
- for prob, idx in zip(top_probs.cpu().numpy(), top_indices.cpu().numpy()):
95
- try:
96
- move_uci = self.move_tokenizer.decode(int(idx))
97
- is_legal = move_uci in legal_moves
98
- predictions.append((move_uci, float(prob), is_legal))
99
- except ValueError:
100
- continue
101
-
102
- return predictions
103
-
104
- def evaluate_puzzle(self, puzzle_data, max_moves=10):
105
- """
106
- Evaluate a single puzzle.
107
-
108
- Args:
109
- puzzle_data: Dictionary with puzzle information
110
- max_moves: Maximum number of moves to try in the solution
111
-
112
- Returns:
113
- Dictionary with evaluation results
114
- """
115
- board = chess.Board(puzzle_data["fen"])
116
- solution_moves = puzzle_data["moves_uci"]
117
-
118
- results = {
119
- "puzzle_id": puzzle_data["puzzle_id"],
120
- "rating": puzzle_data["rating"],
121
- "themes": puzzle_data["themes"],
122
- "solution_length": len(solution_moves),
123
- "moves_tried": [],
124
- "first_move_correct": False,
125
- "fully_solved": False,
126
- "moves_correct": 0,
127
- "predictions": [],
128
- }
129
-
130
- # Try to solve the puzzle move by move
131
- for move_idx in range(0, len(solution_moves), 2): # Only predict our moves (every other move)
132
- if move_idx >= max_moves:
133
- break
134
-
135
- # Apply opponent's move first (if not the first move)
136
- if move_idx > 0:
137
- opponent_move = chess.Move.from_uci(solution_moves[move_idx - 1])
138
- if opponent_move in board.legal_moves:
139
- board.push(opponent_move)
140
- else:
141
- # Opponent move is illegal - puzzle data might be corrupted
142
- results["error"] = "Illegal opponent move in solution"
143
- break
144
-
145
- # Predict our move
146
- expected_move = solution_moves[move_idx]
147
- predictions = self.predict_move(board, top_k=5)
148
-
149
- if not predictions:
150
- results["error"] = "Model produced no valid predictions"
151
- break
152
-
153
- predicted_move = predictions[0][0]
154
- predicted_prob = predictions[0][1]
155
- is_correct = predicted_move == expected_move
156
-
157
- results["moves_tried"].append(
158
- {
159
- "move_number": move_idx // 2 + 1,
160
- "expected": expected_move,
161
- "predicted": predicted_move,
162
- "probability": predicted_prob,
163
- "correct": is_correct,
164
- "top_5_predictions": [{"move": m, "prob": p, "legal": l} for m, p, l in predictions],
165
- }
166
- )
167
-
168
- if is_correct:
169
- results["moves_correct"] += 1
170
- if move_idx == 0:
171
- results["first_move_correct"] = True
172
-
173
- # Apply our correct move
174
- our_move = chess.Move.from_uci(predicted_move)
175
- board.push(our_move)
176
- else:
177
- # Wrong move - puzzle failed
178
- break
179
-
180
- # Check if fully solved
181
- if results["moves_correct"] == (len(solution_moves) + 1) // 2:
182
- results["fully_solved"] = True
183
-
184
- return results
185
-
186
- def evaluate_dataset(self, dataset, num_puzzles=None):
187
- """
188
- Evaluate the model on a puzzle dataset.
189
-
190
- Args:
191
- dataset: LichessPuzzleFullSolutionDataset
192
- num_puzzles: Number of puzzles to evaluate (None = all)
193
-
194
- Returns:
195
- Dictionary with aggregate results
196
- """
197
- num_puzzles = min(num_puzzles or len(dataset), len(dataset))
198
-
199
- print(f"Evaluating {num_puzzles} puzzles...")
200
-
201
- results = {
202
- "num_puzzles": num_puzzles,
203
- "first_move_accuracy": 0,
204
- "full_solution_accuracy": 0,
205
- "average_moves_correct": 0,
206
- "by_rating": {},
207
- "by_theme": {},
208
- "puzzle_results": [],
209
- }
210
-
211
- first_move_correct = 0
212
- fully_solved = 0
213
- total_moves_correct = 0
214
- total_moves = 0
215
-
216
- # Track by rating buckets
217
- rating_buckets = {
218
- "0-1000": {"first": 0, "full": 0, "total": 0},
219
- "1000-1500": {"first": 0, "full": 0, "total": 0},
220
- "1500-2000": {"first": 0, "full": 0, "total": 0},
221
- "2000-2500": {"first": 0, "full": 0, "total": 0},
222
- "2500+": {"first": 0, "full": 0, "total": 0},
223
- }
224
-
225
- # Track by theme
226
- theme_stats = {}
227
-
228
- for i in tqdm(range(num_puzzles), desc="Evaluating puzzles"):
229
- puzzle_data = dataset[i]
230
- puzzle_result = self.evaluate_puzzle(puzzle_data)
231
-
232
- # Aggregate statistics
233
- if puzzle_result["first_move_correct"]:
234
- first_move_correct += 1
235
- if puzzle_result["fully_solved"]:
236
- fully_solved += 1
237
-
238
- total_moves_correct += puzzle_result["moves_correct"]
239
- total_moves += puzzle_result["solution_length"] // 2 + 1
240
-
241
- # Rating bucket
242
- rating = puzzle_result["rating"]
243
- if rating < 1000:
244
- bucket = "0-1000"
245
- elif rating < 1500:
246
- bucket = "1000-1500"
247
- elif rating < 2000:
248
- bucket = "1500-2000"
249
- elif rating < 2500:
250
- bucket = "2000-2500"
251
- else:
252
- bucket = "2500+"
253
-
254
- rating_buckets[bucket]["total"] += 1
255
- if puzzle_result["first_move_correct"]:
256
- rating_buckets[bucket]["first"] += 1
257
- if puzzle_result["fully_solved"]:
258
- rating_buckets[bucket]["full"] += 1
259
-
260
- # Theme statistics
261
- for theme in puzzle_result["themes"]:
262
- if theme not in theme_stats:
263
- theme_stats[theme] = {"first": 0, "full": 0, "total": 0}
264
- theme_stats[theme]["total"] += 1
265
- if puzzle_result["first_move_correct"]:
266
- theme_stats[theme]["first"] += 1
267
- if puzzle_result["fully_solved"]:
268
- theme_stats[theme]["full"] += 1
269
-
270
- results["puzzle_results"].append(puzzle_result)
271
-
272
- # Calculate aggregate metrics
273
- results["first_move_accuracy"] = 100.0 * first_move_correct / num_puzzles
274
- results["full_solution_accuracy"] = 100.0 * fully_solved / num_puzzles
275
- results["average_moves_correct"] = total_moves_correct / num_puzzles
276
- results["move_accuracy"] = 100.0 * total_moves_correct / total_moves if total_moves > 0 else 0
277
-
278
- # Rating bucket statistics
279
- for bucket, stats in rating_buckets.items():
280
- if stats["total"] > 0:
281
- results["by_rating"][bucket] = {
282
- "count": stats["total"],
283
- "first_move_accuracy": 100.0 * stats["first"] / stats["total"],
284
- "full_solution_accuracy": 100.0 * stats["full"] / stats["total"],
285
- }
286
-
287
- # Theme statistics (top 20 themes by frequency)
288
- sorted_themes = sorted(theme_stats.items(), key=lambda x: x[1]["total"], reverse=True)[:20]
289
- for theme, stats in sorted_themes:
290
- results["by_theme"][theme] = {
291
- "count": stats["total"],
292
- "first_move_accuracy": 100.0 * stats["first"] / stats["total"],
293
- "full_solution_accuracy": 100.0 * stats["full"] / stats["total"],
294
- }
295
-
296
- return results
297
-
298
-
299
- def main():
300
- parser = argparse.ArgumentParser(description="Evaluate Position2Move model on chess puzzles")
301
- parser.add_argument("--model", type=str, required=True, help="Path to model checkpoint")
302
- parser.add_argument(
303
- "--puzzle-data", type=str, default="data/lichess_db_puzzle.csv.zst", help="Path to puzzle .csv.zst file"
304
- )
305
- parser.add_argument("--num-puzzles", type=int, default=1000, help="Number of puzzles to evaluate")
306
- parser.add_argument("--min-rating", type=int, default=None, help="Minimum puzzle rating")
307
- parser.add_argument("--max-rating", type=int, default=None, help="Maximum puzzle rating")
308
- parser.add_argument("--themes", type=str, default=None, help="Comma-separated puzzle themes to filter by")
309
- parser.add_argument("--output", type=str, default=None, help="Output JSON file for results")
310
-
311
- args = parser.parse_args()
312
-
313
- # Parse themes
314
- themes = args.themes.split(",") if args.themes else None
315
-
316
- # Setup device
317
- device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
318
- print(f"Using device: {device}")
319
-
320
- # Load model
321
- print(f"\nLoading model from {args.model}...")
322
- model_path = Path(args.model)
323
-
324
- # Support both .pth and .safetensors formats
325
- if model_path.suffix == ".safetensors":
326
- # Load config from same directory
327
- from safetensors import safe_open
328
- import json
329
-
330
- config_path = model_path.parent / "config.json"
331
- with open(config_path, "r") as f:
332
- config = json.load(f)
333
-
334
- model = Position2MoveModel(**config)
335
-
336
- # Load weights from safetensors
337
- with safe_open(str(model_path), framework="pt", device=str(device)) as f:
338
- state_dict = {k: f.get_tensor(k) for k in f.keys()}
339
-
340
- # Handle compiled model prefix (_orig_mod.)
341
- if any(k.startswith("_orig_mod.") for k in state_dict.keys()):
342
- state_dict = {k.replace("_orig_mod.", ""): v for k, v in state_dict.items()}
343
-
344
- model.load_state_dict(state_dict)
345
-
346
- print(f"Model loaded from safetensors")
347
- else:
348
- # Load from .pth checkpoint
349
- checkpoint = torch.load(args.model, map_location=device, weights_only=False)
350
- config = checkpoint["config"]
351
-
352
- model = Position2MoveModel(**config)
353
- model.load_state_dict(checkpoint["model_state_dict"])
354
- print(f"Model loaded (epoch {checkpoint.get('epoch', 'unknown')})")
355
-
356
- # Load puzzle dataset
357
- print(f"\nLoading puzzle dataset...")
358
- dataset = LichessPuzzleFullSolutionDataset(
359
- puzzle_path=args.puzzle_data,
360
- min_rating=args.min_rating,
361
- max_rating=args.max_rating,
362
- themes=themes,
363
- max_puzzles=args.num_puzzles,
364
- )
365
-
366
- # Evaluate
367
- print("\n" + "=" * 70)
368
- print("Starting evaluation")
369
- print("=" * 70)
370
-
371
- evaluator = PuzzleEvaluator(model, device=device)
372
- results = evaluator.evaluate_dataset(dataset, num_puzzles=args.num_puzzles)
373
-
374
- # Print summary
375
- print("\n" + "=" * 70)
376
- print("EVALUATION RESULTS")
377
- print("=" * 70)
378
- print(f"Total puzzles: {results['num_puzzles']}")
379
- print(f"First move accuracy: {results['first_move_accuracy']:.2f}%")
380
- print(f"Full solution accuracy: {results['full_solution_accuracy']:.2f}%")
381
- print(f"Move accuracy: {results['move_accuracy']:.2f}%")
382
- print(f"Average moves correct: {results['average_moves_correct']:.2f}")
383
-
384
- print("\n" + "-" * 70)
385
- print("By Rating:")
386
- print("-" * 70)
387
- for rating, stats in sorted(results["by_rating"].items()):
388
- print(
389
- f" {rating:>12}: {stats['count']:4} puzzles | "
390
- f"First: {stats['first_move_accuracy']:5.2f}% | "
391
- f"Full: {stats['full_solution_accuracy']:5.2f}%"
392
- )
393
-
394
- print("\n" + "-" * 70)
395
- print("Top Themes:")
396
- print("-" * 70)
397
- for theme, stats in list(results["by_theme"].items())[:10]:
398
- print(
399
- f" {theme:>20}: {stats['count']:4} puzzles | "
400
- f"First: {stats['first_move_accuracy']:5.2f}% | "
401
- f"Full: {stats['full_solution_accuracy']:5.2f}%"
402
- )
403
-
404
- # Save results
405
- if args.output:
406
- output_path = args.output
407
- else:
408
- timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
409
- output_path = f"results/puzzle_eval_{timestamp}.json"
410
-
411
- output_path = Path(output_path)
412
- output_path.parent.mkdir(parents=True, exist_ok=True)
413
-
414
- # Don't save individual puzzle results to keep file size manageable
415
- results_summary = {k: v for k, v in results.items() if k != "puzzle_results"}
416
- results_summary["num_puzzles_detailed"] = len(results["puzzle_results"])
417
-
418
- with open(output_path, "w") as f:
419
- json.dump(results_summary, f, indent=2)
420
-
421
- print(f"\nResults saved to {output_path}")
422
- print("=" * 70)
423
-
424
-
425
- if __name__ == "__main__":
426
- main()
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
frontend/.dockerignore ADDED
@@ -0,0 +1,36 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Dependencies
2
+ node_modules
3
+ .pnp
4
+ .pnp.js
5
+
6
+ # Build outputs
7
+ .next
8
+ out
9
+ build
10
+
11
+ # Testing
12
+ coverage
13
+
14
+ # Misc
15
+ .DS_Store
16
+ *.pem
17
+
18
+ # Debug
19
+ npm-debug.log*
20
+ yarn-debug.log*
21
+ yarn-error.log*
22
+
23
+ # Local env files
24
+ .env*.local
25
+ .env
26
+
27
+ # Vercel
28
+ .vercel
29
+
30
+ # TypeScript
31
+ *.tsbuildinfo
32
+ next-env.d.ts
33
+
34
+ # IDE
35
+ .idea
36
+ .vscode
frontend/.gitignore ADDED
@@ -0,0 +1,41 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # See https://help.github.com/articles/ignoring-files/ for more about ignoring files.
2
+
3
+ # dependencies
4
+ /node_modules
5
+ /.pnp
6
+ .pnp.*
7
+ .yarn/*
8
+ !.yarn/patches
9
+ !.yarn/plugins
10
+ !.yarn/releases
11
+ !.yarn/versions
12
+
13
+ # testing
14
+ /coverage
15
+
16
+ # next.js
17
+ /.next/
18
+ /out/
19
+
20
+ # production
21
+ /build
22
+
23
+ # misc
24
+ .DS_Store
25
+ *.pem
26
+
27
+ # debug
28
+ npm-debug.log*
29
+ yarn-debug.log*
30
+ yarn-error.log*
31
+ .pnpm-debug.log*
32
+
33
+ # env files (can opt-in for committing if needed)
34
+ .env*
35
+
36
+ # vercel
37
+ .vercel
38
+
39
+ # typescript
40
+ *.tsbuildinfo
41
+ next-env.d.ts
frontend/Dockerfile ADDED
@@ -0,0 +1,50 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Frontend Dockerfile for ChessTransformer Next.js app
2
+ FROM node:20-alpine AS base
3
+
4
+ # Install dependencies only when needed
5
+ FROM base AS deps
6
+ RUN apk add --no-cache libc6-compat
7
+ WORKDIR /app
8
+
9
+ # Copy package files
10
+ COPY package.json package-lock.json* ./
11
+
12
+ # Install dependencies
13
+ RUN npm ci
14
+
15
+ # Rebuild the source code only when needed
16
+ FROM base AS builder
17
+ WORKDIR /app
18
+ COPY --from=deps /app/node_modules ./node_modules
19
+ COPY . .
20
+
21
+ # Set the API URL for build time
22
+ ARG NEXT_PUBLIC_API_URL=http://localhost:5001
23
+ ENV NEXT_PUBLIC_API_URL=$NEXT_PUBLIC_API_URL
24
+
25
+ # Build the application
26
+ RUN npm run build
27
+
28
+ # Production image, copy all the files and run next
29
+ FROM base AS runner
30
+ WORKDIR /app
31
+
32
+ ENV NODE_ENV=production
33
+
34
+ # Create non-root user
35
+ RUN addgroup --system --gid 1001 nodejs
36
+ RUN adduser --system --uid 1001 nextjs
37
+
38
+ # Copy built application
39
+ COPY --from=builder /app/public ./public
40
+ COPY --from=builder --chown=nextjs:nodejs /app/.next/standalone ./
41
+ COPY --from=builder --chown=nextjs:nodejs /app/.next/static ./.next/static
42
+
43
+ USER nextjs
44
+
45
+ EXPOSE 3000
46
+
47
+ ENV PORT=3000
48
+ ENV HOSTNAME="0.0.0.0"
49
+
50
+ CMD ["node", "server.js"]