Spaces:
Running
Running
Switch to the Docker demo: Rust engine + Next.js frontend
Browse filesThis view is limited to 50 files because it contains too many changes. See raw diff
- .dockerignore +3 -0
- .gitattributes +1 -1
- Dockerfile +39 -0
- README.md +15 -11
- app.py +0 -243
- chesstransformer/__init__.py +0 -1
- chesstransformer/benchmark/__init__.py +0 -3
- chesstransformer/benchmark/bot_benchmark.py +0 -399
- chesstransformer/bots/__init__.py +0 -5
- chesstransformer/bots/legacy_position2move_bot.py +0 -222
- chesstransformer/bots/pos2move_v2_bot.py +0 -499
- chesstransformer/bots/pos2move_v2_mcts_bot.py +0 -318
- chesstransformer/bots/position2move_bot.py +0 -88
- chesstransformer/bots/random_bot.py +0 -33
- chesstransformer/datasets/__init__.py +0 -10
- chesstransformer/datasets/dataset_h5_convertor.py +0 -404
- chesstransformer/datasets/flat_shard_dataset.py +0 -231
- chesstransformer/datasets/h5_lichess_dataset.py +0 -295
- chesstransformer/datasets/lichess.py +0 -223
- chesstransformer/datasets/lichess_simple_uci.py +0 -136
- chesstransformer/datasets/puzzle_dataset.py +0 -259
- chesstransformer/datasets/shard_format.py +0 -129
- chesstransformer/distributed_muon.py +0 -222
- chesstransformer/introspection/__init__.py +0 -5
- chesstransformer/introspection/analyze.py +0 -450
- chesstransformer/introspection/viewer.html +0 -860
- chesstransformer/models/__init__.py +0 -1
- chesstransformer/models/tokenizer/__init__.py +0 -93
- chesstransformer/models/tokenizer/alphazero_move_encoder.py +0 -127
- chesstransformer/models/tokenizer/move_tokenizer.py +0 -38
- chesstransformer/models/tokenizer/position_tokenizer.py +0 -88
- chesstransformer/models/tokenizer/train_bpe_tokenizer.py +0 -94
- chesstransformer/models/transformer/__init__.py +0 -0
- chesstransformer/models/transformer/attention.py +0 -228
- chesstransformer/models/transformer/gpt.py +0 -79
- chesstransformer/models/transformer/pos2move_v2.py +0 -311
- chesstransformer/models/transformer/position2move.py +0 -329
- chesstransformer/models/transformer/rope.py +0 -104
- chesstransformer/models/transformer/utils.py +0 -46
- chesstransformer/optimizer.py +0 -206
- chesstransformer/trainers/__init__.py +0 -11
- chesstransformer/trainers/moveseq2moveseq_trainer.py +0 -342
- chesstransformer/trainers/pos2move_v2_trainer.py +0 -1128
- chesstransformer/trainers/position2move_trainer.py +0 -682
- chesstransformer/utils/__init__.py +0 -3
- chesstransformer/utils/chess_utils.py +0 -123
- chesstransformer/utils/evaluate_puzzles.py +0 -426
- frontend/.dockerignore +36 -0
- frontend/.gitignore +41 -0
- frontend/Dockerfile +50 -0
.dockerignore
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
**/node_modules
|
| 2 |
+
**/.next
|
| 3 |
+
**/target
|
.gitattributes
CHANGED
|
@@ -1 +1 @@
|
|
| 1 |
-
*.
|
|
|
|
| 1 |
+
*.onnx filter=lfs diff=lfs merge=lfs -text
|
Dockerfile
ADDED
|
@@ -0,0 +1,39 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Hugging Face Docker Space: one process, one port. ct-bot serves the JSON API
|
| 2 |
+
# and the static Next.js build together on 7860.
|
| 3 |
+
# Build context is the bundle prepare.sh assembles (rust/, frontend/, the model).
|
| 4 |
+
|
| 5 |
+
# --- frontend: static export ---------------------------------------------------
|
| 6 |
+
FROM node:20-slim AS web
|
| 7 |
+
WORKDIR /web
|
| 8 |
+
COPY frontend/package.json frontend/package-lock.json ./
|
| 9 |
+
RUN npm ci
|
| 10 |
+
COPY frontend/ ./
|
| 11 |
+
# Empty API URL = same origin: the page calls /api/* on the server that served it.
|
| 12 |
+
ENV NEXT_OUTPUT=export NEXT_PUBLIC_API_URL=""
|
| 13 |
+
RUN npx next build
|
| 14 |
+
|
| 15 |
+
# --- engine --------------------------------------------------------------------
|
| 16 |
+
FROM rust:1.92-slim-trixie AS engine
|
| 17 |
+
# ort's build script downloads the prebuilt ONNX Runtime over TLS; g++ provides
|
| 18 |
+
# the libstdc++ the statically linked runtime needs at link time.
|
| 19 |
+
RUN apt-get update && apt-get install -y --no-install-recommends \
|
| 20 |
+
pkg-config ca-certificates libssl-dev g++ && rm -rf /var/lib/apt/lists/*
|
| 21 |
+
WORKDIR /build
|
| 22 |
+
COPY rust/ ./rust/
|
| 23 |
+
RUN cargo build --release --manifest-path rust/Cargo.toml -p ct-bot
|
| 24 |
+
|
| 25 |
+
# --- runtime -------------------------------------------------------------------
|
| 26 |
+
FROM debian:trixie-slim
|
| 27 |
+
RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates \
|
| 28 |
+
&& rm -rf /var/lib/apt/lists/* && useradd -m -u 1000 user
|
| 29 |
+
WORKDIR /app
|
| 30 |
+
COPY --from=engine /build/rust/target/release/ct-bot /usr/local/bin/ct-bot
|
| 31 |
+
COPY --from=web /web/out ./web
|
| 32 |
+
COPY model.int8.onnx ./model.int8.onnx
|
| 33 |
+
USER user
|
| 34 |
+
|
| 35 |
+
# MCTS_SIMS can be overridden from the Space settings (Variables) without a
|
| 36 |
+
# rebuild. On the free 2-vCPU tier 800 sims is ~2 s a move; 2000 is ~5 s.
|
| 37 |
+
ENV PORT=7860 MCTS_SIMS=800 ORT_THREADS=2 STATIC_DIR=/app/web RUST_LOG=info
|
| 38 |
+
EXPOSE 7860
|
| 39 |
+
CMD ["sh", "-c", "exec ct-bot serve --model /app/model.int8.onnx --threads \"$ORT_THREADS\""]
|
README.md
CHANGED
|
@@ -1,11 +1,10 @@
|
|
| 1 |
---
|
| 2 |
title: ChessTransformer
|
| 3 |
emoji: ♟️
|
| 4 |
-
colorFrom:
|
| 5 |
colorTo: gray
|
| 6 |
-
sdk:
|
| 7 |
-
|
| 8 |
-
app_file: app.py
|
| 9 |
pinned: false
|
| 10 |
license: mit
|
| 11 |
short_description: Play a chess transformer trained only on human games
|
|
@@ -14,15 +13,20 @@ short_description: Play a chess transformer trained only on human games
|
|
| 14 |
# ♟️ ChessTransformer
|
| 15 |
|
| 16 |
Play an **11.7M-parameter transformer trained only on human games** (no self-play,
|
| 17 |
-
no reinforcement learning),
|
| 18 |
-
reaches **~2100 Elo** against Stockfish
|
| 19 |
-
simulation count so moves come back in ~1–2s.
|
| 20 |
|
| 21 |
-
|
| 22 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 23 |
|
| 24 |
Code, weights, and the training pipeline: **https://github.com/tchauffi/ChessTransformer**
|
| 25 |
|
| 26 |
> This `README.md` is the Space config. The deployable Space is assembled by
|
| 27 |
-
> `prepare.sh` (see `DEPLOY.md`)
|
| 28 |
-
> `
|
|
|
|
| 1 |
---
|
| 2 |
title: ChessTransformer
|
| 3 |
emoji: ♟️
|
| 4 |
+
colorFrom: green
|
| 5 |
colorTo: gray
|
| 6 |
+
sdk: docker
|
| 7 |
+
app_port: 7860
|
|
|
|
| 8 |
pinned: false
|
| 9 |
license: mit
|
| 10 |
short_description: Play a chess transformer trained only on human games
|
|
|
|
| 13 |
# ♟️ ChessTransformer
|
| 14 |
|
| 15 |
Play an **11.7M-parameter transformer trained only on human games** (no self-play,
|
| 16 |
+
no reinforcement learning), searching with AlphaZero-style MCTS. At full strength it
|
| 17 |
+
reaches **~2100 Elo** against Stockfish.
|
|
|
|
| 18 |
|
| 19 |
+
- Pick a color and a clock, then play: drag or click to move, premove while it
|
| 20 |
+
thinks, right-click to draw arrows and mark squares.
|
| 21 |
+
- **Watch it play itself**, or press **Autoplay** mid-game to hand it your side.
|
| 22 |
+
- The **Engine search** tab shows the branches it explored for each move; hover one
|
| 23 |
+
to replay that line on the board.
|
| 24 |
+
|
| 25 |
+
The engine is the Rust `ct-bot` (ONNX Runtime, int8, CPU). On this free 2-vCPU
|
| 26 |
+
Space it searches 800 positions a move, about 2 seconds.
|
| 27 |
|
| 28 |
Code, weights, and the training pipeline: **https://github.com/tchauffi/ChessTransformer**
|
| 29 |
|
| 30 |
> This `README.md` is the Space config. The deployable Space is assembled by
|
| 31 |
+
> `prepare.sh` (see `DEPLOY.md`): this file, the Dockerfile, the `rust/` and
|
| 32 |
+
> `frontend/` sources, and the int8 model.
|
app.py
DELETED
|
@@ -1,243 +0,0 @@
|
|
| 1 |
-
"""ChessTransformer — Hugging Face Space (Gradio).
|
| 2 |
-
|
| 3 |
-
Play the transformer chess engine in the browser. Runs on CPU: the model is
|
| 4 |
-
only 11.7M params, so MCTS at a modest sim count returns a move in ~1-2s.
|
| 5 |
-
|
| 6 |
-
Weights are resolved in this order:
|
| 7 |
-
1. ``MODEL_PATH`` env var (a checkpoint dir with model.safetensors + config)
|
| 8 |
-
2. a ``model/`` dir next to this file (how ``prepare.sh`` ships the Space)
|
| 9 |
-
3. the repo's ``data/models/pos2move_v2.1`` (for local dev runs)
|
| 10 |
-
4. download from GitHub (LFS media) as a last resort
|
| 11 |
-
"""
|
| 12 |
-
|
| 13 |
-
from __future__ import annotations
|
| 14 |
-
|
| 15 |
-
import itertools
|
| 16 |
-
import os
|
| 17 |
-
import sys
|
| 18 |
-
from pathlib import Path
|
| 19 |
-
|
| 20 |
-
import chess
|
| 21 |
-
import gradio as gr
|
| 22 |
-
from gradio_chessboard import Chessboard
|
| 23 |
-
|
| 24 |
-
# ── make `chesstransformer` importable ──────────────────────────────────────
|
| 25 |
-
# On the Space the package is vendored next to this file (cwd is on sys.path).
|
| 26 |
-
# For local dev runs from the repo, add the repo's src/ to the path.
|
| 27 |
-
_repo_src = Path(__file__).resolve().parents[1] / "src"
|
| 28 |
-
if (_repo_src / "chesstransformer").exists():
|
| 29 |
-
sys.path.insert(0, str(_repo_src))
|
| 30 |
-
|
| 31 |
-
from chesstransformer.bots import Pos2MoveV2MctsBot # noqa: E402
|
| 32 |
-
|
| 33 |
-
START_FEN = chess.STARTING_FEN
|
| 34 |
-
DEFAULT_SIMS = int(os.environ.get("MCTS_SIMS", "400"))
|
| 35 |
-
|
| 36 |
-
# GitHub locations for the last-resort weight download.
|
| 37 |
-
_GH = "tchauffi/ChessTransformer"
|
| 38 |
-
_BRANCH = os.environ.get("MODEL_BRANCH", "main")
|
| 39 |
-
_RAW = f"https://raw.githubusercontent.com/{_GH}/{_BRANCH}/data/models/pos2move_v2.1"
|
| 40 |
-
_LFS = f"https://media.githubusercontent.com/media/{_GH}/{_BRANCH}/data/models/pos2move_v2.1"
|
| 41 |
-
|
| 42 |
-
|
| 43 |
-
def _download_weights() -> str:
|
| 44 |
-
import urllib.request
|
| 45 |
-
|
| 46 |
-
dest = Path("/tmp/pos2move_v2.1")
|
| 47 |
-
dest.mkdir(parents=True, exist_ok=True)
|
| 48 |
-
for fname, url in [("model_config.json", f"{_RAW}/model_config.json"),
|
| 49 |
-
("model.safetensors", f"{_LFS}/model.safetensors")]:
|
| 50 |
-
out = dest / fname
|
| 51 |
-
if not out.exists():
|
| 52 |
-
print(f"Downloading {fname} …")
|
| 53 |
-
urllib.request.urlretrieve(url, out)
|
| 54 |
-
return str(dest)
|
| 55 |
-
|
| 56 |
-
|
| 57 |
-
def resolve_model_dir() -> str:
|
| 58 |
-
env = os.environ.get("MODEL_PATH")
|
| 59 |
-
if env and (Path(env) / "model.safetensors").exists():
|
| 60 |
-
return env
|
| 61 |
-
for cand in (
|
| 62 |
-
Path(__file__).resolve().parent / "model",
|
| 63 |
-
Path("model"),
|
| 64 |
-
Path(__file__).resolve().parents[1] / "data" / "models" / "pos2move_v2.1",
|
| 65 |
-
):
|
| 66 |
-
if (cand / "model.safetensors").exists():
|
| 67 |
-
return str(cand)
|
| 68 |
-
return _download_weights()
|
| 69 |
-
|
| 70 |
-
|
| 71 |
-
# ── load the bot once (CPU, no compile) ─────────────────────────────────────
|
| 72 |
-
MODEL_DIR = resolve_model_dir()
|
| 73 |
-
bot = Pos2MoveV2MctsBot(
|
| 74 |
-
model_dir=MODEL_DIR,
|
| 75 |
-
num_simulations=DEFAULT_SIMS,
|
| 76 |
-
device="cpu",
|
| 77 |
-
compile=False,
|
| 78 |
-
time_limit=0.0,
|
| 79 |
-
# Sample the opening moves (∝ visit count) so games aren't identical every
|
| 80 |
-
# time; switches to deterministic best-move play after the opening.
|
| 81 |
-
move_temp=1.0,
|
| 82 |
-
move_temp_plies=12,
|
| 83 |
-
)
|
| 84 |
-
|
| 85 |
-
|
| 86 |
-
def _human_color(human_color: str) -> bool:
|
| 87 |
-
return chess.BLACK if human_color == "black" else chess.WHITE
|
| 88 |
-
|
| 89 |
-
|
| 90 |
-
def _white_pov_value(board: chess.Board) -> float:
|
| 91 |
-
"""Value-head eval in [-1, 1] from White's perspective (+ = White better)."""
|
| 92 |
-
if board.is_game_over():
|
| 93 |
-
if board.is_checkmate():
|
| 94 |
-
return -1.0 if board.turn == chess.WHITE else 1.0
|
| 95 |
-
return 0.0 # stalemate / draw
|
| 96 |
-
v = bot.get_value(board)
|
| 97 |
-
if v is None:
|
| 98 |
-
return 0.0
|
| 99 |
-
return v if board.turn == chess.WHITE else -v
|
| 100 |
-
|
| 101 |
-
|
| 102 |
-
def eval_bar_html(board: chess.Board, orientation: str) -> str:
|
| 103 |
-
"""A vertical eval bar (White share fills from the bottom of White's side).
|
| 104 |
-
Flipped to match the board orientation so the viewer's side is at the bottom."""
|
| 105 |
-
wv = _white_pov_value(board)
|
| 106 |
-
white_pct = round(max(0.0, min(1.0, (wv + 1) / 2)) * 100, 1)
|
| 107 |
-
white_div = f'<div style="height:{white_pct}%;background:#f5f5f5;"></div>'
|
| 108 |
-
black_div = f'<div style="height:{round(100 - white_pct, 1)}%;background:#3a3a3a;"></div>'
|
| 109 |
-
# orientation "white": Black on top, White on bottom; flipped when "black".
|
| 110 |
-
segments = black_div + white_div if orientation == "white" else white_div + black_div
|
| 111 |
-
return (
|
| 112 |
-
f'<div title="Model eval (White POV): {wv:+.2f}" '
|
| 113 |
-
'style="display:flex;flex-direction:column;width:26px;height:460px;'
|
| 114 |
-
'border:1px solid #999;border-radius:4px;overflow:hidden;">'
|
| 115 |
-
f'{segments}</div>'
|
| 116 |
-
)
|
| 117 |
-
|
| 118 |
-
|
| 119 |
-
def _result_text(board: chess.Board, human_color: str) -> str:
|
| 120 |
-
outcome = board.outcome()
|
| 121 |
-
if outcome is None:
|
| 122 |
-
return ""
|
| 123 |
-
if outcome.winner is None:
|
| 124 |
-
return f"Draw ({outcome.termination.name.lower().replace('_', ' ')}). New game?"
|
| 125 |
-
who = "You win! 🎉" if outcome.winner == _human_color(human_color) else "ChessTransformer wins."
|
| 126 |
-
return f"Checkmate — {who} New game?"
|
| 127 |
-
|
| 128 |
-
|
| 129 |
-
def _bot_move(board: chess.Board, sims: int):
|
| 130 |
-
"""Play the side to move with the bot; return its SAN and value."""
|
| 131 |
-
bot.num_simulations = int(sims)
|
| 132 |
-
uci, value = bot.predict(board)
|
| 133 |
-
san = board.san(chess.Move.from_uci(uci))
|
| 134 |
-
board.push_uci(uci)
|
| 135 |
-
return san, value
|
| 136 |
-
|
| 137 |
-
|
| 138 |
-
def on_move(fen: str, sims: int, human_color: str):
|
| 139 |
-
"""Human just moved; reply with the bot's move (whichever side is to move).
|
| 140 |
-
The board orientation matches the human's color, so the eval bar does too.
|
| 141 |
-
|
| 142 |
-
A generator: first yields a "thinking" status (so the player sees the bot is
|
| 143 |
-
working during the ~1-3s CPU search), then yields the bot's reply."""
|
| 144 |
-
board = chess.Board(fen)
|
| 145 |
-
if board.is_game_over():
|
| 146 |
-
yield board.fen(), _result_text(board, human_color), eval_bar_html(board, human_color)
|
| 147 |
-
return
|
| 148 |
-
|
| 149 |
-
# Immediate feedback while the search runs (board/eval unchanged for now).
|
| 150 |
-
yield gr.update(), "⏳ ChessTransformer is thinking…", gr.update()
|
| 151 |
-
|
| 152 |
-
san, value = _bot_move(board, sims)
|
| 153 |
-
|
| 154 |
-
if board.is_game_over():
|
| 155 |
-
status = f"Bot played **{san}**. {_result_text(board, human_color)}"
|
| 156 |
-
else:
|
| 157 |
-
eval_str = f" (eval {value:+.2f})" if value is not None else ""
|
| 158 |
-
status = f"Bot played **{san}**{eval_str}. Your move."
|
| 159 |
-
yield board.fen(), status, eval_bar_html(board, human_color)
|
| 160 |
-
|
| 161 |
-
|
| 162 |
-
# Monotonic id so every New game yields a *distinct* setup dict — otherwise a
|
| 163 |
-
# same-colour replay returns an identical dict and the gr.render block, seeing no
|
| 164 |
-
# change, wouldn't rebuild (and reset) the board.
|
| 165 |
-
_game_counter = itertools.count(1)
|
| 166 |
-
|
| 167 |
-
|
| 168 |
-
def new_game(play_as: str, sims: int):
|
| 169 |
-
"""Start a fresh game. If the human plays Black, the bot (White) opens.
|
| 170 |
-
|
| 171 |
-
Returns (setup, eval_bar). The setup dict drives a gr.render block — changing
|
| 172 |
-
it rebuilds the board so its orientation (chessboard.js reads it only at
|
| 173 |
-
construction) actually flips to the human's side."""
|
| 174 |
-
human_color = "black" if play_as == "Black" else "white"
|
| 175 |
-
board = chess.Board()
|
| 176 |
-
if human_color == "black":
|
| 177 |
-
san, _ = _bot_move(board, sims)
|
| 178 |
-
status = f"New game — you are **Black**. Bot opened with **{san}**. Your move."
|
| 179 |
-
else:
|
| 180 |
-
status = "New game — you are **White**. Make your move."
|
| 181 |
-
setup = {"fen": board.fen(), "orientation": human_color, "color": human_color,
|
| 182 |
-
"status": status, "nonce": next(_game_counter)}
|
| 183 |
-
return setup, eval_bar_html(board, human_color)
|
| 184 |
-
|
| 185 |
-
|
| 186 |
-
with gr.Blocks(title="ChessTransformer", theme=gr.themes.Soft()) as demo:
|
| 187 |
-
gr.Markdown(
|
| 188 |
-
"# ♟️ ChessTransformer\n"
|
| 189 |
-
"An **11.7M-parameter transformer trained only on human games** "
|
| 190 |
-
"(no self-play, no RL), playing via AlphaZero-style MCTS. "
|
| 191 |
-
"It reaches **~2100 Elo** vs Stockfish at full strength; this CPU demo "
|
| 192 |
-
"runs at a lower sim count so moves come back quickly.\n\n"
|
| 193 |
-
"**Pick your color and hit New game.** Drag a piece to move — the bot "
|
| 194 |
-
"replies automatically. "
|
| 195 |
-
"[GitHub repo →](https://github.com/tchauffi/ChessTransformer)"
|
| 196 |
-
)
|
| 197 |
-
setup = gr.State({"fen": START_FEN, "orientation": "white", "color": "white",
|
| 198 |
-
"status": "You are **White**. Make your move.", "nonce": 0})
|
| 199 |
-
|
| 200 |
-
with gr.Row():
|
| 201 |
-
# Solid min_widths so the board container always has a size when the
|
| 202 |
-
# gr.render block (re)mounts it — chessboard.js reloads the whole page if
|
| 203 |
-
# it inits at 0 width, which is what caused the mobile flicker/reload loop.
|
| 204 |
-
eval_col = gr.Column(scale=0, min_width=44)
|
| 205 |
-
board_col = gr.Column(scale=3, min_width=340)
|
| 206 |
-
ctrl_col = gr.Column(scale=1, min_width=200)
|
| 207 |
-
|
| 208 |
-
with eval_col:
|
| 209 |
-
eval_bar = gr.HTML(eval_bar_html(chess.Board(), "white"), visible=False)
|
| 210 |
-
|
| 211 |
-
with ctrl_col:
|
| 212 |
-
play_as = gr.Radio(["White", "Black"], value="White", label="Play as")
|
| 213 |
-
sims = gr.Slider(32, 1200, value=DEFAULT_SIMS, step=8,
|
| 214 |
-
label="Engine strength (MCTS sims/move)",
|
| 215 |
-
info="Higher = stronger but slower on CPU")
|
| 216 |
-
show_eval = gr.Checkbox(False, label="Show evaluation bar")
|
| 217 |
-
new_btn = gr.Button("New game", variant="primary")
|
| 218 |
-
|
| 219 |
-
# The board lives in a gr.render so switching color rebuilds it with the new
|
| 220 |
-
# orientation (chessboard.js reads orientation only at construction). The
|
| 221 |
-
# min_width above keeps the container sized so the remount can't init at 0.
|
| 222 |
-
with board_col:
|
| 223 |
-
@gr.render(inputs=setup)
|
| 224 |
-
def render_board(cfg):
|
| 225 |
-
board = Chessboard(value=cfg["fen"], game_mode=True, min_width=320,
|
| 226 |
-
orientation=cfg["orientation"], label="Drag to move")
|
| 227 |
-
status = gr.Markdown(cfg["status"])
|
| 228 |
-
color = cfg["color"]
|
| 229 |
-
|
| 230 |
-
def handle(fen, s): # generator → streams the "thinking" update first
|
| 231 |
-
yield from on_move(fen, s, color)
|
| 232 |
-
|
| 233 |
-
# show_progress="hidden": our "⏳ thinking…" status is the signal, so
|
| 234 |
-
# Gradio's default overlay doesn't sit over the board.
|
| 235 |
-
board.move(handle, inputs=[board, sims], outputs=[board, status, eval_bar],
|
| 236 |
-
show_progress="hidden")
|
| 237 |
-
|
| 238 |
-
show_eval.change(lambda show: gr.update(visible=show), inputs=show_eval, outputs=eval_bar)
|
| 239 |
-
new_btn.click(new_game, inputs=[play_as, sims], outputs=[setup, eval_bar])
|
| 240 |
-
|
| 241 |
-
|
| 242 |
-
if __name__ == "__main__":
|
| 243 |
-
demo.launch()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
chesstransformer/__init__.py
DELETED
|
@@ -1 +0,0 @@
|
|
| 1 |
-
"""This is the main package for the ChessTransformer project."""
|
|
|
|
|
|
chesstransformer/benchmark/__init__.py
DELETED
|
@@ -1,3 +0,0 @@
|
|
| 1 |
-
from .bot_benchmark import BotBenchmark
|
| 2 |
-
|
| 3 |
-
__all__ = ["BotBenchmark"]
|
|
|
|
|
|
|
|
|
|
|
|
chesstransformer/benchmark/bot_benchmark.py
DELETED
|
@@ -1,399 +0,0 @@
|
|
| 1 |
-
"""
|
| 2 |
-
Bot benchmarking system for comparing chess bot performance.
|
| 3 |
-
|
| 4 |
-
Usage:
|
| 5 |
-
python -m chesstransformer.benchmark.bot_benchmark \\
|
| 6 |
-
--bot1-type position2move --bot1-checkpoint path/to/checkpoint \\
|
| 7 |
-
--bot2-type random \\
|
| 8 |
-
--num-games 100 \\
|
| 9 |
-
--output results.json
|
| 10 |
-
"""
|
| 11 |
-
|
| 12 |
-
import chess
|
| 13 |
-
import time
|
| 14 |
-
import json
|
| 15 |
-
from pathlib import Path
|
| 16 |
-
from typing import List, Tuple, Optional
|
| 17 |
-
from dataclasses import dataclass, asdict
|
| 18 |
-
from tqdm import tqdm
|
| 19 |
-
|
| 20 |
-
from chesstransformer.bots import Position2MoveBot, LegacyPosition2MoveBot, RandomBot
|
| 21 |
-
|
| 22 |
-
|
| 23 |
-
@dataclass
|
| 24 |
-
class GameResult:
|
| 25 |
-
"""Single game result statistics."""
|
| 26 |
-
|
| 27 |
-
game_id: int
|
| 28 |
-
white_bot: str
|
| 29 |
-
black_bot: str
|
| 30 |
-
result: str # "1-0", "0-1", "1/2-1/2"
|
| 31 |
-
num_moves: int
|
| 32 |
-
white_time: float
|
| 33 |
-
black_time: float
|
| 34 |
-
termination: str # "checkmate", "stalemate", "insufficient_material", "max_moves"
|
| 35 |
-
|
| 36 |
-
def to_dict(self):
|
| 37 |
-
return asdict(self)
|
| 38 |
-
|
| 39 |
-
|
| 40 |
-
@dataclass
|
| 41 |
-
class BenchmarkStats:
|
| 42 |
-
"""Aggregate statistics for a bot."""
|
| 43 |
-
|
| 44 |
-
bot_name: str
|
| 45 |
-
total_games: int
|
| 46 |
-
wins: int
|
| 47 |
-
losses: int
|
| 48 |
-
draws: int
|
| 49 |
-
avg_moves_per_game: float
|
| 50 |
-
avg_time_per_move: float
|
| 51 |
-
total_time: float
|
| 52 |
-
win_rate: float
|
| 53 |
-
win_as_white: int
|
| 54 |
-
win_as_black: int
|
| 55 |
-
|
| 56 |
-
def to_dict(self):
|
| 57 |
-
return asdict(self)
|
| 58 |
-
|
| 59 |
-
|
| 60 |
-
class BotBenchmark:
|
| 61 |
-
"""Benchmark system for comparing chess bots."""
|
| 62 |
-
|
| 63 |
-
def __init__(self, max_moves: int = 500, time_limit: Optional[float] = None):
|
| 64 |
-
"""
|
| 65 |
-
Initialize benchmark.
|
| 66 |
-
|
| 67 |
-
Args:
|
| 68 |
-
max_moves: Maximum moves per game before declaring draw
|
| 69 |
-
time_limit: Optional time limit per move in seconds
|
| 70 |
-
"""
|
| 71 |
-
self.max_moves = max_moves
|
| 72 |
-
self.time_limit = time_limit
|
| 73 |
-
self.game_results: List[GameResult] = []
|
| 74 |
-
self.bot_counter = 0 # Counter for unique bot IDs
|
| 75 |
-
|
| 76 |
-
def create_bot(
|
| 77 |
-
self, bot_type: str, checkpoint_path: Optional[str] = None, bot_name: Optional[str] = None, **kwargs
|
| 78 |
-
):
|
| 79 |
-
"""
|
| 80 |
-
Factory method to create bots.
|
| 81 |
-
|
| 82 |
-
Args:
|
| 83 |
-
bot_type: Type of bot ("position2move", "legacy_position2move", "random")
|
| 84 |
-
checkpoint_path: Path to model checkpoint (for Position2MoveBot)
|
| 85 |
-
bot_name: Optional custom name for the bot (for tracking multiple instances)
|
| 86 |
-
**kwargs: Additional bot-specific arguments
|
| 87 |
-
"""
|
| 88 |
-
if bot_type.lower() == "position2move":
|
| 89 |
-
if checkpoint_path:
|
| 90 |
-
bot = Position2MoveBot(model_path=checkpoint_path)
|
| 91 |
-
else:
|
| 92 |
-
bot = Position2MoveBot()
|
| 93 |
-
elif bot_type.lower() == "legacy_position2move":
|
| 94 |
-
if checkpoint_path:
|
| 95 |
-
bot = LegacyPosition2MoveBot(model_path=checkpoint_path)
|
| 96 |
-
else:
|
| 97 |
-
bot = LegacyPosition2MoveBot()
|
| 98 |
-
elif bot_type.lower() == "random":
|
| 99 |
-
bot = RandomBot()
|
| 100 |
-
else:
|
| 101 |
-
raise ValueError(f"Unknown bot type: {bot_type}")
|
| 102 |
-
|
| 103 |
-
# Assign unique identifier
|
| 104 |
-
if bot_name:
|
| 105 |
-
bot._benchmark_id = bot_name
|
| 106 |
-
else:
|
| 107 |
-
self.bot_counter += 1
|
| 108 |
-
bot._benchmark_id = f"{bot.__class__.__name__}_{self.bot_counter}"
|
| 109 |
-
|
| 110 |
-
return bot
|
| 111 |
-
|
| 112 |
-
def play_game(self, white_bot, black_bot, game_id: int, verbose: bool = False) -> GameResult:
|
| 113 |
-
"""
|
| 114 |
-
Play a single game between two bots.
|
| 115 |
-
|
| 116 |
-
Args:
|
| 117 |
-
white_bot: Bot playing white
|
| 118 |
-
black_bot: Bot playing black
|
| 119 |
-
game_id: Unique game identifier
|
| 120 |
-
verbose: Print move-by-move progress
|
| 121 |
-
|
| 122 |
-
Returns:
|
| 123 |
-
GameResult with statistics
|
| 124 |
-
"""
|
| 125 |
-
board = chess.Board()
|
| 126 |
-
white_time = 0.0
|
| 127 |
-
black_time = 0.0
|
| 128 |
-
move_count = 0
|
| 129 |
-
|
| 130 |
-
while not board.is_game_over() and move_count < self.max_moves:
|
| 131 |
-
current_bot = white_bot if board.turn == chess.WHITE else black_bot
|
| 132 |
-
|
| 133 |
-
start_time = time.time()
|
| 134 |
-
|
| 135 |
-
move, _ = current_bot.predict(board)
|
| 136 |
-
|
| 137 |
-
elapsed = time.time() - start_time
|
| 138 |
-
|
| 139 |
-
if board.turn == chess.WHITE:
|
| 140 |
-
white_time += elapsed
|
| 141 |
-
else:
|
| 142 |
-
black_time += elapsed
|
| 143 |
-
|
| 144 |
-
board.push(chess.Move.from_uci(move))
|
| 145 |
-
move_count += 1
|
| 146 |
-
|
| 147 |
-
if verbose and move_count % 10 == 0:
|
| 148 |
-
print(f"Move {move_count}: {move}")
|
| 149 |
-
|
| 150 |
-
# Determine result and termination reason
|
| 151 |
-
if board.is_checkmate():
|
| 152 |
-
result = "0-1" if board.turn == chess.WHITE else "1-0"
|
| 153 |
-
termination = "checkmate"
|
| 154 |
-
elif board.is_stalemate():
|
| 155 |
-
result = "1/2-1/2"
|
| 156 |
-
termination = "stalemate"
|
| 157 |
-
elif board.is_insufficient_material():
|
| 158 |
-
result = "1/2-1/2"
|
| 159 |
-
termination = "insufficient_material"
|
| 160 |
-
elif board.is_seventyfive_moves():
|
| 161 |
-
result = "1/2-1/2"
|
| 162 |
-
termination = "75_move_rule"
|
| 163 |
-
elif board.is_fivefold_repetition():
|
| 164 |
-
result = "1/2-1/2"
|
| 165 |
-
termination = "repetition"
|
| 166 |
-
elif move_count >= self.max_moves:
|
| 167 |
-
result = "1/2-1/2"
|
| 168 |
-
termination = "max_moves"
|
| 169 |
-
else:
|
| 170 |
-
# Should not reach here if game ended normally
|
| 171 |
-
result = "1/2-1/2"
|
| 172 |
-
termination = "unknown"
|
| 173 |
-
|
| 174 |
-
return GameResult(
|
| 175 |
-
game_id=game_id,
|
| 176 |
-
white_bot=white_bot._benchmark_id,
|
| 177 |
-
black_bot=black_bot._benchmark_id,
|
| 178 |
-
result=result,
|
| 179 |
-
num_moves=move_count,
|
| 180 |
-
white_time=white_time,
|
| 181 |
-
black_time=black_time,
|
| 182 |
-
termination=termination,
|
| 183 |
-
)
|
| 184 |
-
|
| 185 |
-
def run_benchmark(
|
| 186 |
-
self, bot1, bot2, num_games: int, alternate_colors: bool = True, verbose: bool = False
|
| 187 |
-
) -> Tuple[BenchmarkStats, BenchmarkStats]:
|
| 188 |
-
"""
|
| 189 |
-
Run full benchmark between two bots.
|
| 190 |
-
|
| 191 |
-
Args:
|
| 192 |
-
bot1: First bot
|
| 193 |
-
bot2: Second bot
|
| 194 |
-
num_games: Number of games to play
|
| 195 |
-
alternate_colors: Alternate colors each game
|
| 196 |
-
verbose: Print game-by-game results
|
| 197 |
-
|
| 198 |
-
Returns:
|
| 199 |
-
Tuple of (bot1_stats, bot2_stats)
|
| 200 |
-
"""
|
| 201 |
-
self.game_results = []
|
| 202 |
-
|
| 203 |
-
pbar = tqdm(range(num_games), desc="Playing games")
|
| 204 |
-
for game_id in pbar:
|
| 205 |
-
if alternate_colors and game_id % 2 == 1:
|
| 206 |
-
# Swap colors
|
| 207 |
-
white_bot, black_bot = bot2, bot1
|
| 208 |
-
else:
|
| 209 |
-
white_bot, black_bot = bot1, bot2
|
| 210 |
-
|
| 211 |
-
result = self.play_game(white_bot, black_bot, game_id, verbose=verbose)
|
| 212 |
-
self.game_results.append(result)
|
| 213 |
-
|
| 214 |
-
if verbose:
|
| 215 |
-
print(f"Game {game_id + 1}: {result.white_bot} vs {result.black_bot} = {result.result}")
|
| 216 |
-
|
| 217 |
-
# Update progress bar with current stats
|
| 218 |
-
bot1_wins = sum(1 for r in self.game_results if self._bot_won(r, bot1._benchmark_id))
|
| 219 |
-
bot2_wins = sum(1 for r in self.game_results if self._bot_won(r, bot2._benchmark_id))
|
| 220 |
-
pbar.set_postfix({bot1._benchmark_id: bot1_wins, bot2._benchmark_id: bot2_wins})
|
| 221 |
-
|
| 222 |
-
# Calculate statistics
|
| 223 |
-
bot1_stats = self._calculate_stats(bot1._benchmark_id)
|
| 224 |
-
bot2_stats = self._calculate_stats(bot2._benchmark_id)
|
| 225 |
-
|
| 226 |
-
return bot1_stats, bot2_stats
|
| 227 |
-
|
| 228 |
-
def _bot_won(self, result: GameResult, bot_name: str) -> bool:
|
| 229 |
-
"""Check if bot won the game."""
|
| 230 |
-
if bot_name == result.white_bot and result.result == "1-0":
|
| 231 |
-
return True
|
| 232 |
-
if bot_name == result.black_bot and result.result == "0-1":
|
| 233 |
-
return True
|
| 234 |
-
return False
|
| 235 |
-
|
| 236 |
-
def _calculate_stats(self, bot_name: str) -> BenchmarkStats:
|
| 237 |
-
"""Calculate aggregate statistics for a bot."""
|
| 238 |
-
bot_results = []
|
| 239 |
-
wins = 0
|
| 240 |
-
losses = 0
|
| 241 |
-
draws = 0
|
| 242 |
-
win_as_white = 0
|
| 243 |
-
win_as_black = 0
|
| 244 |
-
total_time = 0.0
|
| 245 |
-
total_moves = 0
|
| 246 |
-
|
| 247 |
-
for result in self.game_results:
|
| 248 |
-
is_white = result.white_bot == bot_name
|
| 249 |
-
is_black = result.black_bot == bot_name
|
| 250 |
-
|
| 251 |
-
if not (is_white or is_black):
|
| 252 |
-
continue
|
| 253 |
-
|
| 254 |
-
bot_results.append(result)
|
| 255 |
-
total_moves += result.num_moves
|
| 256 |
-
|
| 257 |
-
if is_white:
|
| 258 |
-
total_time += result.white_time
|
| 259 |
-
if result.result == "1-0":
|
| 260 |
-
wins += 1
|
| 261 |
-
win_as_white += 1
|
| 262 |
-
elif result.result == "0-1":
|
| 263 |
-
losses += 1
|
| 264 |
-
else:
|
| 265 |
-
draws += 1
|
| 266 |
-
else:
|
| 267 |
-
total_time += result.black_time
|
| 268 |
-
if result.result == "0-1":
|
| 269 |
-
wins += 1
|
| 270 |
-
win_as_black += 1
|
| 271 |
-
elif result.result == "1-0":
|
| 272 |
-
losses += 1
|
| 273 |
-
else:
|
| 274 |
-
draws += 1
|
| 275 |
-
|
| 276 |
-
total_games = len(bot_results)
|
| 277 |
-
avg_moves = total_moves / total_games if total_games > 0 else 0
|
| 278 |
-
avg_time_per_move = total_time / total_moves if total_moves > 0 else 0
|
| 279 |
-
win_rate = wins / total_games if total_games > 0 else 0
|
| 280 |
-
|
| 281 |
-
return BenchmarkStats(
|
| 282 |
-
bot_name=bot_name,
|
| 283 |
-
total_games=total_games,
|
| 284 |
-
wins=wins,
|
| 285 |
-
losses=losses,
|
| 286 |
-
draws=draws,
|
| 287 |
-
avg_moves_per_game=avg_moves,
|
| 288 |
-
avg_time_per_move=avg_time_per_move,
|
| 289 |
-
total_time=total_time,
|
| 290 |
-
win_rate=win_rate,
|
| 291 |
-
win_as_white=win_as_white,
|
| 292 |
-
win_as_black=win_as_black,
|
| 293 |
-
)
|
| 294 |
-
|
| 295 |
-
def save_results(self, output_path: str):
|
| 296 |
-
"""Save benchmark results to JSON file."""
|
| 297 |
-
output = {
|
| 298 |
-
"config": {"max_moves": self.max_moves, "time_limit": self.time_limit, "num_games": len(self.game_results)},
|
| 299 |
-
"games": [r.to_dict() for r in self.game_results],
|
| 300 |
-
"stats": {
|
| 301 |
-
bot_name: self._calculate_stats(bot_name).to_dict()
|
| 302 |
-
for bot_name in set([r.white_bot for r in self.game_results] + [r.black_bot for r in self.game_results])
|
| 303 |
-
},
|
| 304 |
-
}
|
| 305 |
-
|
| 306 |
-
output_file = Path(output_path)
|
| 307 |
-
output_file.parent.mkdir(parents=True, exist_ok=True)
|
| 308 |
-
|
| 309 |
-
with output_file.open("w", encoding="utf-8") as f:
|
| 310 |
-
json.dump(output, f, indent=2)
|
| 311 |
-
|
| 312 |
-
print(f"Results saved to: {output_path}")
|
| 313 |
-
|
| 314 |
-
def print_summary(self, bot1_stats: BenchmarkStats, bot2_stats: BenchmarkStats):
|
| 315 |
-
"""Print formatted summary of benchmark results."""
|
| 316 |
-
print("\n" + "=" * 80)
|
| 317 |
-
print("BENCHMARK RESULTS")
|
| 318 |
-
print("=" * 80)
|
| 319 |
-
|
| 320 |
-
for stats in [bot1_stats, bot2_stats]:
|
| 321 |
-
print(f"\n{stats.bot_name}:")
|
| 322 |
-
print(f" Total Games: {stats.total_games}")
|
| 323 |
-
print(f" Wins: {stats.wins} ({stats.win_rate:.1%})")
|
| 324 |
-
print(f" - As White: {stats.win_as_white}")
|
| 325 |
-
print(f" - As Black: {stats.win_as_black}")
|
| 326 |
-
print(f" Losses: {stats.losses}")
|
| 327 |
-
print(f" Draws: {stats.draws}")
|
| 328 |
-
print(f" Avg Moves/Game: {stats.avg_moves_per_game:.1f}")
|
| 329 |
-
print(f" Avg Time/Move: {stats.avg_time_per_move:.3f}s")
|
| 330 |
-
print(f" Total Time: {stats.total_time:.1f}s")
|
| 331 |
-
|
| 332 |
-
print("\n" + "=" * 80)
|
| 333 |
-
|
| 334 |
-
|
| 335 |
-
def main():
|
| 336 |
-
"""CLI entry point for bot benchmarking."""
|
| 337 |
-
import argparse
|
| 338 |
-
|
| 339 |
-
parser = argparse.ArgumentParser(description="Benchmark chess bots")
|
| 340 |
-
parser.add_argument(
|
| 341 |
-
"--bot1-type",
|
| 342 |
-
type=str,
|
| 343 |
-
required=True,
|
| 344 |
-
choices=["position2move", "legacy_position2move", "random"],
|
| 345 |
-
help="Type of first bot",
|
| 346 |
-
)
|
| 347 |
-
parser.add_argument("--bot1-checkpoint", type=str, default=None, help="Checkpoint path for bot1 (if applicable)")
|
| 348 |
-
parser.add_argument(
|
| 349 |
-
"--bot1-name", type=str, default=None, help="Custom name for bot1 (useful when comparing same bot types)"
|
| 350 |
-
)
|
| 351 |
-
parser.add_argument(
|
| 352 |
-
"--bot2-type",
|
| 353 |
-
type=str,
|
| 354 |
-
required=True,
|
| 355 |
-
choices=["position2move", "legacy_position2move", "random"],
|
| 356 |
-
help="Type of second bot",
|
| 357 |
-
)
|
| 358 |
-
parser.add_argument("--bot2-checkpoint", type=str, default=None, help="Checkpoint path for bot2 (if applicable)")
|
| 359 |
-
parser.add_argument(
|
| 360 |
-
"--bot2-name", type=str, default=None, help="Custom name for bot2 (useful when comparing same bot types)"
|
| 361 |
-
)
|
| 362 |
-
parser.add_argument("--num-games", type=int, default=100, help="Number of games to play")
|
| 363 |
-
parser.add_argument("--max-moves", type=int, default=500, help="Maximum moves per game")
|
| 364 |
-
parser.add_argument("--time-limit", type=float, default=None, help="Time limit per move in seconds")
|
| 365 |
-
parser.add_argument("--output", type=str, default="benchmark_results.json", help="Output file for results")
|
| 366 |
-
parser.add_argument("--no-alternate", action="store_true", help="Don't alternate colors between games")
|
| 367 |
-
parser.add_argument("--verbose", action="store_true", help="Print detailed game information")
|
| 368 |
-
|
| 369 |
-
args = parser.parse_args()
|
| 370 |
-
|
| 371 |
-
# Create benchmark
|
| 372 |
-
benchmark = BotBenchmark(max_moves=args.max_moves, time_limit=args.time_limit)
|
| 373 |
-
|
| 374 |
-
# Create bots
|
| 375 |
-
print(f"Creating {args.bot1_type} bot...")
|
| 376 |
-
bot1 = benchmark.create_bot(args.bot1_type, checkpoint_path=args.bot1_checkpoint, bot_name=args.bot1_name)
|
| 377 |
-
|
| 378 |
-
print(f"Creating {args.bot2_type} bot...")
|
| 379 |
-
bot2 = benchmark.create_bot(args.bot2_type, checkpoint_path=args.bot2_checkpoint, bot_name=args.bot2_name)
|
| 380 |
-
|
| 381 |
-
# Run benchmark
|
| 382 |
-
print(f"\nRunning benchmark: {args.num_games} games")
|
| 383 |
-
print(f"Max moves per game: {args.max_moves}")
|
| 384 |
-
if args.time_limit:
|
| 385 |
-
print(f"Time limit per move: {args.time_limit}s")
|
| 386 |
-
|
| 387 |
-
bot1_stats, bot2_stats = benchmark.run_benchmark(
|
| 388 |
-
bot1, bot2, num_games=args.num_games, alternate_colors=not args.no_alternate, verbose=args.verbose
|
| 389 |
-
)
|
| 390 |
-
|
| 391 |
-
# Print summary
|
| 392 |
-
benchmark.print_summary(bot1_stats, bot2_stats)
|
| 393 |
-
|
| 394 |
-
# Save results
|
| 395 |
-
benchmark.save_results(args.output)
|
| 396 |
-
|
| 397 |
-
|
| 398 |
-
if __name__ == "__main__":
|
| 399 |
-
main()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
chesstransformer/bots/__init__.py
DELETED
|
@@ -1,5 +0,0 @@
|
|
| 1 |
-
from .pos2move_v2_bot import Pos2MoveV2Bot
|
| 2 |
-
from .pos2move_v2_mcts_bot import Pos2MoveV2MctsBot
|
| 3 |
-
from .random_bot import RandomBot
|
| 4 |
-
|
| 5 |
-
__all__ = ["Pos2MoveV2Bot", "Pos2MoveV2MctsBot", "RandomBot"]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
chesstransformer/bots/legacy_position2move_bot.py
DELETED
|
@@ -1,222 +0,0 @@
|
|
| 1 |
-
"""Legacy Position2Move bot for V2 model compatibility.
|
| 2 |
-
|
| 3 |
-
The V2 model was trained with an older architecture that uses:
|
| 4 |
-
- MultiHeadAttention instead of GroupedQueryAttention
|
| 5 |
-
- LayerNorm with gamma/beta instead of RMSNorm with weight
|
| 6 |
-
- Standard GELU FeedForward instead of SwiGLU
|
| 7 |
-
- No CLS token, castling, en passant, or halfmove embeddings
|
| 8 |
-
"""
|
| 9 |
-
|
| 10 |
-
from pathlib import Path
|
| 11 |
-
import json
|
| 12 |
-
import torch
|
| 13 |
-
import torch.nn as nn
|
| 14 |
-
import chess
|
| 15 |
-
from safetensors import safe_open
|
| 16 |
-
from chesstransformer.models.transformer.attention import MultiHeadAttention
|
| 17 |
-
from chesstransformer.models.tokenizer import PostionTokenizer, MoveTokenizer
|
| 18 |
-
|
| 19 |
-
|
| 20 |
-
class LayerNorm(nn.Module):
|
| 21 |
-
"""Legacy LayerNorm with gamma/beta naming."""
|
| 22 |
-
|
| 23 |
-
def __init__(self, embed_dim):
|
| 24 |
-
super().__init__()
|
| 25 |
-
self.gamma = nn.Parameter(torch.ones(embed_dim))
|
| 26 |
-
self.beta = nn.Parameter(torch.zeros(embed_dim))
|
| 27 |
-
self.eps = 1e-6
|
| 28 |
-
|
| 29 |
-
def forward(self, x):
|
| 30 |
-
mean = x.mean(dim=-1, keepdim=True)
|
| 31 |
-
var = x.var(dim=-1, unbiased=False, keepdim=True)
|
| 32 |
-
return self.gamma * (x - mean) / torch.sqrt(var + self.eps) + self.beta
|
| 33 |
-
|
| 34 |
-
|
| 35 |
-
class FeedForward(nn.Module):
|
| 36 |
-
"""Legacy FeedForward network."""
|
| 37 |
-
|
| 38 |
-
def __init__(self, embed_dim):
|
| 39 |
-
super().__init__()
|
| 40 |
-
self.layers = nn.Sequential(
|
| 41 |
-
nn.Linear(embed_dim, 4 * embed_dim),
|
| 42 |
-
nn.GELU(approximate="tanh"),
|
| 43 |
-
nn.Linear(4 * embed_dim, embed_dim),
|
| 44 |
-
)
|
| 45 |
-
|
| 46 |
-
def forward(self, x):
|
| 47 |
-
return self.layers(x)
|
| 48 |
-
|
| 49 |
-
|
| 50 |
-
class LegacyTransformerBlock(nn.Module):
|
| 51 |
-
"""Legacy transformer block for V2 model."""
|
| 52 |
-
|
| 53 |
-
def __init__(
|
| 54 |
-
self,
|
| 55 |
-
embed_dim,
|
| 56 |
-
num_heads,
|
| 57 |
-
context_length,
|
| 58 |
-
dropout,
|
| 59 |
-
qkv_bias,
|
| 60 |
-
mask_future=False,
|
| 61 |
-
):
|
| 62 |
-
super().__init__()
|
| 63 |
-
self.att = MultiHeadAttention(
|
| 64 |
-
d_in=embed_dim,
|
| 65 |
-
d_out=embed_dim,
|
| 66 |
-
num_heads=num_heads,
|
| 67 |
-
context_length=context_length,
|
| 68 |
-
dropout=dropout,
|
| 69 |
-
qkv_bias=qkv_bias,
|
| 70 |
-
mask_future=mask_future,
|
| 71 |
-
)
|
| 72 |
-
self.feed_forward = FeedForward(embed_dim)
|
| 73 |
-
self.norm1 = LayerNorm(embed_dim)
|
| 74 |
-
self.norm2 = LayerNorm(embed_dim)
|
| 75 |
-
self.dropout_shortcut = nn.Dropout(dropout)
|
| 76 |
-
|
| 77 |
-
def forward(self, x):
|
| 78 |
-
shortcut = x
|
| 79 |
-
x = self.norm1(x)
|
| 80 |
-
x = self.att(x)
|
| 81 |
-
x = self.dropout_shortcut(x)
|
| 82 |
-
x = x + shortcut
|
| 83 |
-
|
| 84 |
-
shortcut = x
|
| 85 |
-
x = self.norm2(x)
|
| 86 |
-
x = self.feed_forward(x)
|
| 87 |
-
x = self.dropout_shortcut(x)
|
| 88 |
-
x = x + shortcut
|
| 89 |
-
|
| 90 |
-
return x
|
| 91 |
-
|
| 92 |
-
|
| 93 |
-
class LegacyPosition2MoveModel(nn.Module):
|
| 94 |
-
"""Legacy Position2Move model architecture for V2 compatibility."""
|
| 95 |
-
|
| 96 |
-
def __init__(
|
| 97 |
-
self,
|
| 98 |
-
vocab_size: int = 13,
|
| 99 |
-
move_vocab_size: int = 1968,
|
| 100 |
-
embed_dim: int = 512,
|
| 101 |
-
nb_transformer_layers: int = 8,
|
| 102 |
-
num_heads: int = 8,
|
| 103 |
-
dropout: float = 0.1,
|
| 104 |
-
kvq_bias: bool = False,
|
| 105 |
-
mask_future: bool = False,
|
| 106 |
-
):
|
| 107 |
-
super().__init__()
|
| 108 |
-
|
| 109 |
-
# Board piece embeddings
|
| 110 |
-
self.token_embedding = nn.Embedding(vocab_size, embed_dim)
|
| 111 |
-
self.position_embedding = nn.Embedding(64, embed_dim) # 64 squares
|
| 112 |
-
self.player_embedding = nn.Embedding(2, embed_dim) # 2 players
|
| 113 |
-
|
| 114 |
-
self.dropout = nn.Dropout(dropout)
|
| 115 |
-
self.transformer_blocks = nn.Sequential(
|
| 116 |
-
*[
|
| 117 |
-
LegacyTransformerBlock(
|
| 118 |
-
embed_dim=embed_dim,
|
| 119 |
-
num_heads=num_heads,
|
| 120 |
-
context_length=64,
|
| 121 |
-
dropout=dropout,
|
| 122 |
-
qkv_bias=kvq_bias,
|
| 123 |
-
mask_future=mask_future,
|
| 124 |
-
)
|
| 125 |
-
for _ in range(nb_transformer_layers)
|
| 126 |
-
]
|
| 127 |
-
)
|
| 128 |
-
|
| 129 |
-
self.norm = LayerNorm(embed_dim)
|
| 130 |
-
self.lm_head = nn.Linear(embed_dim, move_vocab_size, bias=False)
|
| 131 |
-
|
| 132 |
-
def forward(self, x, is_white):
|
| 133 |
-
batch_size, seq_len = x.shape
|
| 134 |
-
|
| 135 |
-
# Get embeddings
|
| 136 |
-
token_emb = self.token_embedding(x)
|
| 137 |
-
pos_indices = torch.arange(seq_len, device=x.device)
|
| 138 |
-
pos_emb = self.position_embedding(pos_indices)
|
| 139 |
-
player_emb = self.player_embedding(is_white.long())
|
| 140 |
-
|
| 141 |
-
# Combine embeddings
|
| 142 |
-
x = token_emb + pos_emb.unsqueeze(0) + player_emb.unsqueeze(1)
|
| 143 |
-
x = self.dropout(x)
|
| 144 |
-
|
| 145 |
-
# Transformer blocks
|
| 146 |
-
x = self.transformer_blocks(x)
|
| 147 |
-
x = self.norm(x)
|
| 148 |
-
|
| 149 |
-
# Mean pooling
|
| 150 |
-
x = x.mean(dim=1)
|
| 151 |
-
|
| 152 |
-
# Output
|
| 153 |
-
logits = self.lm_head(x)
|
| 154 |
-
return logits
|
| 155 |
-
|
| 156 |
-
|
| 157 |
-
data_folder = Path(__file__).parents[1] / "data"
|
| 158 |
-
|
| 159 |
-
|
| 160 |
-
class LegacyPosition2MoveBot:
|
| 161 |
-
"""Bot using the legacy V2 model architecture."""
|
| 162 |
-
|
| 163 |
-
def __init__(
|
| 164 |
-
self,
|
| 165 |
-
model_path: str = str((data_folder / "models/position2moveV2.0/model.safetensors").resolve()),
|
| 166 |
-
device: str = "cpu",
|
| 167 |
-
):
|
| 168 |
-
self.device = device
|
| 169 |
-
self.position_tokenizer = PostionTokenizer()
|
| 170 |
-
self.move_tokenizer = MoveTokenizer()
|
| 171 |
-
|
| 172 |
-
config_path = Path(model_path).parent / "config.json"
|
| 173 |
-
|
| 174 |
-
with open(config_path, "r") as f:
|
| 175 |
-
self.config = json.load(f)
|
| 176 |
-
|
| 177 |
-
# Load legacy model
|
| 178 |
-
self.model = LegacyPosition2MoveModel(**self.config).to(device)
|
| 179 |
-
|
| 180 |
-
with safe_open(model_path, framework="pt", device=device) as f:
|
| 181 |
-
state_dict = {k: f.get_tensor(k) for k in f.keys()}
|
| 182 |
-
self.model.load_state_dict(state_dict)
|
| 183 |
-
|
| 184 |
-
self.model.eval()
|
| 185 |
-
|
| 186 |
-
@torch.no_grad()
|
| 187 |
-
def predict(self, board: chess.Board):
|
| 188 |
-
tokens_ids = self.position_tokenizer.encode(board)
|
| 189 |
-
torch_input = torch.tensor(tokens_ids).unsqueeze(0).long().to(self.device)
|
| 190 |
-
|
| 191 |
-
is_white = board.turn
|
| 192 |
-
is_white = torch.tensor([is_white]).bool().to(self.device)
|
| 193 |
-
|
| 194 |
-
logits = self.model(torch_input, is_white)
|
| 195 |
-
|
| 196 |
-
legal_moves = [m.uci() for m in board.legal_moves]
|
| 197 |
-
|
| 198 |
-
mask = torch.full((logits.size(-1),), float("-inf")).to(self.device)
|
| 199 |
-
for move in legal_moves:
|
| 200 |
-
move_id = self.move_tokenizer.encode(move)
|
| 201 |
-
mask[move_id] = 0.0
|
| 202 |
-
|
| 203 |
-
masked_logits = logits[0] + mask
|
| 204 |
-
probs = torch.softmax(masked_logits, dim=-1)
|
| 205 |
-
|
| 206 |
-
# Sample from distribution with temperature
|
| 207 |
-
predicted_move_id = torch.multinomial(probs * 0.2, num_samples=1).item()
|
| 208 |
-
proba = probs[predicted_move_id].item()
|
| 209 |
-
|
| 210 |
-
predicted_move = self.move_tokenizer.decode(predicted_move_id)
|
| 211 |
-
return predicted_move, proba
|
| 212 |
-
|
| 213 |
-
|
| 214 |
-
if __name__ == "__main__":
|
| 215 |
-
bot = LegacyPosition2MoveBot()
|
| 216 |
-
|
| 217 |
-
board = chess.Board()
|
| 218 |
-
print("Initial board:")
|
| 219 |
-
print(board)
|
| 220 |
-
|
| 221 |
-
move, proba = bot.predict(board)
|
| 222 |
-
print(f"Bot suggests move: {move} with probability {proba:.4f}")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
chesstransformer/bots/pos2move_v2_bot.py
DELETED
|
@@ -1,499 +0,0 @@
|
|
| 1 |
-
"""Bot for Pos2MoveV2 with alpha-beta search + BF16."""
|
| 2 |
-
|
| 3 |
-
from __future__ import annotations
|
| 4 |
-
|
| 5 |
-
import json
|
| 6 |
-
import time as _time
|
| 7 |
-
from pathlib import Path
|
| 8 |
-
|
| 9 |
-
import chess
|
| 10 |
-
import chess.polyglot
|
| 11 |
-
import numpy as np
|
| 12 |
-
import torch
|
| 13 |
-
from safetensors import safe_open
|
| 14 |
-
|
| 15 |
-
from chesstransformer.models.tokenizer import PostionTokenizer
|
| 16 |
-
from chesstransformer.models.tokenizer.alphazero_move_encoder import move_to_action_plane
|
| 17 |
-
from chesstransformer.models.transformer.pos2move_v2 import Pos2MoveV2
|
| 18 |
-
|
| 19 |
-
_DEFAULT_RUN = Path(__file__).parents[3] / "data" / "models" / "pos2move_v2.1"
|
| 20 |
-
|
| 21 |
-
# Fixed batch-size buckets. Batched evals are zero-padded up to the next bucket
|
| 22 |
-
# so the (optionally compiled) model only ever sees a handful of static shapes.
|
| 23 |
-
_BATCH_BUCKETS = (4, 8, 16, 32)
|
| 24 |
-
|
| 25 |
-
|
| 26 |
-
class _SearchTimeout(Exception):
|
| 27 |
-
pass
|
| 28 |
-
|
| 29 |
-
|
| 30 |
-
class Pos2MoveV2Bot:
|
| 31 |
-
def __init__(
|
| 32 |
-
self,
|
| 33 |
-
model_dir: str | Path = _DEFAULT_RUN,
|
| 34 |
-
depth: int = 3,
|
| 35 |
-
top_p: float = 0.90,
|
| 36 |
-
time_limit: float = 30.0,
|
| 37 |
-
tt_size: int = 500_000,
|
| 38 |
-
device: str = "cuda" if torch.cuda.is_available() else "cpu",
|
| 39 |
-
use_ema: bool = False,
|
| 40 |
-
compile: bool = True,
|
| 41 |
-
quiescence_depth: int = 4,
|
| 42 |
-
):
|
| 43 |
-
self.depth = depth
|
| 44 |
-
self.top_p = top_p
|
| 45 |
-
self.time_limit = time_limit
|
| 46 |
-
self.quiescence_depth = quiescence_depth
|
| 47 |
-
self.nodes_searched = 0
|
| 48 |
-
self.tt_hits = 0
|
| 49 |
-
self.completed_depth = 0
|
| 50 |
-
self.device = device
|
| 51 |
-
self.position_tokenizer = PostionTokenizer()
|
| 52 |
-
|
| 53 |
-
model_dir = Path(model_dir)
|
| 54 |
-
|
| 55 |
-
config_path = model_dir / "model_config.json"
|
| 56 |
-
if not config_path.exists():
|
| 57 |
-
config_path = model_dir.parent.parent / "model_config.json"
|
| 58 |
-
if not config_path.exists():
|
| 59 |
-
raise FileNotFoundError(f"No model_config.json found for {model_dir}")
|
| 60 |
-
|
| 61 |
-
with open(config_path) as f:
|
| 62 |
-
config = json.load(f)
|
| 63 |
-
|
| 64 |
-
model = Pos2MoveV2(**config)
|
| 65 |
-
|
| 66 |
-
safetensors_path = model_dir / "model.safetensors"
|
| 67 |
-
if not safetensors_path.exists():
|
| 68 |
-
raise FileNotFoundError(f"No model.safetensors in {model_dir}")
|
| 69 |
-
with safe_open(str(safetensors_path), framework="pt", device="cpu") as f:
|
| 70 |
-
state_dict = {k: f.get_tensor(k) for k in f.keys()}
|
| 71 |
-
|
| 72 |
-
if any(k.startswith("_orig_mod.") for k in state_dict):
|
| 73 |
-
state_dict = {k.replace("_orig_mod.", ""): v for k, v in state_dict.items()}
|
| 74 |
-
|
| 75 |
-
model.load_state_dict(state_dict)
|
| 76 |
-
|
| 77 |
-
if use_ema:
|
| 78 |
-
ema_path = model_dir / "ema_state.pt"
|
| 79 |
-
if not ema_path.exists():
|
| 80 |
-
raise FileNotFoundError(f"No ema_state.pt in {model_dir}")
|
| 81 |
-
ema_state = torch.load(str(ema_path), map_location="cpu", weights_only=True)
|
| 82 |
-
ema_state = {k.replace("_orig_mod.", ""): v for k, v in ema_state.items()}
|
| 83 |
-
missing, unexpected = model.load_state_dict(ema_state, strict=False)
|
| 84 |
-
if unexpected:
|
| 85 |
-
print(f"WARNING: EMA contains unexpected keys: {unexpected}")
|
| 86 |
-
print(f"EMA weights loaded ({len(ema_state)} tensors; {len(missing)} buffers kept).")
|
| 87 |
-
|
| 88 |
-
model.eval()
|
| 89 |
-
self.model = model.bfloat16().to(device)
|
| 90 |
-
|
| 91 |
-
self._buf_board = torch.zeros(1, 64, dtype=torch.long, device=device)
|
| 92 |
-
self._buf_player = torch.zeros(1, dtype=torch.long, device=device)
|
| 93 |
-
self._buf_castling = torch.zeros(1, dtype=torch.long, device=device)
|
| 94 |
-
self._buf_ep = torch.zeros(1, dtype=torch.long, device=device)
|
| 95 |
-
self._batch_bufs: dict[int, tuple] = {}
|
| 96 |
-
|
| 97 |
-
self._tt_size = tt_size
|
| 98 |
-
self._tt: dict[int, tuple[np.ndarray, float]] = {}
|
| 99 |
-
self._deadline: float = 0.0
|
| 100 |
-
|
| 101 |
-
# CUDA graphs collapse the Python/kernel-launch overhead that dominates
|
| 102 |
-
# this tiny model at small batch sizes. Bucketed batches keep the number
|
| 103 |
-
# of captured graphs small. Falls back to eager if compilation fails.
|
| 104 |
-
self._compiled = False
|
| 105 |
-
if compile and device == "cuda":
|
| 106 |
-
self._eager_model = self.model
|
| 107 |
-
try:
|
| 108 |
-
self.model = torch.compile(self.model, mode="reduce-overhead")
|
| 109 |
-
self._compiled = True
|
| 110 |
-
except Exception as e: # pragma: no cover - environment dependent
|
| 111 |
-
print(f"WARNING: torch.compile unavailable ({e}); using eager.")
|
| 112 |
-
self.model = self._eager_model
|
| 113 |
-
|
| 114 |
-
self._warmup()
|
| 115 |
-
|
| 116 |
-
print(
|
| 117 |
-
f"Pos2MoveV2Bot loaded (device={device}, depth={depth}, "
|
| 118 |
-
f"top_p={top_p:.0%}, params={sum(p.numel() for p in self.model.parameters()) / 1e6:.1f}M)"
|
| 119 |
-
)
|
| 120 |
-
|
| 121 |
-
@torch.no_grad()
|
| 122 |
-
def _warmup(self):
|
| 123 |
-
try:
|
| 124 |
-
# Batch-1 path (single-position _evaluate).
|
| 125 |
-
for _ in range(3):
|
| 126 |
-
self.model(self._buf_board, self._buf_player, self._buf_castling, self._buf_ep)
|
| 127 |
-
# Each bucket shape used by _batch_run, so compilation / CUDA-graph
|
| 128 |
-
# capture happens up front rather than mid-search.
|
| 129 |
-
for bucket in _BATCH_BUCKETS:
|
| 130 |
-
b, p, c, e = self._batch_buffers(bucket)
|
| 131 |
-
for _ in range(3):
|
| 132 |
-
self.model(b, p, c, e)
|
| 133 |
-
except Exception as e: # pragma: no cover - environment dependent
|
| 134 |
-
if self._compiled:
|
| 135 |
-
print(f"WARNING: compiled warmup failed ({e}); falling back to eager.")
|
| 136 |
-
self.model = self._eager_model
|
| 137 |
-
self._compiled = False
|
| 138 |
-
self.model(self._buf_board, self._buf_player, self._buf_castling, self._buf_ep)
|
| 139 |
-
else:
|
| 140 |
-
raise
|
| 141 |
-
if self.device == "cuda":
|
| 142 |
-
torch.cuda.synchronize()
|
| 143 |
-
|
| 144 |
-
def _adaptive_time_limit(self, board: chess.Board) -> float:
|
| 145 |
-
if self.time_limit <= 0:
|
| 146 |
-
return 0.0
|
| 147 |
-
ply = len(board.move_stack)
|
| 148 |
-
move_number = ply // 2 + 1
|
| 149 |
-
if move_number <= 10:
|
| 150 |
-
scale = 0.10
|
| 151 |
-
elif move_number <= 20:
|
| 152 |
-
scale = 0.40
|
| 153 |
-
elif move_number <= 40:
|
| 154 |
-
scale = 1.0
|
| 155 |
-
else:
|
| 156 |
-
scale = 0.60
|
| 157 |
-
return self.time_limit * scale
|
| 158 |
-
|
| 159 |
-
def predict(self, board: chess.Board) -> tuple[str, float | None]:
|
| 160 |
-
self.nodes_searched = 0
|
| 161 |
-
self.tt_hits = 0
|
| 162 |
-
self.completed_depth = 0
|
| 163 |
-
|
| 164 |
-
effective_limit = self._adaptive_time_limit(board)
|
| 165 |
-
self._deadline = _time.monotonic() + effective_limit if effective_limit > 0 else float("inf")
|
| 166 |
-
|
| 167 |
-
best_move = None
|
| 168 |
-
best_value = 0.0
|
| 169 |
-
|
| 170 |
-
for d in range(1, self.depth + 1):
|
| 171 |
-
stack_len = len(board.move_stack)
|
| 172 |
-
try:
|
| 173 |
-
move, value = self._search_root(board, d)
|
| 174 |
-
best_move = move
|
| 175 |
-
best_value = value
|
| 176 |
-
self.completed_depth = d
|
| 177 |
-
except _SearchTimeout:
|
| 178 |
-
while len(board.move_stack) > stack_len:
|
| 179 |
-
board.pop()
|
| 180 |
-
break
|
| 181 |
-
|
| 182 |
-
if best_move is None:
|
| 183 |
-
best_move = next(iter(board.legal_moves))
|
| 184 |
-
|
| 185 |
-
return best_move.uci(), best_value
|
| 186 |
-
|
| 187 |
-
def get_value(self, board: chess.Board) -> float:
|
| 188 |
-
_, value = self._evaluate(board)
|
| 189 |
-
return value
|
| 190 |
-
|
| 191 |
-
def _check_time(self):
|
| 192 |
-
if _time.monotonic() >= self._deadline:
|
| 193 |
-
raise _SearchTimeout()
|
| 194 |
-
|
| 195 |
-
@torch.no_grad()
|
| 196 |
-
def _evaluate(self, board: chess.Board) -> tuple[np.ndarray, float]:
|
| 197 |
-
key = chess.polyglot.zobrist_hash(board)
|
| 198 |
-
cached = self._tt.get(key)
|
| 199 |
-
if cached is not None:
|
| 200 |
-
self.tt_hits += 1
|
| 201 |
-
return cached
|
| 202 |
-
|
| 203 |
-
position = self.position_tokenizer.encode(board)
|
| 204 |
-
self._buf_board[0] = torch.tensor(position, dtype=torch.long, device=self.device)
|
| 205 |
-
self._buf_player[0] = int(board.turn)
|
| 206 |
-
|
| 207 |
-
castling = 0
|
| 208 |
-
if board.has_kingside_castling_rights(chess.WHITE):
|
| 209 |
-
castling |= 1
|
| 210 |
-
if board.has_queenside_castling_rights(chess.WHITE):
|
| 211 |
-
castling |= 2
|
| 212 |
-
if board.has_kingside_castling_rights(chess.BLACK):
|
| 213 |
-
castling |= 4
|
| 214 |
-
if board.has_queenside_castling_rights(chess.BLACK):
|
| 215 |
-
castling |= 8
|
| 216 |
-
self._buf_castling[0] = castling
|
| 217 |
-
|
| 218 |
-
ep_file = chess.square_file(board.ep_square) if board.has_legal_en_passant() else 8
|
| 219 |
-
self._buf_ep[0] = ep_file
|
| 220 |
-
|
| 221 |
-
move_logits, value_t = self.model(
|
| 222 |
-
self._buf_board, self._buf_player, self._buf_castling, self._buf_ep
|
| 223 |
-
)
|
| 224 |
-
|
| 225 |
-
logits = move_logits[0].float().cpu().numpy()
|
| 226 |
-
value = float(value_t.item())
|
| 227 |
-
|
| 228 |
-
if self._tt_size > 0 and len(self._tt) >= self._tt_size:
|
| 229 |
-
del self._tt[next(iter(self._tt))]
|
| 230 |
-
self._tt[key] = (logits, value)
|
| 231 |
-
|
| 232 |
-
return logits, value
|
| 233 |
-
|
| 234 |
-
def _batch_buffers(self, bucket: int):
|
| 235 |
-
"""Persistent zero-padded device buffers per bucket size.
|
| 236 |
-
|
| 237 |
-
Using a small fixed set of batch shapes keeps torch.compile / CUDA
|
| 238 |
-
graphs from recompiling (and re-capturing) on every distinct batch size.
|
| 239 |
-
"""
|
| 240 |
-
bufs = self._batch_bufs.get(bucket)
|
| 241 |
-
if bufs is None:
|
| 242 |
-
bufs = (
|
| 243 |
-
torch.zeros(bucket, 64, dtype=torch.long, device=self.device),
|
| 244 |
-
torch.zeros(bucket, dtype=torch.long, device=self.device),
|
| 245 |
-
torch.zeros(bucket, dtype=torch.long, device=self.device),
|
| 246 |
-
torch.zeros(bucket, dtype=torch.long, device=self.device),
|
| 247 |
-
)
|
| 248 |
-
self._batch_bufs[bucket] = bufs
|
| 249 |
-
return bufs
|
| 250 |
-
|
| 251 |
-
@staticmethod
|
| 252 |
-
def _bucket_for(n: int) -> int:
|
| 253 |
-
for b in _BATCH_BUCKETS:
|
| 254 |
-
if n <= b:
|
| 255 |
-
return b
|
| 256 |
-
return _BATCH_BUCKETS[-1]
|
| 257 |
-
|
| 258 |
-
@torch.no_grad()
|
| 259 |
-
def _batch_run(self, items: list[tuple[np.ndarray, int, int, int, int]]):
|
| 260 |
-
n = len(items)
|
| 261 |
-
bucket = self._bucket_for(n)
|
| 262 |
-
board_arr, player_arr, castling_arr, ep_arr = self._batch_buffers(bucket)
|
| 263 |
-
# Zero the padding rows (token/index 0 is a valid embedding index).
|
| 264 |
-
board_arr.zero_()
|
| 265 |
-
player_arr.zero_()
|
| 266 |
-
castling_arr.zero_()
|
| 267 |
-
ep_arr.zero_()
|
| 268 |
-
|
| 269 |
-
for i, (pos, player, castling, ep, _) in enumerate(items):
|
| 270 |
-
board_arr[i] = torch.tensor(pos, dtype=torch.long)
|
| 271 |
-
player_arr[i] = player
|
| 272 |
-
castling_arr[i] = castling
|
| 273 |
-
ep_arr[i] = ep
|
| 274 |
-
|
| 275 |
-
move_logits, value_t = self.model(board_arr, player_arr, castling_arr, ep_arr)
|
| 276 |
-
# Slice off padding before the host copy.
|
| 277 |
-
logits_np = move_logits[:n].float().cpu().numpy()
|
| 278 |
-
values_np = value_t[:n].float().cpu().numpy()
|
| 279 |
-
|
| 280 |
-
for i, (_, _, _, _, key) in enumerate(items):
|
| 281 |
-
if self._tt_size > 0 and len(self._tt) >= self._tt_size:
|
| 282 |
-
del self._tt[next(iter(self._tt))]
|
| 283 |
-
self._tt[key] = (logits_np[i], float(values_np[i].item()))
|
| 284 |
-
|
| 285 |
-
def _encode_position(self, board: chess.Board) -> tuple[np.ndarray, int, int, int]:
|
| 286 |
-
position = self.position_tokenizer.encode(board)
|
| 287 |
-
player = int(board.turn)
|
| 288 |
-
castling = 0
|
| 289 |
-
if board.has_kingside_castling_rights(chess.WHITE):
|
| 290 |
-
castling |= 1
|
| 291 |
-
if board.has_queenside_castling_rights(chess.WHITE):
|
| 292 |
-
castling |= 2
|
| 293 |
-
if board.has_kingside_castling_rights(chess.BLACK):
|
| 294 |
-
castling |= 4
|
| 295 |
-
if board.has_queenside_castling_rights(chess.BLACK):
|
| 296 |
-
castling |= 8
|
| 297 |
-
ep_file = chess.square_file(board.ep_square) if board.has_legal_en_passant() else 8
|
| 298 |
-
return position, player, castling, ep_file
|
| 299 |
-
|
| 300 |
-
def _prefetch_children(self, board: chess.Board, moves: list[chess.Move]):
|
| 301 |
-
uncached = []
|
| 302 |
-
for move in moves:
|
| 303 |
-
board.push(move)
|
| 304 |
-
key = chess.polyglot.zobrist_hash(board)
|
| 305 |
-
# Cheap terminal proxy: positions with no legal moves (checkmate /
|
| 306 |
-
# stalemate) are scored directly by _alpha_beta, so skip a wasted
|
| 307 |
-
# model eval on them. Claimable-draw positions (threefold / 50-move)
|
| 308 |
-
# still have legal moves; caching their eval is harmless because
|
| 309 |
-
# _alpha_beta's authoritative is_game_over check overrides it.
|
| 310 |
-
if key not in self._tt and any(board.legal_moves):
|
| 311 |
-
pos, player, castling, ep_file = self._encode_position(board)
|
| 312 |
-
uncached.append((pos, player, castling, ep_file, key))
|
| 313 |
-
board.pop()
|
| 314 |
-
if len(uncached) >= 2:
|
| 315 |
-
self._batch_run(uncached)
|
| 316 |
-
|
| 317 |
-
def _top_p_for_depth(self, depth: int, max_depth: int) -> float:
|
| 318 |
-
plies_from_root = max_depth - depth
|
| 319 |
-
if plies_from_root == 0:
|
| 320 |
-
return self.top_p
|
| 321 |
-
elif plies_from_root <= 2:
|
| 322 |
-
return self.top_p * 0.7
|
| 323 |
-
else:
|
| 324 |
-
return self.top_p * 0.5
|
| 325 |
-
|
| 326 |
-
def _search_root(self, board: chess.Board, depth: int) -> tuple[chess.Move, float]:
|
| 327 |
-
ordered_moves = self._order_moves(board, self.top_p)
|
| 328 |
-
if not ordered_moves:
|
| 329 |
-
return chess.Move.null(), 0.0
|
| 330 |
-
self._prefetch_children(board, [m for m, _ in ordered_moves])
|
| 331 |
-
best_move = ordered_moves[0][0]
|
| 332 |
-
alpha = float("-inf")
|
| 333 |
-
for move, _ in ordered_moves:
|
| 334 |
-
self._check_time()
|
| 335 |
-
board.push(move)
|
| 336 |
-
value = -self._alpha_beta(board, depth - 1, -float("inf"), -alpha, depth)
|
| 337 |
-
board.pop()
|
| 338 |
-
if value > alpha:
|
| 339 |
-
alpha = value
|
| 340 |
-
best_move = move
|
| 341 |
-
return best_move, alpha
|
| 342 |
-
|
| 343 |
-
def _terminal_value(self, board: chess.Board) -> float | None:
|
| 344 |
-
"""Negamax terminal value (side-to-move POV), or None if not terminal.
|
| 345 |
-
|
| 346 |
-
Uses cheap checks instead of ``is_game_over(claim_draw=True)`` (whose
|
| 347 |
-
``can_claim_threefold_repetition`` dominated the profile): no-legal-move
|
| 348 |
-
positions are mate/stalemate, and draws are detected via material,
|
| 349 |
-
3-fold repetition and the 50-move clock — without the expensive
|
| 350 |
-
"a move *could* force a repetition" probe.
|
| 351 |
-
"""
|
| 352 |
-
if not any(board.legal_moves):
|
| 353 |
-
return -1.0 if board.is_check() else 0.0
|
| 354 |
-
if (
|
| 355 |
-
board.is_insufficient_material()
|
| 356 |
-
or board.is_repetition(3)
|
| 357 |
-
or board.halfmove_clock >= 100
|
| 358 |
-
):
|
| 359 |
-
return 0.0
|
| 360 |
-
return None
|
| 361 |
-
|
| 362 |
-
@torch.no_grad()
|
| 363 |
-
def _quiescence(self, board: chess.Board, alpha: float, beta: float, qdepth: int) -> float:
|
| 364 |
-
"""Resolve tactical noise before trusting the value head.
|
| 365 |
-
|
| 366 |
-
Stand-pat on the value head, then search only forcing moves (captures
|
| 367 |
-
and promotions) so the leaf isn't evaluated in the middle of an
|
| 368 |
-
unresolved exchange (the horizon effect).
|
| 369 |
-
"""
|
| 370 |
-
self.nodes_searched += 1
|
| 371 |
-
self._check_time()
|
| 372 |
-
|
| 373 |
-
term = self._terminal_value(board)
|
| 374 |
-
if term is not None:
|
| 375 |
-
return term
|
| 376 |
-
|
| 377 |
-
logits, stand_pat = self._evaluate(board)
|
| 378 |
-
if stand_pat >= beta:
|
| 379 |
-
return beta
|
| 380 |
-
if stand_pat > alpha:
|
| 381 |
-
alpha = stand_pat
|
| 382 |
-
if qdepth <= 0:
|
| 383 |
-
return alpha
|
| 384 |
-
|
| 385 |
-
noisy = []
|
| 386 |
-
for move in board.legal_moves:
|
| 387 |
-
if board.is_capture(move) or move.promotion:
|
| 388 |
-
plane = move_to_action_plane(move.from_square, move.to_square, move.promotion)
|
| 389 |
-
noisy.append((float(logits[move.from_square, plane]), move))
|
| 390 |
-
if not noisy:
|
| 391 |
-
return alpha
|
| 392 |
-
noisy.sort(key=lambda x: -x[0])
|
| 393 |
-
|
| 394 |
-
for _, move in noisy:
|
| 395 |
-
board.push(move)
|
| 396 |
-
score = -self._quiescence(board, -beta, -alpha, qdepth - 1)
|
| 397 |
-
board.pop()
|
| 398 |
-
if score >= beta:
|
| 399 |
-
return beta
|
| 400 |
-
if score > alpha:
|
| 401 |
-
alpha = score
|
| 402 |
-
return alpha
|
| 403 |
-
|
| 404 |
-
def _alpha_beta(self, board: chess.Board, depth: int, alpha: float, beta: float, max_depth: int) -> float:
|
| 405 |
-
self.nodes_searched += 1
|
| 406 |
-
self._check_time()
|
| 407 |
-
|
| 408 |
-
term = self._terminal_value(board)
|
| 409 |
-
if term is not None:
|
| 410 |
-
return term
|
| 411 |
-
|
| 412 |
-
if depth <= 0:
|
| 413 |
-
if self.quiescence_depth > 0:
|
| 414 |
-
return self._quiescence(board, alpha, beta, self.quiescence_depth)
|
| 415 |
-
_, value = self._evaluate(board)
|
| 416 |
-
return value
|
| 417 |
-
|
| 418 |
-
p = self._top_p_for_depth(depth, max_depth)
|
| 419 |
-
ordered_moves = self._order_moves(board, p)
|
| 420 |
-
self._prefetch_children(board, [m for m, _ in ordered_moves])
|
| 421 |
-
|
| 422 |
-
for move, _ in ordered_moves:
|
| 423 |
-
board.push(move)
|
| 424 |
-
score = -self._alpha_beta(board, depth - 1, -beta, -alpha, max_depth)
|
| 425 |
-
board.pop()
|
| 426 |
-
if score >= beta:
|
| 427 |
-
return beta
|
| 428 |
-
if score > alpha:
|
| 429 |
-
alpha = score
|
| 430 |
-
return alpha
|
| 431 |
-
|
| 432 |
-
def _order_moves(self, board: chess.Board, p: float, min_moves: int = 2, max_moves: int = 30):
|
| 433 |
-
logits, _ = self._evaluate(board)
|
| 434 |
-
moves = []
|
| 435 |
-
raw_scores = []
|
| 436 |
-
for move in board.legal_moves:
|
| 437 |
-
plane = move_to_action_plane(move.from_square, move.to_square, move.promotion)
|
| 438 |
-
raw_scores.append(float(logits[move.from_square, plane]))
|
| 439 |
-
moves.append(move)
|
| 440 |
-
if not moves:
|
| 441 |
-
return []
|
| 442 |
-
|
| 443 |
-
scores = np.array(raw_scores, dtype=np.float64)
|
| 444 |
-
scores -= scores.max()
|
| 445 |
-
exp_scores = np.exp(scores)
|
| 446 |
-
probs = exp_scores / exp_scores.sum()
|
| 447 |
-
|
| 448 |
-
order = np.argsort(-probs)
|
| 449 |
-
cumsum = 0.0
|
| 450 |
-
selected = []
|
| 451 |
-
for idx in order:
|
| 452 |
-
selected.append((moves[idx], float(probs[idx])))
|
| 453 |
-
cumsum += probs[idx]
|
| 454 |
-
if cumsum >= p and len(selected) >= min_moves:
|
| 455 |
-
break
|
| 456 |
-
if len(selected) >= max_moves:
|
| 457 |
-
break
|
| 458 |
-
|
| 459 |
-
while len(selected) < min_moves and len(selected) < len(moves):
|
| 460 |
-
selected.append((moves[order[len(selected)]], float(probs[order[len(selected)]])))
|
| 461 |
-
|
| 462 |
-
return selected
|
| 463 |
-
|
| 464 |
-
|
| 465 |
-
if __name__ == "__main__":
|
| 466 |
-
import argparse
|
| 467 |
-
import time
|
| 468 |
-
|
| 469 |
-
parser = argparse.ArgumentParser()
|
| 470 |
-
parser.add_argument("model_dir", nargs="?", default=str(_DEFAULT_RUN))
|
| 471 |
-
parser.add_argument("depth", nargs="?", type=int, default=3)
|
| 472 |
-
parser.add_argument("top_p", nargs="?", type=float, default=0.90)
|
| 473 |
-
parser.add_argument("--ema", action="store_true")
|
| 474 |
-
args = parser.parse_args()
|
| 475 |
-
|
| 476 |
-
bot = Pos2MoveV2Bot(
|
| 477 |
-
model_dir=args.model_dir, depth=args.depth, top_p=args.top_p,
|
| 478 |
-
time_limit=30.0, use_ema=args.ema,
|
| 479 |
-
)
|
| 480 |
-
|
| 481 |
-
board = chess.Board()
|
| 482 |
-
print(f"\nStarting position:\n{board}\n")
|
| 483 |
-
|
| 484 |
-
for i in range(10):
|
| 485 |
-
t0 = time.time()
|
| 486 |
-
move_uci, value = bot.predict(board)
|
| 487 |
-
elapsed = time.time() - t0
|
| 488 |
-
print(
|
| 489 |
-
f"{'White' if board.turn else 'Black'} plays {move_uci} "
|
| 490 |
-
f"(depth={bot.completed_depth}/{args.depth}, value={value:+.3f}, "
|
| 491 |
-
f"nodes={bot.nodes_searched}, tt_hits={bot.tt_hits}, time={elapsed:.2f}s)"
|
| 492 |
-
)
|
| 493 |
-
board.push_uci(move_uci)
|
| 494 |
-
print(board)
|
| 495 |
-
print()
|
| 496 |
-
|
| 497 |
-
if board.is_game_over():
|
| 498 |
-
print(f"Game over: {board.result()}")
|
| 499 |
-
break
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
chesstransformer/bots/pos2move_v2_mcts_bot.py
DELETED
|
@@ -1,318 +0,0 @@
|
|
| 1 |
-
"""AlphaZero-style PUCT MCTS engine for Pos2MoveV2.
|
| 2 |
-
|
| 3 |
-
Subclasses :class:`Pos2MoveV2Bot` to reuse its (torch.compiled, transposition-
|
| 4 |
-
cached) ``_evaluate`` — the policy head supplies move priors and the value head
|
| 5 |
-
scores leaves. Move selection is by visit count (the standard AlphaZero choice).
|
| 6 |
-
|
| 7 |
-
Node representation is **edge-centric**: a node stores its children's visit
|
| 8 |
-
counts / values / priors as numpy arrays, and child ``_Node`` objects are created
|
| 9 |
-
lazily only when a child is actually descended into. This avoids allocating a
|
| 10 |
-
node per legal move at every expansion (which dominated latency) and lets PUCT
|
| 11 |
-
selection run vectorized over the arrays. Leaves are collected in batches with
|
| 12 |
-
virtual loss so the network sees one batched forward per wave.
|
| 13 |
-
"""
|
| 14 |
-
|
| 15 |
-
from __future__ import annotations
|
| 16 |
-
|
| 17 |
-
import math
|
| 18 |
-
|
| 19 |
-
import chess
|
| 20 |
-
import numpy as np
|
| 21 |
-
|
| 22 |
-
from chesstransformer.bots.pos2move_v2_bot import _BATCH_BUCKETS, _DEFAULT_RUN, Pos2MoveV2Bot
|
| 23 |
-
from chesstransformer.models.tokenizer.alphazero_move_encoder import move_to_action_plane
|
| 24 |
-
|
| 25 |
-
|
| 26 |
-
class _Node:
|
| 27 |
-
"""A search node. Child visit stats live in arrays here (edge-centric); child
|
| 28 |
-
``_Node`` objects are created lazily on first descent. A node is *expanded*
|
| 29 |
-
once ``moves`` is set."""
|
| 30 |
-
|
| 31 |
-
__slots__ = ("moves", "priors", "child_N", "child_W", "children")
|
| 32 |
-
|
| 33 |
-
def __init__(self):
|
| 34 |
-
self.moves: list[chess.Move] | None = None
|
| 35 |
-
self.priors: np.ndarray | None = None
|
| 36 |
-
self.child_N: np.ndarray | None = None # int64, per child
|
| 37 |
-
self.child_W: np.ndarray | None = None # float64, child-POV total value
|
| 38 |
-
self.children: list[_Node | None] | None = None
|
| 39 |
-
|
| 40 |
-
@property
|
| 41 |
-
def expanded(self) -> bool:
|
| 42 |
-
return self.moves is not None
|
| 43 |
-
|
| 44 |
-
|
| 45 |
-
class Pos2MoveV2MctsBot(Pos2MoveV2Bot):
|
| 46 |
-
def __init__(
|
| 47 |
-
self,
|
| 48 |
-
*args,
|
| 49 |
-
num_simulations: int = 800,
|
| 50 |
-
c_puct: float = 1.0,
|
| 51 |
-
prior_temp: float = 1.0,
|
| 52 |
-
fpu: float | None = 0.2,
|
| 53 |
-
sim_batch: int = 16,
|
| 54 |
-
tree_reuse: bool = True,
|
| 55 |
-
move_temp: float = 0.0,
|
| 56 |
-
move_temp_plies: int = 0,
|
| 57 |
-
**kwargs,
|
| 58 |
-
):
|
| 59 |
-
super().__init__(*args, **kwargs)
|
| 60 |
-
self.num_simulations = num_simulations
|
| 61 |
-
self.c_puct = c_puct
|
| 62 |
-
# Move selection. By default the move is argmax(visit count) — fully
|
| 63 |
-
# deterministic (what the gauntlet / API rely on). For variety (e.g. so
|
| 64 |
-
# the opening isn't identical every game), set move_temp > 0 to *sample*
|
| 65 |
-
# the move ∝ visit_count**(1/move_temp) for the first move_temp_plies
|
| 66 |
-
# half-moves, then fall back to argmax. move_temp=1 samples proportional
|
| 67 |
-
# to visits; higher is more random.
|
| 68 |
-
self.move_temp = move_temp
|
| 69 |
-
self.move_temp_plies = move_temp_plies
|
| 70 |
-
# Tree reuse: keep the searched subtree under the moves actually played
|
| 71 |
-
# and re-root it next move, so prior visits carry over (effectively
|
| 72 |
-
# deeper search for free). Requires the caller to keep board.move_stack
|
| 73 |
-
# across calls; falls back to a fresh tree otherwise (e.g. boards rebuilt
|
| 74 |
-
# from FEN with no history).
|
| 75 |
-
self.tree_reuse = tree_reuse
|
| 76 |
-
self._reuse_root: _Node | None = None
|
| 77 |
-
self._reuse_moves: list | None = None
|
| 78 |
-
# prior_temp > 1 flattens the policy priors so good-but-low-prior moves
|
| 79 |
-
# still get explored (the value head can then promote them). fpu (first-
|
| 80 |
-
# play-urgency): value unvisited children as parent_Q - fpu; None keeps
|
| 81 |
-
# the legacy behavior (unvisited contribute 0 to selection).
|
| 82 |
-
self.prior_temp = prior_temp
|
| 83 |
-
self.fpu = fpu
|
| 84 |
-
# Leaves collected per network call. >1 amortizes the GPU->CPU sync that
|
| 85 |
-
# otherwise dominates (one stall per simulation). 1 = legacy single-leaf.
|
| 86 |
-
# Capped at the largest batch bucket so _batch_run's preallocated buffers
|
| 87 |
-
# (and the captured CUDA graphs) are never overrun.
|
| 88 |
-
self.sim_batch = max(1, min(sim_batch, _BATCH_BUCKETS[-1]))
|
| 89 |
-
|
| 90 |
-
# ── expansion ────────────────────────────────────────────────────────
|
| 91 |
-
def _compute_priors(self, moves: list[chess.Move], logits: np.ndarray) -> np.ndarray:
|
| 92 |
-
"""Softmax policy priors over legal moves, with temperature flattening."""
|
| 93 |
-
scores = np.array(
|
| 94 |
-
[
|
| 95 |
-
logits[m.from_square, move_to_action_plane(m.from_square, m.to_square, m.promotion)]
|
| 96 |
-
for m in moves
|
| 97 |
-
],
|
| 98 |
-
dtype=np.float64,
|
| 99 |
-
)
|
| 100 |
-
scores /= self.prior_temp
|
| 101 |
-
scores -= scores.max()
|
| 102 |
-
exp = np.exp(scores)
|
| 103 |
-
return exp / exp.sum()
|
| 104 |
-
|
| 105 |
-
def _expand_node(self, node: _Node, moves: list[chess.Move], priors: np.ndarray):
|
| 106 |
-
self.nodes_searched += 1
|
| 107 |
-
n = len(moves)
|
| 108 |
-
node.moves = moves
|
| 109 |
-
node.priors = priors
|
| 110 |
-
node.child_N = np.zeros(n, dtype=np.int64)
|
| 111 |
-
node.child_W = np.zeros(n, dtype=np.float64)
|
| 112 |
-
node.children = [None] * n
|
| 113 |
-
|
| 114 |
-
def _expand_leaf(self, node: _Node, board: chess.Board) -> float:
|
| 115 |
-
"""Evaluate ``board`` (policy + value) and expand ``node``; return value."""
|
| 116 |
-
logits, value = self._evaluate(board)
|
| 117 |
-
moves = list(board.legal_moves)
|
| 118 |
-
self._expand_node(node, moves, self._compute_priors(moves, logits))
|
| 119 |
-
return value
|
| 120 |
-
|
| 121 |
-
# ── selection / descent / backup ─────────────────────────────────────
|
| 122 |
-
def _select_idx(self, node: _Node) -> int:
|
| 123 |
-
"""Vectorized PUCT: pick the child index maximizing -Q + U."""
|
| 124 |
-
N = node.child_N
|
| 125 |
-
total = int(N.sum())
|
| 126 |
-
sqrt_total = math.sqrt(1.0 + total) # +1 ≈ the node's own eval visit
|
| 127 |
-
# value-to-parent = -child_Q (child stores value from its own POV).
|
| 128 |
-
q_parent = np.zeros_like(node.child_W)
|
| 129 |
-
np.divide(-node.child_W, N, out=q_parent, where=N > 0)
|
| 130 |
-
if self.fpu is not None:
|
| 131 |
-
parent_q = (-node.child_W.sum() / total) if total > 0 else 0.0
|
| 132 |
-
q_parent[N == 0] = parent_q - self.fpu
|
| 133 |
-
u = (self.c_puct * sqrt_total) * node.priors / (1.0 + N)
|
| 134 |
-
return int(np.argmax(q_parent + u))
|
| 135 |
-
|
| 136 |
-
def _descend(self, node: _Node, board: chess.Board, apply_vl: bool):
|
| 137 |
-
"""Walk to an unexpanded leaf, pushing moves. Returns (leaf_node, path),
|
| 138 |
-
where path is the list of (node, child_idx) edges traversed."""
|
| 139 |
-
path: list[tuple[_Node, int]] = []
|
| 140 |
-
while node.expanded:
|
| 141 |
-
idx = self._select_idx(node)
|
| 142 |
-
if apply_vl:
|
| 143 |
-
# Virtual loss: a virtual *win* in the child's POV looks like a
|
| 144 |
-
# loss to this node under -Q selection, so concurrent descents in
|
| 145 |
-
# the batch avoid the path.
|
| 146 |
-
node.child_N[idx] += 1
|
| 147 |
-
node.child_W[idx] += 1.0
|
| 148 |
-
path.append((node, idx))
|
| 149 |
-
board.push(node.moves[idx])
|
| 150 |
-
child = node.children[idx]
|
| 151 |
-
if child is None:
|
| 152 |
-
child = _Node()
|
| 153 |
-
node.children[idx] = child
|
| 154 |
-
node = child
|
| 155 |
-
return node, path
|
| 156 |
-
|
| 157 |
-
@staticmethod
|
| 158 |
-
def _backup(path: list[tuple[_Node, int]], value: float):
|
| 159 |
-
"""Negamax backup over edges (no virtual loss)."""
|
| 160 |
-
v = value
|
| 161 |
-
for node, idx in reversed(path):
|
| 162 |
-
node.child_N[idx] += 1
|
| 163 |
-
node.child_W[idx] += v
|
| 164 |
-
v = -v
|
| 165 |
-
|
| 166 |
-
@staticmethod
|
| 167 |
-
def _backup_vl(path: list[tuple[_Node, int]], value: float):
|
| 168 |
-
"""Backup that undoes the per-edge virtual loss (+1 added during descent)
|
| 169 |
-
and applies the real, sign-alternating value. N already counted."""
|
| 170 |
-
v = value
|
| 171 |
-
for node, idx in reversed(path):
|
| 172 |
-
node.child_W[idx] += v - 1.0
|
| 173 |
-
v = -v
|
| 174 |
-
|
| 175 |
-
# ── search drivers ───────────────────────────────────────────────────
|
| 176 |
-
def _simulate(self, root: _Node, board: chess.Board):
|
| 177 |
-
node, path = self._descend(root, board, apply_vl=False)
|
| 178 |
-
pushed = len(path)
|
| 179 |
-
term = self._terminal_value(board)
|
| 180 |
-
value = term if term is not None else self._expand_leaf(node, board)
|
| 181 |
-
self._backup(path, value)
|
| 182 |
-
for _ in range(pushed):
|
| 183 |
-
board.pop()
|
| 184 |
-
|
| 185 |
-
def _run_batch(self, root: _Node, board: chess.Board, batch_n: int):
|
| 186 |
-
"""Collect up to ``batch_n`` leaves (virtual loss during descent),
|
| 187 |
-
evaluate the unique uncached non-terminal ones in one batched forward,
|
| 188 |
-
then expand and back them up."""
|
| 189 |
-
leaves = [] # (path, leaf_node, key, moves, cached eval or None)
|
| 190 |
-
items_by_key = {} # key -> (pos, player, castling, ep, key)
|
| 191 |
-
|
| 192 |
-
for _ in range(batch_n):
|
| 193 |
-
node, path = self._descend(root, board, apply_vl=True)
|
| 194 |
-
term = self._terminal_value(board)
|
| 195 |
-
if term is not None:
|
| 196 |
-
self._backup_vl(path, term)
|
| 197 |
-
else:
|
| 198 |
-
key = chess.polyglot.zobrist_hash(board)
|
| 199 |
-
moves = list(board.legal_moves)
|
| 200 |
-
# Reuse cached policy+value (transpositions / reused subtree /
|
| 201 |
-
# prior moves); only batch positions we haven't evaluated yet.
|
| 202 |
-
# Capture the cached entry *now*: once the TT is at capacity,
|
| 203 |
-
# _batch_run's inserts evict oldest entries, so a hit found here
|
| 204 |
-
# may no longer be in the TT after the batch runs.
|
| 205 |
-
cached = self._tt.get(key)
|
| 206 |
-
if cached is not None:
|
| 207 |
-
self.tt_hits += 1
|
| 208 |
-
elif key not in items_by_key:
|
| 209 |
-
pos, player, castling, ep = self._encode_position(board)
|
| 210 |
-
items_by_key[key] = (pos, player, castling, ep, key)
|
| 211 |
-
leaves.append((path, node, key, moves, cached))
|
| 212 |
-
for _ in range(len(path)):
|
| 213 |
-
board.pop()
|
| 214 |
-
|
| 215 |
-
if items_by_key:
|
| 216 |
-
self._batch_run(list(items_by_key.values())) # populates self._tt
|
| 217 |
-
|
| 218 |
-
for path, node, key, moves, cached in leaves:
|
| 219 |
-
logits, value = cached if cached is not None else self._tt[key]
|
| 220 |
-
if not node.expanded:
|
| 221 |
-
self._expand_node(node, moves, self._compute_priors(moves, logits))
|
| 222 |
-
self._backup_vl(path, value)
|
| 223 |
-
|
| 224 |
-
# ── tree reuse ───────────────────────────────────────────────────────
|
| 225 |
-
def _take_reuse_root(self, board: chess.Board) -> _Node | None:
|
| 226 |
-
"""Re-root the retained tree to ``board`` if it follows the previous
|
| 227 |
-
search position by a few plies. Consumes the stored tree."""
|
| 228 |
-
root = self._reuse_root
|
| 229 |
-
self._reuse_root = None
|
| 230 |
-
if root is None or self._reuse_moves is None:
|
| 231 |
-
return None
|
| 232 |
-
ms = board.move_stack
|
| 233 |
-
rp = len(self._reuse_moves)
|
| 234 |
-
gap = len(ms) - rp
|
| 235 |
-
if gap < 1 or gap > 4 or list(ms[:rp]) != self._reuse_moves:
|
| 236 |
-
return None
|
| 237 |
-
node = root
|
| 238 |
-
for mv in ms[rp:]:
|
| 239 |
-
if not node.expanded:
|
| 240 |
-
return None
|
| 241 |
-
try:
|
| 242 |
-
idx = node.moves.index(mv)
|
| 243 |
-
except ValueError:
|
| 244 |
-
return None
|
| 245 |
-
child = node.children[idx]
|
| 246 |
-
if child is None or not child.expanded:
|
| 247 |
-
return None
|
| 248 |
-
node = child
|
| 249 |
-
return node if node.expanded else None
|
| 250 |
-
|
| 251 |
-
def predict(self, board: chess.Board) -> tuple[str, float | None]:
|
| 252 |
-
self.nodes_searched = 0
|
| 253 |
-
self.tt_hits = 0
|
| 254 |
-
self.completed_depth = self.num_simulations
|
| 255 |
-
|
| 256 |
-
if not any(board.legal_moves):
|
| 257 |
-
return chess.Move.null().uci(), 0.0
|
| 258 |
-
|
| 259 |
-
root = self._take_reuse_root(board) if self.tree_reuse else None
|
| 260 |
-
if root is None:
|
| 261 |
-
root = _Node()
|
| 262 |
-
self._expand_leaf(root, board)
|
| 263 |
-
|
| 264 |
-
if self.sim_batch <= 1:
|
| 265 |
-
for _ in range(self.num_simulations):
|
| 266 |
-
self._simulate(root, board)
|
| 267 |
-
else:
|
| 268 |
-
done = 0
|
| 269 |
-
while done < self.num_simulations:
|
| 270 |
-
n = min(self.sim_batch, self.num_simulations - done)
|
| 271 |
-
self._run_batch(root, board, n)
|
| 272 |
-
done += n
|
| 273 |
-
|
| 274 |
-
ply = 2 * (board.fullmove_number - 1) + (0 if board.turn == chess.WHITE else 1)
|
| 275 |
-
if self.move_temp > 0 and ply < self.move_temp_plies and root.child_N.sum() > 0:
|
| 276 |
-
# Sample ∝ visit_count**(1/temp) over the opening plies for variety.
|
| 277 |
-
weights = root.child_N.astype(np.float64) ** (1.0 / self.move_temp)
|
| 278 |
-
total = weights.sum()
|
| 279 |
-
best_idx = int(np.random.choice(len(weights), p=weights / total)) if total > 0 \
|
| 280 |
-
else int(np.argmax(root.child_N))
|
| 281 |
-
else:
|
| 282 |
-
best_idx = int(np.argmax(root.child_N))
|
| 283 |
-
best_move = root.moves[best_idx]
|
| 284 |
-
n = int(root.child_N[best_idx])
|
| 285 |
-
# Root value from side-to-move POV (negate child's POV).
|
| 286 |
-
best_value = float(-root.child_W[best_idx] / n) if n > 0 else 0.0
|
| 287 |
-
|
| 288 |
-
if self.tree_reuse:
|
| 289 |
-
self._reuse_root = root
|
| 290 |
-
self._reuse_moves = list(board.move_stack)
|
| 291 |
-
|
| 292 |
-
return best_move.uci(), best_value
|
| 293 |
-
|
| 294 |
-
|
| 295 |
-
if __name__ == "__main__":
|
| 296 |
-
import argparse
|
| 297 |
-
import time
|
| 298 |
-
|
| 299 |
-
parser = argparse.ArgumentParser()
|
| 300 |
-
parser.add_argument("model_dir", nargs="?", default=str(_DEFAULT_RUN))
|
| 301 |
-
parser.add_argument("--sims", type=int, default=800)
|
| 302 |
-
parser.add_argument("--c-puct", type=float, default=1.0)
|
| 303 |
-
parser.add_argument("--ema", action="store_true")
|
| 304 |
-
args = parser.parse_args()
|
| 305 |
-
|
| 306 |
-
bot = Pos2MoveV2MctsBot(
|
| 307 |
-
model_dir=args.model_dir, num_simulations=args.sims, c_puct=args.c_puct,
|
| 308 |
-
time_limit=0.0, use_ema=args.ema,
|
| 309 |
-
)
|
| 310 |
-
board = chess.Board()
|
| 311 |
-
for _ in range(8):
|
| 312 |
-
t0 = time.time()
|
| 313 |
-
mv, val = bot.predict(board)
|
| 314 |
-
print(f"{'W' if board.turn else 'B'} {mv} val={val:+.3f} "
|
| 315 |
-
f"sims={bot.num_simulations} nodes={bot.nodes_searched} time={time.time()-t0:.2f}s")
|
| 316 |
-
board.push_uci(mv)
|
| 317 |
-
if board.is_game_over():
|
| 318 |
-
break
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
chesstransformer/bots/position2move_bot.py
DELETED
|
@@ -1,88 +0,0 @@
|
|
| 1 |
-
from pathlib import Path
|
| 2 |
-
import json
|
| 3 |
-
import torch
|
| 4 |
-
import chess
|
| 5 |
-
from safetensors import safe_open
|
| 6 |
-
from chesstransformer.models.transformer.position2move import Position2MoveModel
|
| 7 |
-
from chesstransformer.models.tokenizer import PostionTokenizer, MoveTokenizer
|
| 8 |
-
|
| 9 |
-
data_foler = Path(__file__).parents[1] / "data"
|
| 10 |
-
|
| 11 |
-
|
| 12 |
-
class Position2MoveBot:
|
| 13 |
-
def __init__(
|
| 14 |
-
self,
|
| 15 |
-
model_path: str = str((data_foler / "models/position2moveV2.1/best_model/model.safetensors").resolve()),
|
| 16 |
-
device: str = "cpu",
|
| 17 |
-
):
|
| 18 |
-
self.device = device
|
| 19 |
-
self.position_tokenizer = PostionTokenizer()
|
| 20 |
-
self.move_tokenizer = MoveTokenizer()
|
| 21 |
-
|
| 22 |
-
config_path = Path(model_path).parent / "config.json"
|
| 23 |
-
|
| 24 |
-
with open(config_path, "r") as f:
|
| 25 |
-
self.config = json.load(f)
|
| 26 |
-
|
| 27 |
-
# Load model
|
| 28 |
-
self.model = Position2MoveModel(**self.config).to(device)
|
| 29 |
-
|
| 30 |
-
with safe_open(model_path, framework="pt", device=device) as f:
|
| 31 |
-
state_dict = {k: f.get_tensor(k) for k in f.keys()}
|
| 32 |
-
|
| 33 |
-
# Handle compiled model prefix (_orig_mod.)
|
| 34 |
-
if any(k.startswith("_orig_mod.") for k in state_dict.keys()):
|
| 35 |
-
state_dict = {k.replace("_orig_mod.", ""): v for k, v in state_dict.items()}
|
| 36 |
-
|
| 37 |
-
self.model.load_state_dict(state_dict)
|
| 38 |
-
|
| 39 |
-
self.model.eval()
|
| 40 |
-
|
| 41 |
-
@torch.no_grad()
|
| 42 |
-
def predict(self, board: chess.Board):
|
| 43 |
-
tokens_ids = self.position_tokenizer.encode(board)
|
| 44 |
-
torch_input = torch.tensor(tokens_ids).unsqueeze(0).long().to(self.device) # Add batch dimension
|
| 45 |
-
|
| 46 |
-
is_white = board.turn
|
| 47 |
-
is_white = torch.tensor([is_white]).bool().to(self.device) # Add batch dimension
|
| 48 |
-
|
| 49 |
-
logits = self.model(torch_input, is_white)
|
| 50 |
-
|
| 51 |
-
legal_moves = [m.uci() for m in board.legal_moves]
|
| 52 |
-
|
| 53 |
-
mask = torch.full((logits.size(-1),), float("-inf")).to(self.device)
|
| 54 |
-
for move in legal_moves:
|
| 55 |
-
move_id = self.move_tokenizer.encode(move)
|
| 56 |
-
mask[move_id] = 0.0
|
| 57 |
-
|
| 58 |
-
masked_logits = logits[0] + mask # Assuming batch size of 1
|
| 59 |
-
probs = torch.softmax(masked_logits, dim=-1)
|
| 60 |
-
# draw the move from the distribution
|
| 61 |
-
predicted_move_id = torch.multinomial(probs * 0.2, num_samples=1).item()
|
| 62 |
-
proba = probs[predicted_move_id].item()
|
| 63 |
-
|
| 64 |
-
predicted_move = self.move_tokenizer.decode(predicted_move_id)
|
| 65 |
-
return predicted_move, proba
|
| 66 |
-
|
| 67 |
-
|
| 68 |
-
if __name__ == "__main__":
|
| 69 |
-
bot = Position2MoveBot()
|
| 70 |
-
|
| 71 |
-
board = chess.Board()
|
| 72 |
-
|
| 73 |
-
print("Initial board:")
|
| 74 |
-
print(board)
|
| 75 |
-
|
| 76 |
-
move, proba = bot.predict(board)
|
| 77 |
-
print(f"Bot suggests move: {move} with probability {proba:.4f}")
|
| 78 |
-
|
| 79 |
-
board.push_uci(move)
|
| 80 |
-
print("Board after bot move:")
|
| 81 |
-
print(board)
|
| 82 |
-
|
| 83 |
-
move, proba = bot.predict(board)
|
| 84 |
-
print(f"Bot suggests move: {move} with probability {proba:.4f}")
|
| 85 |
-
|
| 86 |
-
board.push_uci(move)
|
| 87 |
-
print("Board after bot move:")
|
| 88 |
-
print(board)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
chesstransformer/bots/random_bot.py
DELETED
|
@@ -1,33 +0,0 @@
|
|
| 1 |
-
import chess
|
| 2 |
-
import random
|
| 3 |
-
|
| 4 |
-
|
| 5 |
-
class RandomBot:
|
| 6 |
-
def __init__(self):
|
| 7 |
-
pass
|
| 8 |
-
|
| 9 |
-
def predict(self, board: chess.Board) -> chess.Move:
|
| 10 |
-
legal_moves = list(board.legal_moves)
|
| 11 |
-
if not legal_moves:
|
| 12 |
-
raise ValueError("No legal moves available.")
|
| 13 |
-
# pick a random move
|
| 14 |
-
move = random.choice(legal_moves)
|
| 15 |
-
return move.uci(), None
|
| 16 |
-
|
| 17 |
-
|
| 18 |
-
if __name__ == "__main__":
|
| 19 |
-
bot = RandomBot()
|
| 20 |
-
board = chess.Board()
|
| 21 |
-
print("Initial board:")
|
| 22 |
-
print(board)
|
| 23 |
-
move = bot.predict(board)
|
| 24 |
-
print(f"RandomBot suggests move: {move}")
|
| 25 |
-
board.push(move)
|
| 26 |
-
print("Board after move:")
|
| 27 |
-
print(board)
|
| 28 |
-
|
| 29 |
-
move = bot.predict(board)
|
| 30 |
-
print(f"RandomBot suggests move: {move}")
|
| 31 |
-
board.push(move)
|
| 32 |
-
print("Board after move:")
|
| 33 |
-
print(board)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
chesstransformer/datasets/__init__.py
DELETED
|
@@ -1,10 +0,0 @@
|
|
| 1 |
-
from .lichess import LichessDataset
|
| 2 |
-
from .lichess_simple_uci import LichessSimpleUciDataset
|
| 3 |
-
from .puzzle_dataset import LichessPuzzleDataset, LichessPuzzleFullSolutionDataset
|
| 4 |
-
|
| 5 |
-
__all__ = [
|
| 6 |
-
"LichessDataset",
|
| 7 |
-
"LichessSimpleUciDataset",
|
| 8 |
-
"LichessPuzzleDataset",
|
| 9 |
-
"LichessPuzzleFullSolutionDataset",
|
| 10 |
-
]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
chesstransformer/datasets/dataset_h5_convertor.py
DELETED
|
@@ -1,404 +0,0 @@
|
|
| 1 |
-
import h5py
|
| 2 |
-
import chess.pgn
|
| 3 |
-
import zstandard as zstd
|
| 4 |
-
import numpy as np
|
| 5 |
-
from tqdm import tqdm
|
| 6 |
-
from multiprocessing import Pool, cpu_count
|
| 7 |
-
from chesstransformer.models.tokenizer.move_tokenizer import MoveTokenizer
|
| 8 |
-
import io
|
| 9 |
-
from pathlib import Path
|
| 10 |
-
|
| 11 |
-
# Per-worker globals initialised once per process
|
| 12 |
-
_worker_tokenizer = None
|
| 13 |
-
_worker_exclude_tcs = []
|
| 14 |
-
|
| 15 |
-
|
| 16 |
-
def _worker_init(exclude_tcs):
|
| 17 |
-
global _worker_tokenizer, _worker_exclude_tcs
|
| 18 |
-
_worker_tokenizer = MoveTokenizer()
|
| 19 |
-
_worker_exclude_tcs = exclude_tcs
|
| 20 |
-
|
| 21 |
-
|
| 22 |
-
def _process_raw_game(raw_pgn: str):
|
| 23 |
-
"""Full pipeline in each worker: text-filter → parse → tokenize.
|
| 24 |
-
|
| 25 |
-
Moving parse into workers parallelises the most expensive step.
|
| 26 |
-
The fast text-level filter avoids even calling chess.pgn.read_game
|
| 27 |
-
for excluded time controls.
|
| 28 |
-
"""
|
| 29 |
-
global _worker_tokenizer, _worker_exclude_tcs
|
| 30 |
-
|
| 31 |
-
# Fast text-level filter before expensive PGN parsing
|
| 32 |
-
if _worker_exclude_tcs:
|
| 33 |
-
idx = raw_pgn.find('[Event "')
|
| 34 |
-
if idx >= 0:
|
| 35 |
-
end = raw_pgn.find('"]', idx + 8)
|
| 36 |
-
if end >= 0:
|
| 37 |
-
event_lower = raw_pgn[idx + 8 : end].lower()
|
| 38 |
-
if any(tc in event_lower for tc in _worker_exclude_tcs):
|
| 39 |
-
return None
|
| 40 |
-
|
| 41 |
-
game = chess.pgn.read_game(io.StringIO(raw_pgn))
|
| 42 |
-
if game is None:
|
| 43 |
-
return None
|
| 44 |
-
|
| 45 |
-
if game.headers.get("Variant", "Standard") != "Standard":
|
| 46 |
-
return None
|
| 47 |
-
|
| 48 |
-
result_str = game.headers.get("Result", "*")
|
| 49 |
-
if result_str == "1/2-1/2":
|
| 50 |
-
result = 0
|
| 51 |
-
elif result_str == "1-0":
|
| 52 |
-
result = 1
|
| 53 |
-
elif result_str == "0-1":
|
| 54 |
-
result = 2
|
| 55 |
-
else:
|
| 56 |
-
return None
|
| 57 |
-
|
| 58 |
-
try:
|
| 59 |
-
white_elo = int(game.headers.get("WhiteElo", "1500"))
|
| 60 |
-
except ValueError:
|
| 61 |
-
white_elo = 1500
|
| 62 |
-
try:
|
| 63 |
-
black_elo = int(game.headers.get("BlackElo", "1500"))
|
| 64 |
-
except ValueError:
|
| 65 |
-
black_elo = 1500
|
| 66 |
-
|
| 67 |
-
tokens = []
|
| 68 |
-
for move in game.mainline_moves():
|
| 69 |
-
try:
|
| 70 |
-
tokens.append(_worker_tokenizer.encode(move.uci()))
|
| 71 |
-
except ValueError:
|
| 72 |
-
return None
|
| 73 |
-
|
| 74 |
-
if len(tokens) < 10:
|
| 75 |
-
return None
|
| 76 |
-
|
| 77 |
-
return np.array(tokens, dtype=np.int16), white_elo, black_elo, result, len(tokens)
|
| 78 |
-
|
| 79 |
-
|
| 80 |
-
def _iter_raw_games(stream):
|
| 81 |
-
"""Yield individual PGN game strings from a text stream.
|
| 82 |
-
|
| 83 |
-
Splits on '[Event' lines, which always start each game in standard PGN.
|
| 84 |
-
Line-by-line reading keeps memory usage constant regardless of file size.
|
| 85 |
-
"""
|
| 86 |
-
buf = []
|
| 87 |
-
for line in stream:
|
| 88 |
-
if line.startswith("[Event ") and buf:
|
| 89 |
-
yield "".join(buf)
|
| 90 |
-
buf = [line]
|
| 91 |
-
else:
|
| 92 |
-
buf.append(line)
|
| 93 |
-
if buf:
|
| 94 |
-
text = "".join(buf).strip()
|
| 95 |
-
if text:
|
| 96 |
-
yield text
|
| 97 |
-
|
| 98 |
-
|
| 99 |
-
class PGNtoHDF5Converter:
|
| 100 |
-
def __init__(
|
| 101 |
-
self,
|
| 102 |
-
output_path: str,
|
| 103 |
-
chunk_size: int = 10000,
|
| 104 |
-
num_workers: int = None,
|
| 105 |
-
exclude_time_controls: list = None,
|
| 106 |
-
):
|
| 107 |
-
"""
|
| 108 |
-
Convert PGN files to HDF5 format storing complete games as UCI move sequences.
|
| 109 |
-
|
| 110 |
-
Args:
|
| 111 |
-
output_path: Path to save the HDF5 file
|
| 112 |
-
chunk_size: Number of games to accumulate before writing (default: 10000)
|
| 113 |
-
exclude_time_controls: List of time control categories to skip, matched against
|
| 114 |
-
the Lichess Event header (e.g. ["Bullet", "Blitz", "UltraBullet"])
|
| 115 |
-
"""
|
| 116 |
-
self.output_path = output_path
|
| 117 |
-
self.chunk_size = chunk_size
|
| 118 |
-
default_workers = (cpu_count() or 1) - 1
|
| 119 |
-
self.num_workers = max(1, num_workers if num_workers is not None else max(1, default_workers))
|
| 120 |
-
self.exclude_time_controls = [tc.lower() for tc in (exclude_time_controls or [])]
|
| 121 |
-
|
| 122 |
-
def convert(self, input_path: str, max_games: int = None):
|
| 123 |
-
"""Convert PGN files to HDF5. Input can be a file or folder."""
|
| 124 |
-
path = Path(input_path)
|
| 125 |
-
if not path.exists():
|
| 126 |
-
raise FileNotFoundError(f"Path not found: {input_path}")
|
| 127 |
-
if path.is_file():
|
| 128 |
-
self._convert_single_file(path, max_games)
|
| 129 |
-
elif path.is_dir():
|
| 130 |
-
self._convert_folder(path, max_games)
|
| 131 |
-
else:
|
| 132 |
-
raise ValueError(f"Invalid input path: {input_path}")
|
| 133 |
-
|
| 134 |
-
def _convert_folder(self, folder_path: Path, max_games: int = None):
|
| 135 |
-
pgn_files = sorted(list(folder_path.glob("*.pgn")) + list(folder_path.glob("*.pgn.zst")))
|
| 136 |
-
if not pgn_files:
|
| 137 |
-
raise ValueError(f"No PGN files found in {folder_path}")
|
| 138 |
-
|
| 139 |
-
done = self._completed_files()
|
| 140 |
-
pending = [f for f in pgn_files if f.name not in done]
|
| 141 |
-
|
| 142 |
-
print(f"Found {len(pgn_files)} PGN file(s) in {folder_path}")
|
| 143 |
-
if done:
|
| 144 |
-
print(f"Resuming: {len(done)} already converted, {len(pending)} remaining")
|
| 145 |
-
for f in pending:
|
| 146 |
-
print(f" - {f.name}")
|
| 147 |
-
print()
|
| 148 |
-
|
| 149 |
-
if not pending:
|
| 150 |
-
print("Nothing to do — every file is already in the output.")
|
| 151 |
-
return 0
|
| 152 |
-
|
| 153 |
-
total_games = 0
|
| 154 |
-
started = bool(done)
|
| 155 |
-
for pgn_file in pending:
|
| 156 |
-
remaining = None if max_games is None else max(0, max_games - total_games)
|
| 157 |
-
if remaining == 0:
|
| 158 |
-
break
|
| 159 |
-
print(f"\nProcessing: {pgn_file.name}")
|
| 160 |
-
total_games += self._convert_single_file(pgn_file, remaining, append=started)
|
| 161 |
-
started = True
|
| 162 |
-
if max_games and total_games >= max_games:
|
| 163 |
-
print(f"\nReached maximum game limit ({max_games})")
|
| 164 |
-
break
|
| 165 |
-
|
| 166 |
-
print(f"\n{'=' * 60}")
|
| 167 |
-
print(f"All files processed! Total games: {total_games}")
|
| 168 |
-
print(f"Output: {self.output_path}")
|
| 169 |
-
|
| 170 |
-
def _convert_single_file(self, pgn_path: Path, max_games: int = None, append: bool = False):
|
| 171 |
-
if pgn_path.suffix == ".zst" or pgn_path.name.endswith(".pgn.zst"):
|
| 172 |
-
return self._convert_compressed_file(str(pgn_path), max_games, append)
|
| 173 |
-
else:
|
| 174 |
-
return self._convert_uncompressed_file(str(pgn_path), max_games, append)
|
| 175 |
-
|
| 176 |
-
def convert_pgn_zst(self, pgn_path: str, max_games: int = None):
|
| 177 |
-
"""Deprecated: Use convert() instead."""
|
| 178 |
-
return self._convert_compressed_file(pgn_path, max_games, append=False)
|
| 179 |
-
|
| 180 |
-
# ------------------------------------------------------------------
|
| 181 |
-
# Internal helpers
|
| 182 |
-
# ------------------------------------------------------------------
|
| 183 |
-
|
| 184 |
-
_DATASET_NAMES = ("moves", "white_elo", "black_elo", "result", "num_moves")
|
| 185 |
-
|
| 186 |
-
def _completed_files(self) -> list[str]:
|
| 187 |
-
"""Names of PGN files already fully written into the output, for resume."""
|
| 188 |
-
path = Path(self.output_path)
|
| 189 |
-
if not path.exists():
|
| 190 |
-
return []
|
| 191 |
-
try:
|
| 192 |
-
with h5py.File(path, "r") as f:
|
| 193 |
-
return [str(n) for n in f.attrs.get("converted_files", [])]
|
| 194 |
-
except OSError:
|
| 195 |
-
return []
|
| 196 |
-
|
| 197 |
-
def _mark_converted(self, hdf5_file, name: str):
|
| 198 |
-
"""Commit a finished PGN file: record its name and the row watermark."""
|
| 199 |
-
done = [str(n) for n in hdf5_file.attrs.get("converted_files", [])]
|
| 200 |
-
if name not in done:
|
| 201 |
-
done.append(name)
|
| 202 |
-
hdf5_file.attrs["converted_files"] = np.array(done, dtype=h5py.string_dtype())
|
| 203 |
-
hdf5_file.attrs["committed_rows"] = int(min(hdf5_file[n].shape[0] for n in self._DATASET_NAMES))
|
| 204 |
-
|
| 205 |
-
def _rewind_to_watermark(self, hdf5_file):
|
| 206 |
-
"""Discard everything written after the last fully converted PGN file.
|
| 207 |
-
|
| 208 |
-
Length agreement is not enough to detect an interrupted run: `_write_batch`
|
| 209 |
-
resizes all five datasets before writing any of them, so a crash mid-batch
|
| 210 |
-
leaves them equal-length with a junk tail. Worse, the rows a dead run *did*
|
| 211 |
-
write would be duplicated when its PGN file — never added to `converted_files`
|
| 212 |
-
— gets reconverted. Rewinding to the committed watermark drops both.
|
| 213 |
-
"""
|
| 214 |
-
lengths = {n: hdf5_file[n].shape[0] for n in self._DATASET_NAMES}
|
| 215 |
-
mark = int(hdf5_file.attrs.get("committed_rows", -1))
|
| 216 |
-
if mark < 0: # written before watermarks existed; fall back to length agreement
|
| 217 |
-
mark = min(lengths.values())
|
| 218 |
-
if any(length != mark for length in lengths.values()):
|
| 219 |
-
print(f"Rewinding interrupted output: {lengths} -> {mark} rows")
|
| 220 |
-
for name in self._DATASET_NAMES:
|
| 221 |
-
hdf5_file[name].resize((mark,))
|
| 222 |
-
|
| 223 |
-
def _create_datasets(self, hdf5_file):
|
| 224 |
-
"""Create all HDF5 datasets (called only when not appending)."""
|
| 225 |
-
dt = h5py.vlen_dtype(np.dtype("int16"))
|
| 226 |
-
for name, dtype in [
|
| 227 |
-
("moves", dt),
|
| 228 |
-
("white_elo", "int16"),
|
| 229 |
-
("black_elo", "int16"),
|
| 230 |
-
("result", "int8"),
|
| 231 |
-
("num_moves", "int16"),
|
| 232 |
-
]:
|
| 233 |
-
hdf5_file.create_dataset(
|
| 234 |
-
name,
|
| 235 |
-
shape=(0,),
|
| 236 |
-
maxshape=(None,),
|
| 237 |
-
dtype=dtype,
|
| 238 |
-
compression="gzip",
|
| 239 |
-
compression_opts=4,
|
| 240 |
-
chunks=(self.chunk_size,),
|
| 241 |
-
)
|
| 242 |
-
|
| 243 |
-
def _write_batch(self, hdf5_file, games, white_elos, black_elos, results, num_moves):
|
| 244 |
-
old = hdf5_file["moves"].shape[0]
|
| 245 |
-
new = old + len(games)
|
| 246 |
-
for ds in ("moves", "white_elo", "black_elo", "result", "num_moves"):
|
| 247 |
-
hdf5_file[ds].resize((new,))
|
| 248 |
-
# One slice assignment, not one write per game: each per-game write re-read and
|
| 249 |
-
# re-compressed the whole gzip chunk, costing ~1 ms/game in the main process and
|
| 250 |
-
# starving the workers.
|
| 251 |
-
moves_arr = np.empty(len(games), dtype=object)
|
| 252 |
-
moves_arr[:] = games
|
| 253 |
-
try:
|
| 254 |
-
hdf5_file["moves"][old:new] = moves_arr
|
| 255 |
-
except TypeError:
|
| 256 |
-
# h5py re-normalises the object array internally, and when every game in the
|
| 257 |
-
# batch happens to have the same length (always so for a single game) that
|
| 258 |
-
# collapses it to 2-D and the vlen write is refused. Only degenerate batches
|
| 259 |
-
# hit this — typically the 1-game remainder of a file's final flush — so the
|
| 260 |
-
# slow path costs nothing in practice.
|
| 261 |
-
for i, g in enumerate(games):
|
| 262 |
-
hdf5_file["moves"][old + i] = g
|
| 263 |
-
hdf5_file["white_elo"][old:new] = np.array(white_elos, dtype=np.int16)
|
| 264 |
-
hdf5_file["black_elo"][old:new] = np.array(black_elos, dtype=np.int16)
|
| 265 |
-
hdf5_file["result"][old:new] = np.array(results, dtype=np.int8)
|
| 266 |
-
hdf5_file["num_moves"][old:new] = np.array(num_moves, dtype=np.int16)
|
| 267 |
-
|
| 268 |
-
def _run_conversion(self, stream, hdf5_file, max_games) -> tuple[int, int]:
|
| 269 |
-
"""Core loop: stream raw PGN text → workers parse+tokenize → write HDF5.
|
| 270 |
-
|
| 271 |
-
Returns (game_count, total_positions).
|
| 272 |
-
"""
|
| 273 |
-
game_count = 0
|
| 274 |
-
total_positions = 0
|
| 275 |
-
games_buf, we_buf, be_buf, res_buf, nm_buf = [], [], [], [], []
|
| 276 |
-
|
| 277 |
-
pbar = tqdm(desc="Processing games", unit="games")
|
| 278 |
-
|
| 279 |
-
def flush():
|
| 280 |
-
if games_buf:
|
| 281 |
-
self._write_batch(hdf5_file, games_buf, we_buf, be_buf, res_buf, nm_buf)
|
| 282 |
-
games_buf.clear(); we_buf.clear(); be_buf.clear()
|
| 283 |
-
res_buf.clear(); nm_buf.clear()
|
| 284 |
-
|
| 285 |
-
def accumulate(result):
|
| 286 |
-
nonlocal game_count, total_positions
|
| 287 |
-
token_array, we, be, res, nm = result
|
| 288 |
-
games_buf.append(token_array)
|
| 289 |
-
we_buf.append(we); be_buf.append(be)
|
| 290 |
-
res_buf.append(res); nm_buf.append(nm)
|
| 291 |
-
total_positions += nm
|
| 292 |
-
game_count += 1
|
| 293 |
-
pbar.update(1)
|
| 294 |
-
if len(games_buf) >= self.chunk_size:
|
| 295 |
-
flush()
|
| 296 |
-
|
| 297 |
-
use_parallel = self.num_workers > 1
|
| 298 |
-
raw_iter = _iter_raw_games(stream)
|
| 299 |
-
|
| 300 |
-
if use_parallel:
|
| 301 |
-
pool = Pool(
|
| 302 |
-
processes=self.num_workers,
|
| 303 |
-
initializer=_worker_init,
|
| 304 |
-
initargs=(self.exclude_time_controls,),
|
| 305 |
-
)
|
| 306 |
-
early_stop = False
|
| 307 |
-
try:
|
| 308 |
-
# imap_unordered streams results as soon as each worker finishes —
|
| 309 |
-
# no need to wait for a full chunk_size batch before seeing progress.
|
| 310 |
-
for result in pool.imap_unordered(_process_raw_game, raw_iter, chunksize=200):
|
| 311 |
-
if result is None:
|
| 312 |
-
continue
|
| 313 |
-
accumulate(result)
|
| 314 |
-
if max_games and game_count >= max_games:
|
| 315 |
-
early_stop = True
|
| 316 |
-
break
|
| 317 |
-
finally:
|
| 318 |
-
if early_stop:
|
| 319 |
-
pool.terminate()
|
| 320 |
-
else:
|
| 321 |
-
pool.close()
|
| 322 |
-
pool.join()
|
| 323 |
-
else:
|
| 324 |
-
# Sequential fallback (single worker)
|
| 325 |
-
_worker_init(self.exclude_time_controls)
|
| 326 |
-
for raw_pgn in raw_iter:
|
| 327 |
-
result = _process_raw_game(raw_pgn)
|
| 328 |
-
if result is None:
|
| 329 |
-
continue
|
| 330 |
-
accumulate(result)
|
| 331 |
-
if max_games and game_count >= max_games:
|
| 332 |
-
break
|
| 333 |
-
|
| 334 |
-
flush()
|
| 335 |
-
pbar.close()
|
| 336 |
-
return game_count, total_positions
|
| 337 |
-
|
| 338 |
-
def _convert_compressed_file(self, pgn_path: str, max_games: int = None, append: bool = False):
|
| 339 |
-
with open(pgn_path, "rb") as f:
|
| 340 |
-
dctx = zstd.ZstdDecompressor()
|
| 341 |
-
with dctx.stream_reader(f) as reader:
|
| 342 |
-
stream = io.TextIOWrapper(reader, encoding="utf-8")
|
| 343 |
-
mode = "a" if append else "w"
|
| 344 |
-
with h5py.File(self.output_path, mode) as hdf5_file:
|
| 345 |
-
if append and "moves" in hdf5_file:
|
| 346 |
-
self._rewind_to_watermark(hdf5_file)
|
| 347 |
-
else:
|
| 348 |
-
self._create_datasets(hdf5_file)
|
| 349 |
-
game_count, total_positions = self._run_conversion(stream, hdf5_file, max_games)
|
| 350 |
-
if not max_games or game_count < max_games:
|
| 351 |
-
self._mark_converted(hdf5_file, Path(pgn_path).name)
|
| 352 |
-
|
| 353 |
-
self._print_summary(game_count, total_positions)
|
| 354 |
-
return game_count
|
| 355 |
-
|
| 356 |
-
def _convert_uncompressed_file(self, pgn_path: str, max_games: int = None, append: bool = False):
|
| 357 |
-
with open(pgn_path, "r", encoding="utf-8") as pgn_file:
|
| 358 |
-
mode = "a" if append else "w"
|
| 359 |
-
with h5py.File(self.output_path, mode) as hdf5_file:
|
| 360 |
-
if append and "moves" in hdf5_file:
|
| 361 |
-
self._rewind_to_watermark(hdf5_file)
|
| 362 |
-
else:
|
| 363 |
-
self._create_datasets(hdf5_file)
|
| 364 |
-
game_count, total_positions = self._run_conversion(pgn_file, hdf5_file, max_games)
|
| 365 |
-
if not max_games or game_count < max_games:
|
| 366 |
-
self._mark_converted(hdf5_file, Path(pgn_path).name)
|
| 367 |
-
|
| 368 |
-
self._print_summary(game_count, total_positions)
|
| 369 |
-
return game_count
|
| 370 |
-
|
| 371 |
-
def _print_summary(self, game_count, total_positions):
|
| 372 |
-
print("File conversion complete!")
|
| 373 |
-
print(f"Games from this file: {game_count}")
|
| 374 |
-
print(f"Positions from this file: {total_positions}")
|
| 375 |
-
avg = (total_positions / game_count) if game_count else 0.0
|
| 376 |
-
print(f"Average moves per game: {avg:.1f}")
|
| 377 |
-
|
| 378 |
-
|
| 379 |
-
# Usage script
|
| 380 |
-
if __name__ == "__main__":
|
| 381 |
-
import argparse
|
| 382 |
-
|
| 383 |
-
parser = argparse.ArgumentParser(description="Convert PGN/PGN.zst file(s) or folder to HDF5")
|
| 384 |
-
parser.add_argument("--input", type=str, required=True, help="Input .pgn/.pgn.zst file or folder")
|
| 385 |
-
parser.add_argument("--output", type=str, required=True, help="Output .h5 file")
|
| 386 |
-
parser.add_argument("--max-games", type=int, default=None, help="Maximum games to process")
|
| 387 |
-
parser.add_argument("--chunk-size", type=int, default=10000, help="Batch size for writing")
|
| 388 |
-
parser.add_argument("--num-workers", type=int, default=None, help="Worker processes (default: cpu count - 1)")
|
| 389 |
-
parser.add_argument(
|
| 390 |
-
"--exclude-time-controls",
|
| 391 |
-
nargs="+",
|
| 392 |
-
default=[],
|
| 393 |
-
metavar="TC",
|
| 394 |
-
help="Time control categories to skip (e.g. Bullet Blitz UltraBullet)",
|
| 395 |
-
)
|
| 396 |
-
|
| 397 |
-
args = parser.parse_args()
|
| 398 |
-
converter = PGNtoHDF5Converter(
|
| 399 |
-
args.output,
|
| 400 |
-
chunk_size=args.chunk_size,
|
| 401 |
-
num_workers=args.num_workers,
|
| 402 |
-
exclude_time_controls=args.exclude_time_controls,
|
| 403 |
-
)
|
| 404 |
-
converter.convert(args.input, max_games=args.max_games)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
chesstransformer/datasets/flat_shard_dataset.py
DELETED
|
@@ -1,231 +0,0 @@
|
|
| 1 |
-
"""Memmap-backed dataset over the flat shards written by ``scripts/build_shards.py``.
|
| 2 |
-
|
| 3 |
-
Implements ``doc/dataset_shards.md`` §4. Every expensive thing ``HDF5ChessDataset`` does
|
| 4 |
-
per sample — gzip decompression, python-chess replay, legal-move enumeration, tokenizing —
|
| 5 |
-
has already been paid at build time. What is left here is a numpy gather out of page cache.
|
| 6 |
-
|
| 7 |
-
Two decisions carry most of the speed:
|
| 8 |
-
|
| 9 |
-
**Batch-level fetch.** At 202 bytes per sample the per-item Python call overhead dominates
|
| 10 |
-
the actual work, so ``__getitem__`` accepts a *list* of indices and does one vectorised
|
| 11 |
-
gather per batch. Drive it with :func:`make_dataloader`, which wires up a ``BatchSampler``
|
| 12 |
-
and ``batch_size=None`` so the DataLoader hands the whole index list through untouched.
|
| 13 |
-
|
| 14 |
-
**The legal mask is not built on CPU.** Returning ``(64, 73)`` bools would put 2.4 MB per
|
| 15 |
-
batch of 512 back on the H2D wire — the exact cost the shard format exists to remove. The
|
| 16 |
-
loader ships the ``(B, 64)`` uint16 index list (64 KB) and the training loop calls
|
| 17 |
-
:func:`scatter_legal_planes` on device.
|
| 18 |
-
|
| 19 |
-
**Distributed use.** This is a map-style dataset, so rank splitting is
|
| 20 |
-
``DistributedSampler``'s job, not the dataset's — see :func:`make_dataloader`. Map-style +
|
| 21 |
-
``DistributedSampler`` is preferred over an ``IterableDataset`` that owns shard *k* when
|
| 22 |
-
``k % (world_size * num_workers) == rank * num_workers + worker`` because the latter can
|
| 23 |
-
only shuffle within a worker's own shards: sample correlation then survives into the batch,
|
| 24 |
-
and the assignment silently unbalances whenever ``world_size * num_workers`` does not
|
| 25 |
-
divide the shard count. The shards here are a build-parallelism and train/val/test-split
|
| 26 |
-
unit; they deliberately do not constrain who reads what.
|
| 27 |
-
"""
|
| 28 |
-
|
| 29 |
-
import json
|
| 30 |
-
import os
|
| 31 |
-
from pathlib import Path
|
| 32 |
-
|
| 33 |
-
import numpy as np
|
| 34 |
-
import torch
|
| 35 |
-
from torch.utils.data import BatchSampler, DataLoader, RandomSampler, SequentialSampler
|
| 36 |
-
|
| 37 |
-
from chesstransformer.datasets.shard_format import (
|
| 38 |
-
FORMAT_VERSION,
|
| 39 |
-
NUM_ACTION_PLANES,
|
| 40 |
-
NUM_SQUARES,
|
| 41 |
-
PAD_ACTION,
|
| 42 |
-
dtype_from_meta,
|
| 43 |
-
)
|
| 44 |
-
|
| 45 |
-
#: Fields handed to the trainer, in the dtype the model wants them in. Kept narrow on the
|
| 46 |
-
#: wire (uint8/int16) and widened on device — an int64 ``position`` would be 8x the copy
|
| 47 |
-
#: for an embedding lookup that accepts int32 anyway.
|
| 48 |
-
_SCALAR_FIELDS = ("from_square", "action_plane", "castling_rights", "en_passant_file", "is_white")
|
| 49 |
-
|
| 50 |
-
|
| 51 |
-
class FlatShardDataset(torch.utils.data.Dataset):
|
| 52 |
-
"""One split of a shard directory, as one addressable array of samples."""
|
| 53 |
-
|
| 54 |
-
def __init__(self, shard_dir: str | Path, split: str = "train"):
|
| 55 |
-
self.shard_dir = Path(shard_dir)
|
| 56 |
-
self.split = split
|
| 57 |
-
|
| 58 |
-
with (self.shard_dir / "meta.json").open() as fh:
|
| 59 |
-
self.meta = json.load(fh)
|
| 60 |
-
|
| 61 |
-
if self.meta["format_version"] != FORMAT_VERSION:
|
| 62 |
-
raise ValueError(
|
| 63 |
-
f"{self.shard_dir} was built with format_version "
|
| 64 |
-
f"{self.meta['format_version']}, this code reads {FORMAT_VERSION}. Rebuild."
|
| 65 |
-
)
|
| 66 |
-
|
| 67 |
-
if split not in self.meta["splits"]:
|
| 68 |
-
raise KeyError(f"unknown split {split!r}; have {list(self.meta['splits'])}")
|
| 69 |
-
|
| 70 |
-
self.dtype = dtype_from_meta(self.meta["record_dtype"])
|
| 71 |
-
self.pad_action = int(self.meta["pad_action"])
|
| 72 |
-
|
| 73 |
-
wanted = set(self.meta["splits"][split])
|
| 74 |
-
self.shard_meta = [m for m in self.meta["shards"] if m["shard_id"] in wanted]
|
| 75 |
-
if not self.shard_meta:
|
| 76 |
-
raise ValueError(f"split {split!r} of {self.shard_dir} is empty")
|
| 77 |
-
|
| 78 |
-
# Cumulative offsets: a global index resolves to (shard, row) with one searchsorted.
|
| 79 |
-
counts = np.array([m["num_samples"] for m in self.shard_meta], dtype=np.int64)
|
| 80 |
-
self.offsets = np.concatenate([[0], np.cumsum(counts)])
|
| 81 |
-
self._len = int(self.offsets[-1])
|
| 82 |
-
|
| 83 |
-
# Opened lazily, per process. np.memmap survives fork harmlessly, but building the
|
| 84 |
-
# handles in __init__ would map every shard in the parent for no reason.
|
| 85 |
-
self._shards = None
|
| 86 |
-
self._shards_pid = None
|
| 87 |
-
|
| 88 |
-
# The whole point of memmap is that the page cache holds one copy that all workers
|
| 89 |
-
# share. np.load(...) without mmap_mode would give each of N workers its own.
|
| 90 |
-
self.game_ranges = [(m["game_start"], m["game_end"]) for m in self.shard_meta]
|
| 91 |
-
|
| 92 |
-
def __len__(self):
|
| 93 |
-
return self._len
|
| 94 |
-
|
| 95 |
-
@property
|
| 96 |
-
def shards(self):
|
| 97 |
-
pid = os.getpid()
|
| 98 |
-
if self._shards is None or self._shards_pid != pid:
|
| 99 |
-
self._shards = [
|
| 100 |
-
np.load(self.shard_dir / m["path"], mmap_mode="r") for m in self.shard_meta
|
| 101 |
-
]
|
| 102 |
-
self._shards_pid = pid
|
| 103 |
-
return self._shards
|
| 104 |
-
|
| 105 |
-
def __getstate__(self):
|
| 106 |
-
state = self.__dict__.copy()
|
| 107 |
-
state["_shards"] = None
|
| 108 |
-
state["_shards_pid"] = None
|
| 109 |
-
return state
|
| 110 |
-
|
| 111 |
-
def _gather(self, idx: np.ndarray) -> np.ndarray:
|
| 112 |
-
"""Fetch records for a batch of global indices, preserving their order."""
|
| 113 |
-
shard_of = np.searchsorted(self.offsets, idx, side="right") - 1
|
| 114 |
-
rows = idx - self.offsets[shard_of]
|
| 115 |
-
|
| 116 |
-
shards = self.shards
|
| 117 |
-
out = np.empty(len(idx), dtype=self.dtype)
|
| 118 |
-
for sid in np.unique(shard_of):
|
| 119 |
-
where = np.nonzero(shard_of == sid)[0]
|
| 120 |
-
r = rows[where]
|
| 121 |
-
# Read each shard in ascending row order: sequential page touches, even though
|
| 122 |
-
# the batch itself is shuffled.
|
| 123 |
-
order = np.argsort(r, kind="stable")
|
| 124 |
-
out[where[order]] = shards[sid][r[order]]
|
| 125 |
-
return out
|
| 126 |
-
|
| 127 |
-
def __getitem__(self, index):
|
| 128 |
-
"""Accepts a single index or — the fast path — a list/array of them."""
|
| 129 |
-
if isinstance(index, (int, np.integer)):
|
| 130 |
-
batch = self._gather(np.array([index], dtype=np.int64))
|
| 131 |
-
else:
|
| 132 |
-
batch = self._gather(np.asarray(index, dtype=np.int64))
|
| 133 |
-
|
| 134 |
-
# Everything stays at its on-disk width. A batch of 512 is then 103 KB rather than
|
| 135 |
-
# the 5.81 MB the HDF5 path collates, and the widening to int64/bool happens on
|
| 136 |
-
# device in the trainer's unpack_batch, where it is free.
|
| 137 |
-
#
|
| 138 |
-
# int16 rather than uint16 for the two u2 fields because torch has no uint16; both
|
| 139 |
-
# are safely in range (legal_idx <= 4672, move_number max observed 601).
|
| 140 |
-
out = {
|
| 141 |
-
"position": torch.from_numpy(np.ascontiguousarray(batch["position"])),
|
| 142 |
-
"legal_idx": torch.from_numpy(np.ascontiguousarray(batch["legal_idx"]).astype(np.int16)),
|
| 143 |
-
"legal_cnt": torch.from_numpy(np.ascontiguousarray(batch["legal_cnt"])),
|
| 144 |
-
"result": torch.from_numpy(np.ascontiguousarray(batch["result"])),
|
| 145 |
-
"move_number": torch.from_numpy(
|
| 146 |
-
np.ascontiguousarray(batch["move_number"]).astype(np.int16)
|
| 147 |
-
),
|
| 148 |
-
}
|
| 149 |
-
for name in _SCALAR_FIELDS:
|
| 150 |
-
out[name] = torch.from_numpy(np.ascontiguousarray(batch[name]))
|
| 151 |
-
return out
|
| 152 |
-
|
| 153 |
-
def game_id_set(self) -> set:
|
| 154 |
-
"""Game indices this split covers. Used by the split-disjointness assertion."""
|
| 155 |
-
ids = set()
|
| 156 |
-
for g0, g1 in self.game_ranges:
|
| 157 |
-
ids.update(range(g0, g1))
|
| 158 |
-
return ids
|
| 159 |
-
|
| 160 |
-
|
| 161 |
-
def scatter_legal_planes(legal_idx: torch.Tensor, pad_action: int = PAD_ACTION) -> torch.Tensor:
|
| 162 |
-
"""Rebuild the ``(B, 64, 73)`` bool legal mask from the packed index list, on device.
|
| 163 |
-
|
| 164 |
-
``legal_idx`` is ``(B, C)`` with unused slots set to ``pad_action`` (== 64*73), one past
|
| 165 |
-
the end of the action space. Scattering into a tensor with one extra column parks that
|
| 166 |
-
padding in a scratch slot which is then sliced off — if padding were 0 instead, every
|
| 167 |
-
sample would come back with square a1 / plane 0 marked legal.
|
| 168 |
-
"""
|
| 169 |
-
b = legal_idx.size(0)
|
| 170 |
-
flat = torch.zeros(b, pad_action + 1, dtype=torch.bool, device=legal_idx.device)
|
| 171 |
-
flat.scatter_(1, legal_idx.long(), True)
|
| 172 |
-
return flat[:, :pad_action].view(b, NUM_SQUARES, NUM_ACTION_PLANES)
|
| 173 |
-
|
| 174 |
-
|
| 175 |
-
def worker_init_fn(worker_id: int):
|
| 176 |
-
"""Give every worker (and every rank) its own numpy RNG stream.
|
| 177 |
-
|
| 178 |
-
PyTorch reseeds ``random`` and ``torch`` per worker but *not* ``numpy``, so all workers
|
| 179 |
-
inherit the parent's numpy state and draw the identical sequence — and under DDP so
|
| 180 |
-
would every rank. Nothing in this dataset's hot path uses ``np.random`` any more, but
|
| 181 |
-
``HDF5ChessDataset._sample_move_idx`` does, and the fix belongs wherever a worker starts.
|
| 182 |
-
"""
|
| 183 |
-
rank = int(os.environ.get("RANK", 0))
|
| 184 |
-
np.random.seed((torch.initial_seed() + 977 * rank) % (2**32))
|
| 185 |
-
|
| 186 |
-
|
| 187 |
-
def make_dataloader(
|
| 188 |
-
dataset: FlatShardDataset,
|
| 189 |
-
batch_size: int,
|
| 190 |
-
shuffle: bool,
|
| 191 |
-
num_workers: int = 4,
|
| 192 |
-
drop_last: bool = False,
|
| 193 |
-
pin_memory: bool = True,
|
| 194 |
-
prefetch_factor: int = 4,
|
| 195 |
-
seed: int = 0,
|
| 196 |
-
distributed: bool = False,
|
| 197 |
-
) -> DataLoader:
|
| 198 |
-
"""Build the batch-fetch DataLoader this dataset is designed for.
|
| 199 |
-
|
| 200 |
-
``batch_size=None`` disables the DataLoader's own batching so the ``BatchSampler``'s
|
| 201 |
-
index *list* reaches ``__getitem__`` intact and the gather happens once per batch
|
| 202 |
-
rather than once per sample. Without this the dataset still works and runs roughly an
|
| 203 |
-
order of magnitude slower.
|
| 204 |
-
|
| 205 |
-
``distributed=True`` wraps a ``DistributedSampler`` so each rank sees a disjoint
|
| 206 |
-
1/world_size of the split. Remember to call ``loader.sampler.sampler.set_epoch(epoch)``
|
| 207 |
-
— the BatchSampler nests it one level down.
|
| 208 |
-
"""
|
| 209 |
-
if distributed:
|
| 210 |
-
from torch.utils.data import DistributedSampler
|
| 211 |
-
|
| 212 |
-
inner = DistributedSampler(dataset, shuffle=shuffle, seed=seed, drop_last=drop_last)
|
| 213 |
-
elif shuffle:
|
| 214 |
-
generator = torch.Generator()
|
| 215 |
-
generator.manual_seed(seed)
|
| 216 |
-
inner = RandomSampler(dataset, generator=generator)
|
| 217 |
-
else:
|
| 218 |
-
inner = SequentialSampler(dataset)
|
| 219 |
-
|
| 220 |
-
sampler = BatchSampler(inner, batch_size=batch_size, drop_last=drop_last)
|
| 221 |
-
|
| 222 |
-
return DataLoader(
|
| 223 |
-
dataset,
|
| 224 |
-
batch_size=None,
|
| 225 |
-
sampler=sampler,
|
| 226 |
-
num_workers=num_workers,
|
| 227 |
-
pin_memory=pin_memory,
|
| 228 |
-
persistent_workers=num_workers > 0,
|
| 229 |
-
prefetch_factor=prefetch_factor if num_workers > 0 else None,
|
| 230 |
-
worker_init_fn=worker_init_fn,
|
| 231 |
-
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
chesstransformer/datasets/h5_lichess_dataset.py
DELETED
|
@@ -1,295 +0,0 @@
|
|
| 1 |
-
import os
|
| 2 |
-
|
| 3 |
-
import h5py
|
| 4 |
-
import torch
|
| 5 |
-
from torch.utils.data import Dataset
|
| 6 |
-
import numpy as np
|
| 7 |
-
import chess
|
| 8 |
-
from chesstransformer.models.tokenizer.position_tokenizer import PostionTokenizer
|
| 9 |
-
from chesstransformer.models.tokenizer.move_tokenizer import MoveTokenizer
|
| 10 |
-
from chesstransformer.models.tokenizer.alphazero_move_encoder import (
|
| 11 |
-
move_to_action_plane,
|
| 12 |
-
NUM_ACTION_PLANES,
|
| 13 |
-
)
|
| 14 |
-
|
| 15 |
-
|
| 16 |
-
class HDF5ChessDataset(Dataset):
|
| 17 |
-
def __init__(
|
| 18 |
-
self,
|
| 19 |
-
hdf5_path: str,
|
| 20 |
-
cache_size: int = 500,
|
| 21 |
-
min_elo: int = None,
|
| 22 |
-
max_elo: int = None,
|
| 23 |
-
sample_weighting: str = "middlegame",
|
| 24 |
-
skip_opening_plies: int = 0,
|
| 25 |
-
rdcc_nbytes: int = 32 << 20,
|
| 26 |
-
):
|
| 27 |
-
"""
|
| 28 |
-
Efficient dataset that loads games as UCI sequences and samples random positions.
|
| 29 |
-
|
| 30 |
-
Each dataset index corresponds to one game. When a sample is requested, it:
|
| 31 |
-
1. Uses the index to select a game
|
| 32 |
-
2. Samples a position within that game (uniform or phase-weighted)
|
| 33 |
-
3. Replays the game to that position to get the board state
|
| 34 |
-
4. Returns the encoded board state + the next move
|
| 35 |
-
|
| 36 |
-
Args:
|
| 37 |
-
hdf5_path: Path to HDF5 file containing games
|
| 38 |
-
cache_size: Number of games to cache in memory
|
| 39 |
-
min_elo: Minimum average ELO for filtering games (optional)
|
| 40 |
-
max_elo: Maximum average ELO for filtering games (optional)
|
| 41 |
-
sample_weighting: "uniform" (legacy) or "middlegame" (triangular peak around move 25)
|
| 42 |
-
skip_opening_plies: Force sampling to start at this ply (0 = no restriction)
|
| 43 |
-
rdcc_nbytes: HDF5 chunk cache per dataset. ``moves`` is gzipped in chunks of
|
| 44 |
-
10,000 games, so a random read decompresses a whole chunk; h5py's 1 MB
|
| 45 |
-
default cannot hold one and every read pays full decompression. 32 MB
|
| 46 |
-
takes a random ``moves[g]`` from 0.22 ms to 0.12 ms.
|
| 47 |
-
"""
|
| 48 |
-
self.hdf5_path = hdf5_path
|
| 49 |
-
self.cache_size = cache_size
|
| 50 |
-
self.game_cache = {} # Cache for game move sequences
|
| 51 |
-
self.position_tokenizer = PostionTokenizer()
|
| 52 |
-
self.move_tokenizer = MoveTokenizer()
|
| 53 |
-
self.sample_weighting = sample_weighting
|
| 54 |
-
self.skip_opening_plies = skip_opening_plies
|
| 55 |
-
self.rdcc_nbytes = rdcc_nbytes
|
| 56 |
-
|
| 57 |
-
# Long-lived read handle, opened lazily by _h5(). See the note there for why
|
| 58 |
-
# this must never be opened in __init__.
|
| 59 |
-
self._file = None
|
| 60 |
-
self._file_pid = None
|
| 61 |
-
|
| 62 |
-
# Load metadata and filter games by ELO
|
| 63 |
-
with h5py.File(hdf5_path, "r") as f:
|
| 64 |
-
self.num_games = f["moves"].shape[0]
|
| 65 |
-
num_moves = f["num_moves"][:]
|
| 66 |
-
white_elos = f["white_elo"][:]
|
| 67 |
-
black_elos = f["black_elo"][:]
|
| 68 |
-
results = f["result"][:]
|
| 69 |
-
|
| 70 |
-
# Filter games by ELO if specified
|
| 71 |
-
valid_games = np.ones(self.num_games, dtype=bool)
|
| 72 |
-
if min_elo is not None or max_elo is not None:
|
| 73 |
-
avg_elos = (white_elos + black_elos) / 2
|
| 74 |
-
if min_elo is not None:
|
| 75 |
-
valid_games &= avg_elos >= min_elo
|
| 76 |
-
if max_elo is not None:
|
| 77 |
-
valid_games &= avg_elos <= max_elo
|
| 78 |
-
|
| 79 |
-
# Store valid game indices and their move counts
|
| 80 |
-
self.valid_game_indices = np.where(valid_games)[0]
|
| 81 |
-
self.num_moves_per_game = num_moves[valid_games]
|
| 82 |
-
|
| 83 |
-
# Per-game metadata stays in RAM: 1.53M games is 1.5 MB of int8 plus
|
| 84 |
-
# 3 MB of int16, and reading these three scalars back out of HDF5 per
|
| 85 |
-
# sample cost more than the game decode itself. Kept *unfiltered*, so
|
| 86 |
-
# they are indexed by the actual game index, not the dataset index.
|
| 87 |
-
self.white_elos = white_elos
|
| 88 |
-
self.black_elos = black_elos
|
| 89 |
-
self.results = results
|
| 90 |
-
|
| 91 |
-
total_positions = int(self.num_moves_per_game.sum())
|
| 92 |
-
print(f"Loaded dataset: {len(self.valid_game_indices)} games with {total_positions} total positions")
|
| 93 |
-
if min_elo or max_elo:
|
| 94 |
-
print(f"Filtered by ELO: {min_elo or 0} - {max_elo or 'inf'}")
|
| 95 |
-
|
| 96 |
-
def __len__(self):
|
| 97 |
-
"""Return the number of games in the dataset."""
|
| 98 |
-
return len(self.valid_game_indices)
|
| 99 |
-
|
| 100 |
-
def _sample_move_idx(self, max_move_idx: int) -> int:
|
| 101 |
-
"""Sample a position index within a game.
|
| 102 |
-
|
| 103 |
-
- "uniform": legacy behavior, np.random.randint(0, max_move_idx)
|
| 104 |
-
- "middlegame": triangular weight peaking around ply 25,
|
| 105 |
-
downweights opening (rare unique positions) and very late
|
| 106 |
-
endgame (sparse signal). Falls back to uniform for short games.
|
| 107 |
-
|
| 108 |
-
Optionally enforces ``skip_opening_plies`` as a hard floor.
|
| 109 |
-
"""
|
| 110 |
-
lo = min(self.skip_opening_plies, max_move_idx - 1) if max_move_idx > 1 else 0
|
| 111 |
-
hi = max_move_idx # exclusive upper bound
|
| 112 |
-
|
| 113 |
-
if self.sample_weighting == "uniform" or hi - lo <= 8:
|
| 114 |
-
return int(np.random.randint(lo, hi))
|
| 115 |
-
|
| 116 |
-
# Triangular weights: ramp up over plies 0->20, flat 20-40, taper 40->80
|
| 117 |
-
plies = np.arange(lo, hi)
|
| 118 |
-
ramp_up = np.minimum(plies / 20.0, 1.0)
|
| 119 |
-
taper = np.minimum(np.maximum((80.0 - plies) / 40.0, 0.1), 1.0)
|
| 120 |
-
weights = ramp_up * taper
|
| 121 |
-
weights = weights / weights.sum()
|
| 122 |
-
return int(np.random.choice(plies, p=weights))
|
| 123 |
-
|
| 124 |
-
def _h5(self) -> h5py.File:
|
| 125 |
-
"""Return this *process's* HDF5 handle, opening it on first use.
|
| 126 |
-
|
| 127 |
-
Deliberately not opened in ``__init__``. DataLoader workers are forked, and
|
| 128 |
-
HDF5 is not fork-safe: a handle inherited across ``fork()`` is shared library
|
| 129 |
-
state that two processes then mutate independently. It does not raise — it
|
| 130 |
-
returns corrupted data or deadlocks, which is far worse. Opening on first
|
| 131 |
-
``__getitem__`` means every worker gets its own handle after the fork.
|
| 132 |
-
|
| 133 |
-
The PID check covers the case where the parent process reads a sample (e.g.
|
| 134 |
-
``num_workers=0``, or a shape probe) before spawning workers, which would
|
| 135 |
-
otherwise leave an open handle for the children to inherit.
|
| 136 |
-
"""
|
| 137 |
-
pid = os.getpid()
|
| 138 |
-
if self._file is None or self._file_pid != pid:
|
| 139 |
-
self._file = h5py.File(
|
| 140 |
-
self.hdf5_path,
|
| 141 |
-
"r",
|
| 142 |
-
rdcc_nbytes=self.rdcc_nbytes,
|
| 143 |
-
rdcc_nslots=10007, # prime, comfortably > chunks held in the cache
|
| 144 |
-
)
|
| 145 |
-
self._file_pid = pid
|
| 146 |
-
return self._file
|
| 147 |
-
|
| 148 |
-
def __getstate__(self):
|
| 149 |
-
"""Drop the handle when pickled (``spawn`` start method); h5py.File is not
|
| 150 |
-
picklable and each process must open its own anyway."""
|
| 151 |
-
state = self.__dict__.copy()
|
| 152 |
-
state["_file"] = None
|
| 153 |
-
state["_file_pid"] = None
|
| 154 |
-
return state
|
| 155 |
-
|
| 156 |
-
def _get_game_moves(self, game_idx: int) -> np.ndarray:
|
| 157 |
-
"""Load and cache a game's move sequence.
|
| 158 |
-
|
| 159 |
-
Only ``moves`` comes from HDF5; the per-game scalars live in RAM (see __init__).
|
| 160 |
-
"""
|
| 161 |
-
if game_idx in self.game_cache:
|
| 162 |
-
return self.game_cache[game_idx]
|
| 163 |
-
|
| 164 |
-
moves = self._h5()["moves"][game_idx]
|
| 165 |
-
|
| 166 |
-
# Update cache (simple LRU-like behavior)
|
| 167 |
-
if len(self.game_cache) >= self.cache_size:
|
| 168 |
-
# Remove oldest item
|
| 169 |
-
self.game_cache.pop(next(iter(self.game_cache)))
|
| 170 |
-
|
| 171 |
-
self.game_cache[game_idx] = moves
|
| 172 |
-
return moves
|
| 173 |
-
|
| 174 |
-
def __getitem__(self, idx):
|
| 175 |
-
"""
|
| 176 |
-
Returns a position-move pair by:
|
| 177 |
-
1. Using idx to select a game
|
| 178 |
-
2. Uniformly sampling a random position within that game
|
| 179 |
-
3. Replaying the game to get the board state
|
| 180 |
-
4. Returning encoded board + next move
|
| 181 |
-
"""
|
| 182 |
-
# Map dataset index to actual game index
|
| 183 |
-
actual_game_idx = self.valid_game_indices[idx]
|
| 184 |
-
|
| 185 |
-
# Load game moves; the scalars are RAM lookups (indexed by actual game index)
|
| 186 |
-
game_moves = self._get_game_moves(actual_game_idx)
|
| 187 |
-
white_elo = int(self.white_elos[actual_game_idx])
|
| 188 |
-
black_elo = int(self.black_elos[actual_game_idx])
|
| 189 |
-
result = int(self.results[actual_game_idx])
|
| 190 |
-
num_moves = self.num_moves_per_game[idx]
|
| 191 |
-
|
| 192 |
-
# Uniformly sample a position within the game (exclude last move - no next move)
|
| 193 |
-
max_move_idx = num_moves - 1
|
| 194 |
-
if max_move_idx <= 0:
|
| 195 |
-
# Edge case: game with only one move, use position 0
|
| 196 |
-
move_idx = 0
|
| 197 |
-
else:
|
| 198 |
-
move_idx = self._sample_move_idx(int(max_move_idx))
|
| 199 |
-
|
| 200 |
-
# Replay game to position
|
| 201 |
-
board = chess.Board()
|
| 202 |
-
for i in range(move_idx):
|
| 203 |
-
# Decode move token to UCI string
|
| 204 |
-
move_token = int(game_moves[i])
|
| 205 |
-
move_uci = self._decode_move_token(move_token)
|
| 206 |
-
try:
|
| 207 |
-
move = chess.Move.from_uci(move_uci)
|
| 208 |
-
board.push(move)
|
| 209 |
-
except (ValueError, AssertionError):
|
| 210 |
-
# If move is invalid, skip to next position
|
| 211 |
-
# This shouldn't happen with properly encoded data
|
| 212 |
-
pass
|
| 213 |
-
|
| 214 |
-
# Encode current position
|
| 215 |
-
position = self.position_tokenizer.encode(board)
|
| 216 |
-
position_tensor = torch.tensor(position, dtype=torch.long)
|
| 217 |
-
|
| 218 |
-
legal_moves_planes = torch.zeros(64, NUM_ACTION_PLANES, dtype=torch.bool)
|
| 219 |
-
for move in board.legal_moves:
|
| 220 |
-
plane = move_to_action_plane(move.from_square, move.to_square, move.promotion)
|
| 221 |
-
legal_moves_planes[move.from_square, plane] = True
|
| 222 |
-
|
| 223 |
-
# Get next move
|
| 224 |
-
next_move_token = torch.tensor(int(game_moves[move_idx]), dtype=torch.long)
|
| 225 |
-
|
| 226 |
-
# Parse next move for grid format (from/to squares + promotion)
|
| 227 |
-
next_move_uci = self._decode_move_token(int(game_moves[move_idx]))
|
| 228 |
-
next_move = chess.Move.from_uci(next_move_uci)
|
| 229 |
-
from_square = next_move.from_square
|
| 230 |
-
to_square = next_move.to_square
|
| 231 |
-
|
| 232 |
-
# AlphaZero action plane for this move
|
| 233 |
-
action_plane = move_to_action_plane(from_square, to_square, next_move.promotion)
|
| 234 |
-
|
| 235 |
-
# Get turn indicator
|
| 236 |
-
is_white = board.turn # True for white, False for black
|
| 237 |
-
|
| 238 |
-
# Get castling rights as a single integer (0-15)
|
| 239 |
-
# Bit 0: white kingside, Bit 1: white queenside, Bit 2: black kingside, Bit 3: black queenside
|
| 240 |
-
castling_rights = 0
|
| 241 |
-
if board.has_kingside_castling_rights(chess.WHITE):
|
| 242 |
-
castling_rights |= 1
|
| 243 |
-
if board.has_queenside_castling_rights(chess.WHITE):
|
| 244 |
-
castling_rights |= 2
|
| 245 |
-
if board.has_kingside_castling_rights(chess.BLACK):
|
| 246 |
-
castling_rights |= 4
|
| 247 |
-
if board.has_queenside_castling_rights(chess.BLACK):
|
| 248 |
-
castling_rights |= 8
|
| 249 |
-
|
| 250 |
-
# Get en passant file (0-7 for a-h, 8 for none)
|
| 251 |
-
if board.has_legal_en_passant():
|
| 252 |
-
en_passant_file = chess.square_file(board.ep_square)
|
| 253 |
-
else:
|
| 254 |
-
en_passant_file = 8 # No en passant
|
| 255 |
-
|
| 256 |
-
return {
|
| 257 |
-
"position": position_tensor,
|
| 258 |
-
"move": next_move_token,
|
| 259 |
-
"is_white": is_white,
|
| 260 |
-
"castling_rights": castling_rights,
|
| 261 |
-
"en_passant_file": en_passant_file,
|
| 262 |
-
"move_number": move_idx,
|
| 263 |
-
"white_elo": white_elo,
|
| 264 |
-
"black_elo": black_elo,
|
| 265 |
-
"result": result,
|
| 266 |
-
"from_square": from_square,
|
| 267 |
-
"action_plane": action_plane,
|
| 268 |
-
"legal_moves_planes": legal_moves_planes,
|
| 269 |
-
}
|
| 270 |
-
|
| 271 |
-
def _decode_move_token(self, token: int) -> str:
|
| 272 |
-
"""Decode a move token back to UCI string."""
|
| 273 |
-
return self.move_tokenizer.decode(token)
|
| 274 |
-
|
| 275 |
-
|
| 276 |
-
if __name__ == "__main__":
|
| 277 |
-
# Example usage
|
| 278 |
-
import sys
|
| 279 |
-
|
| 280 |
-
if len(sys.argv) < 2:
|
| 281 |
-
print("Usage: python h5_lichess_dataset.py <path_to_h5_file>")
|
| 282 |
-
sys.exit(1)
|
| 283 |
-
|
| 284 |
-
dataset = HDF5ChessDataset(sys.argv[1], cache_size=100)
|
| 285 |
-
print(f"Dataset length: {len(dataset)}")
|
| 286 |
-
|
| 287 |
-
# Test a few samples
|
| 288 |
-
for i in range(min(5, len(dataset))):
|
| 289 |
-
sample = dataset[i]
|
| 290 |
-
print(f"\nSample {i}:")
|
| 291 |
-
print(f" Position shape: {sample['position'].shape}")
|
| 292 |
-
print(f" Move token: {sample['move'].item()}")
|
| 293 |
-
print(f" Turn: {'White' if sample['is_white'] else 'Black'}")
|
| 294 |
-
print(f" ELOs: {sample['white_elo']} vs {sample['black_elo']}")
|
| 295 |
-
print(f" Move number: {sample['move_number']} over {dataset.num_moves_per_game[i]} moves")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
chesstransformer/datasets/lichess.py
DELETED
|
@@ -1,223 +0,0 @@
|
|
| 1 |
-
from torch.utils.data import Dataset
|
| 2 |
-
import zstandard as zstd
|
| 3 |
-
import chess.pgn
|
| 4 |
-
import io
|
| 5 |
-
import torch
|
| 6 |
-
|
| 7 |
-
|
| 8 |
-
def extract_games_optimized(filepath, num_games=1000000, batch_size=1000):
|
| 9 |
-
"""
|
| 10 |
-
Optimized game extraction with progress tracking.
|
| 11 |
-
Process in batches and show estimated time remaining.
|
| 12 |
-
|
| 13 |
-
Args:
|
| 14 |
-
filepath: Path to the compressed PGN file
|
| 15 |
-
num_games: Number of games to extract (default: 1,000,000)
|
| 16 |
-
batch_size: Number of games to process in each batch (default: 1,000)
|
| 17 |
-
|
| 18 |
-
Returns:
|
| 19 |
-
List of extracted games
|
| 20 |
-
"""
|
| 21 |
-
games = []
|
| 22 |
-
dctx = zstd.ZstdDecompressor()
|
| 23 |
-
with open(filepath, "rb") as f:
|
| 24 |
-
with dctx.stream_reader(f) as reader:
|
| 25 |
-
text_stream = io.TextIOWrapper(reader, encoding="utf-8")
|
| 26 |
-
pgn = chess.pgn.read_game(text_stream)
|
| 27 |
-
game_count = 0
|
| 28 |
-
batch_count = 0
|
| 29 |
-
|
| 30 |
-
while pgn is not None and game_count < num_games:
|
| 31 |
-
# check if the game ended by a win/loss/draw
|
| 32 |
-
if pgn.headers.get("Result") not in ["1-0", "0-1", "1/2-1/2"]:
|
| 33 |
-
pgn = chess.pgn.read_game(text_stream)
|
| 34 |
-
continue
|
| 35 |
-
|
| 36 |
-
# Skip Chess960 games - only use classical chess
|
| 37 |
-
variant = pgn.headers.get("Variant", "Standard")
|
| 38 |
-
if variant != "Standard":
|
| 39 |
-
pgn = chess.pgn.read_game(text_stream)
|
| 40 |
-
continue
|
| 41 |
-
|
| 42 |
-
games.append(pgn)
|
| 43 |
-
game_count += 1
|
| 44 |
-
batch_count += 1
|
| 45 |
-
|
| 46 |
-
if batch_count >= batch_size:
|
| 47 |
-
batch_count = 0
|
| 48 |
-
print(f"Extracted {game_count}/{num_games} games...")
|
| 49 |
-
|
| 50 |
-
pgn = chess.pgn.read_game(text_stream)
|
| 51 |
-
|
| 52 |
-
print(f"Finished extracting {len(games)} games.")
|
| 53 |
-
return games
|
| 54 |
-
|
| 55 |
-
|
| 56 |
-
def encode_game_moves(game, notation="uci", seprator=" <STEP> "):
|
| 57 |
-
"""
|
| 58 |
-
Encode the moves of a single game using the provided tokenizer.
|
| 59 |
-
|
| 60 |
-
Args:
|
| 61 |
-
game: A chess.pgn.Game object
|
| 62 |
-
notation: Notation format for moves ("uci" or "san", default: "uci")
|
| 63 |
-
seprator: String to separate moves (default: " <STEP> ")
|
| 64 |
-
|
| 65 |
-
Returns:
|
| 66 |
-
List of moves as a single formatted string
|
| 67 |
-
"""
|
| 68 |
-
if notation == "uci":
|
| 69 |
-
move_sequence = seprator.join([move.uci() for move in game.mainline_moves()])
|
| 70 |
-
elif notation == "san":
|
| 71 |
-
board = game.board()
|
| 72 |
-
move_sequence = seprator.join(
|
| 73 |
-
[
|
| 74 |
-
board.san(move)
|
| 75 |
-
for move in game.mainline_moves()
|
| 76 |
-
if not board.push(move) # Push the move to update the board state
|
| 77 |
-
]
|
| 78 |
-
)
|
| 79 |
-
else:
|
| 80 |
-
raise ValueError("Unsupported notation format. Use 'uci' or 'san'.")
|
| 81 |
-
|
| 82 |
-
ending_condition = seprator + {
|
| 83 |
-
"1-0": "<1-0>",
|
| 84 |
-
"0-1": "<0-1>",
|
| 85 |
-
"1/2-1/2": "<1/2-1/2>",
|
| 86 |
-
}.get(game.headers.get("Result", ""), "")
|
| 87 |
-
if ending_condition:
|
| 88 |
-
move_sequence += ending_condition
|
| 89 |
-
|
| 90 |
-
return move_sequence
|
| 91 |
-
|
| 92 |
-
|
| 93 |
-
class LichessDataset(Dataset):
|
| 94 |
-
def __init__(
|
| 95 |
-
self,
|
| 96 |
-
dataset_path,
|
| 97 |
-
tokenizer,
|
| 98 |
-
notation="uci",
|
| 99 |
-
context_length=512,
|
| 100 |
-
nb_games=100000,
|
| 101 |
-
):
|
| 102 |
-
"""
|
| 103 |
-
Lichess dataset for training chess move prediction models.
|
| 104 |
-
|
| 105 |
-
Args:
|
| 106 |
-
dataset_path: Path to the lichess dataset file (PGN format)
|
| 107 |
-
tokenizer: Tokenizer object to convert moves to token IDs
|
| 108 |
-
notation: Notation format for moves ("uci" or "san", default: "uci")
|
| 109 |
-
context_length: Maximum sequence length for model input (default: 512)
|
| 110 |
-
"""
|
| 111 |
-
self.tokenizer = tokenizer
|
| 112 |
-
self.context_length = context_length
|
| 113 |
-
self.notation = notation
|
| 114 |
-
|
| 115 |
-
print(f"Loading dataset from {dataset_path}...")
|
| 116 |
-
self.games = extract_games_optimized(dataset_path, num_games=nb_games)
|
| 117 |
-
print(f"Total games loaded: {len(self.games)}")
|
| 118 |
-
|
| 119 |
-
self.encoded_games = []
|
| 120 |
-
# encode and stack games together
|
| 121 |
-
for idx, game in enumerate(self.games):
|
| 122 |
-
encoded_moves = encode_game_moves(game, notation=self.notation)
|
| 123 |
-
token_ids = self.tokenizer.encode(encoded_moves).ids
|
| 124 |
-
|
| 125 |
-
self.encoded_games.append(token_ids)
|
| 126 |
-
|
| 127 |
-
# flatten the list of token ids
|
| 128 |
-
self.encoded_games = [token_id for game in self.encoded_games for token_id in game]
|
| 129 |
-
|
| 130 |
-
def __len__(self):
|
| 131 |
-
return len(self.encoded_games) // self.context_length
|
| 132 |
-
|
| 133 |
-
def __getitem__(self, idx):
|
| 134 |
-
"""
|
| 135 |
-
Get a sequence of token IDs for training.
|
| 136 |
-
|
| 137 |
-
Args:
|
| 138 |
-
idx: Index of the game to retrieve
|
| 139 |
-
Returns:
|
| 140 |
-
input, target: Tuple of input and target token ID lists
|
| 141 |
-
"""
|
| 142 |
-
start_idx = idx * self.context_length
|
| 143 |
-
end_idx = start_idx + self.context_length + 1 # +1 for target shift
|
| 144 |
-
|
| 145 |
-
if end_idx > len(self.encoded_games):
|
| 146 |
-
raise IndexError("Index out of range for the dataset.")
|
| 147 |
-
|
| 148 |
-
input_ids = self.encoded_games[start_idx : end_idx - 1]
|
| 149 |
-
target_ids = self.encoded_games[start_idx + 1 : end_idx]
|
| 150 |
-
|
| 151 |
-
input_tensor = torch.tensor(input_ids, dtype=torch.long)
|
| 152 |
-
target_tensor = torch.tensor(target_ids, dtype=torch.long)
|
| 153 |
-
|
| 154 |
-
return input_tensor, target_tensor
|
| 155 |
-
|
| 156 |
-
|
| 157 |
-
if __name__ == "__main__":
|
| 158 |
-
from pathlib import Path
|
| 159 |
-
import argparse
|
| 160 |
-
from chesstransformer.models.tokenizer import create_bpe_tokenizer
|
| 161 |
-
|
| 162 |
-
def arg_parser():
|
| 163 |
-
parser = argparse.ArgumentParser(description="Test LichessDataset")
|
| 164 |
-
parser.add_argument(
|
| 165 |
-
"--dataset_path",
|
| 166 |
-
type=str,
|
| 167 |
-
default="data/Lichess Rated Games 2017.pgn.zst",
|
| 168 |
-
help="Path to the lichess dataset file (PGN format)",
|
| 169 |
-
)
|
| 170 |
-
parser.add_argument(
|
| 171 |
-
"--num_games",
|
| 172 |
-
type=int,
|
| 173 |
-
default=10000,
|
| 174 |
-
help="Number of games to load for testing (default: 1000)",
|
| 175 |
-
)
|
| 176 |
-
parser.add_argument(
|
| 177 |
-
"--vocab_path",
|
| 178 |
-
type=str,
|
| 179 |
-
default="src/chesstransformer/data/tokenizer_models/bpe_tokenizer_vocab2000.json",
|
| 180 |
-
help="Path to the trained tokenizer file",
|
| 181 |
-
)
|
| 182 |
-
parser.add_argument(
|
| 183 |
-
"--context_length",
|
| 184 |
-
type=int,
|
| 185 |
-
default=512,
|
| 186 |
-
help="Context length for sequences (default: 512)",
|
| 187 |
-
)
|
| 188 |
-
parser.add_argument(
|
| 189 |
-
"--notation",
|
| 190 |
-
type=str,
|
| 191 |
-
choices=["uci", "san"],
|
| 192 |
-
default="uci",
|
| 193 |
-
help="Notation format for moves (default: uci)",
|
| 194 |
-
)
|
| 195 |
-
args = parser.parse_args()
|
| 196 |
-
return args
|
| 197 |
-
|
| 198 |
-
args = arg_parser()
|
| 199 |
-
dataset_path = Path(args.dataset_path)
|
| 200 |
-
num_games = args.num_games
|
| 201 |
-
vocab_path = Path(args.vocab_path)
|
| 202 |
-
context_length = args.context_length
|
| 203 |
-
notation = args.notation
|
| 204 |
-
|
| 205 |
-
import tokenizers
|
| 206 |
-
|
| 207 |
-
tokenizer = tokenizers.Tokenizer.from_file(str(vocab_path))
|
| 208 |
-
print(f"Loading Lichess dataset from {dataset_path}...")
|
| 209 |
-
dataset = LichessDataset(
|
| 210 |
-
dataset_path=dataset_path,
|
| 211 |
-
tokenizer=tokenizer,
|
| 212 |
-
notation=notation,
|
| 213 |
-
context_length=context_length,
|
| 214 |
-
nb_games=num_games,
|
| 215 |
-
)
|
| 216 |
-
|
| 217 |
-
print(f"Dataset loaded with {len(dataset)} games.")
|
| 218 |
-
input_ids, target_ids = dataset[0]
|
| 219 |
-
print(f"Sample input IDs: {input_ids}")
|
| 220 |
-
print(f"Sample target IDs: {target_ids}")
|
| 221 |
-
print(f"Encoded and stacked {len(dataset.encoded_games)} token IDs.")
|
| 222 |
-
|
| 223 |
-
print(f"Decoded sample moves: {tokenizer.decode(input_ids)}")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
chesstransformer/datasets/lichess_simple_uci.py
DELETED
|
@@ -1,136 +0,0 @@
|
|
| 1 |
-
from pathlib import Path
|
| 2 |
-
import json
|
| 3 |
-
from torch.utils.data import Dataset
|
| 4 |
-
import zstandard as zstd
|
| 5 |
-
import chess.pgn
|
| 6 |
-
import io
|
| 7 |
-
import torch
|
| 8 |
-
import chess
|
| 9 |
-
|
| 10 |
-
|
| 11 |
-
root_path = Path(__file__).parents[1]
|
| 12 |
-
print(f"Root path: {root_path}")
|
| 13 |
-
|
| 14 |
-
|
| 15 |
-
class LichessSimpleUciDataset(Dataset):
|
| 16 |
-
def __init__(
|
| 17 |
-
self,
|
| 18 |
-
dataset_path,
|
| 19 |
-
vocab_path=root_path / "data/tokenizer_models/uci_vocab.json",
|
| 20 |
-
context_length=512,
|
| 21 |
-
num_samples=100_000,
|
| 22 |
-
):
|
| 23 |
-
"""
|
| 24 |
-
Lichess dataset in the simple UCI format."""
|
| 25 |
-
|
| 26 |
-
self.dataset_path = dataset_path
|
| 27 |
-
self.num_samples = num_samples
|
| 28 |
-
self.vocab_path = vocab_path
|
| 29 |
-
self.context_length = context_length
|
| 30 |
-
self._load_vocab()
|
| 31 |
-
self.games = self._extract_games()
|
| 32 |
-
|
| 33 |
-
def _extract_games(self, only_check_mate=True):
|
| 34 |
-
games = []
|
| 35 |
-
dctx = zstd.ZstdDecompressor()
|
| 36 |
-
with open(self.dataset_path, "rb") as f:
|
| 37 |
-
with dctx.stream_reader(f) as reader:
|
| 38 |
-
text_stream = io.TextIOWrapper(reader, encoding="utf-8")
|
| 39 |
-
game_count = 0
|
| 40 |
-
|
| 41 |
-
while game_count < self.num_samples:
|
| 42 |
-
pgn = chess.pgn.read_game(text_stream)
|
| 43 |
-
if pgn is None:
|
| 44 |
-
break
|
| 45 |
-
|
| 46 |
-
# Check if the game ended by a win/loss/draw
|
| 47 |
-
if pgn.headers.get("Result") not in ["1-0", "0-1", "1/2-1/2"]:
|
| 48 |
-
continue
|
| 49 |
-
|
| 50 |
-
# Skip Chess960 games - only use classical chess
|
| 51 |
-
variant = pgn.headers.get("Variant", "Standard")
|
| 52 |
-
if variant != "Standard":
|
| 53 |
-
continue
|
| 54 |
-
|
| 55 |
-
games.append(pgn)
|
| 56 |
-
game_count += 1
|
| 57 |
-
|
| 58 |
-
if game_count % 10000 == 0:
|
| 59 |
-
print(f"Extracted {game_count}/{self.num_samples} games...", end="\r")
|
| 60 |
-
|
| 61 |
-
print(f"\nFinished extracting {len(games)} games.")
|
| 62 |
-
return games
|
| 63 |
-
|
| 64 |
-
def _load_vocab(self):
|
| 65 |
-
with open(self.vocab_path, "r") as f:
|
| 66 |
-
self.vocab = json.load(f)
|
| 67 |
-
self.token_to_id = {token: idx for idx, token in enumerate(self.vocab)}
|
| 68 |
-
|
| 69 |
-
def __len__(self):
|
| 70 |
-
return len(self.games)
|
| 71 |
-
|
| 72 |
-
def __getitem__(self, idx):
|
| 73 |
-
game = self.games[idx]
|
| 74 |
-
token_ids = self.encode_game(game)
|
| 75 |
-
legals_mask = self.get_legal_moves_mask(game)
|
| 76 |
-
target_ids = token_ids[1:] + [-100] # Shifted targets with -100 for padding
|
| 77 |
-
return {
|
| 78 |
-
"input_ids": torch.tensor(token_ids, dtype=torch.long),
|
| 79 |
-
"target_ids": torch.tensor(target_ids, dtype=torch.long),
|
| 80 |
-
"legal_moves_mask": legals_mask,
|
| 81 |
-
}
|
| 82 |
-
|
| 83 |
-
def encode_game(self, game):
|
| 84 |
-
move_list = [move.uci() for move in game.mainline_moves()]
|
| 85 |
-
move_list = ["<START>"] + move_list + ["<END>"]
|
| 86 |
-
token_ids = self.encode_moves(move_list)
|
| 87 |
-
return token_ids
|
| 88 |
-
|
| 89 |
-
def encode_moves(self, moves):
|
| 90 |
-
token_ids = []
|
| 91 |
-
for move in moves:
|
| 92 |
-
if move in self.token_to_id:
|
| 93 |
-
token_ids.append(self.token_to_id[move])
|
| 94 |
-
else:
|
| 95 |
-
token_ids.append(self.token_to_id["<UNK>"])
|
| 96 |
-
|
| 97 |
-
return token_ids
|
| 98 |
-
|
| 99 |
-
def get_legal_moves_mask(self, game):
|
| 100 |
-
"""
|
| 101 |
-
For each move in the game, generate a mask of legal moves at that position.
|
| 102 |
-
Returns a list of masks, where each mask is a tensor of shape (vocab_size,)
|
| 103 |
-
with 1.0 for legal moves and 0.0 for illegal moves.
|
| 104 |
-
"""
|
| 105 |
-
board = game.board()
|
| 106 |
-
masks = []
|
| 107 |
-
|
| 108 |
-
for move in game.mainline_moves():
|
| 109 |
-
# Get all legal moves at current position
|
| 110 |
-
legal_moves = [m.uci() for m in board.legal_moves]
|
| 111 |
-
|
| 112 |
-
# Create mask for all tokens in vocabulary
|
| 113 |
-
mask = torch.zeros(len(self.vocab), dtype=torch.float32)
|
| 114 |
-
|
| 115 |
-
# Set 1.0 for legal moves
|
| 116 |
-
for legal_move in legal_moves:
|
| 117 |
-
if legal_move in self.token_to_id:
|
| 118 |
-
mask[self.token_to_id[legal_move]] = 1.0
|
| 119 |
-
|
| 120 |
-
masks.append(mask)
|
| 121 |
-
|
| 122 |
-
# Apply the move to the board for next iteration
|
| 123 |
-
board.push(move)
|
| 124 |
-
|
| 125 |
-
# Add mask for <START> and <END> tokens
|
| 126 |
-
start_end_mask = torch.zeros(len(self.vocab), dtype=torch.float32)
|
| 127 |
-
start_end_mask[self.token_to_id["<START>"]] = 1.0
|
| 128 |
-
start_end_mask[self.token_to_id["<END>"]] = 1.0
|
| 129 |
-
masks = [start_end_mask] + masks + [start_end_mask]
|
| 130 |
-
|
| 131 |
-
return torch.stack(masks)
|
| 132 |
-
|
| 133 |
-
def decode_game(self, token_ids):
|
| 134 |
-
id_to_token = {idx: token for token, idx in self.token_to_id.items()}
|
| 135 |
-
moves = [id_to_token.get(tid, "<UNK>") for tid in token_ids if tid in id_to_token]
|
| 136 |
-
return " ".join(moves)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
chesstransformer/datasets/puzzle_dataset.py
DELETED
|
@@ -1,259 +0,0 @@
|
|
| 1 |
-
import csv
|
| 2 |
-
import zstandard as zstd
|
| 3 |
-
import torch
|
| 4 |
-
from torch.utils.data import Dataset
|
| 5 |
-
import chess
|
| 6 |
-
from typing import Optional
|
| 7 |
-
from chesstransformer.models.tokenizer.position_tokenizer import PostionTokenizer
|
| 8 |
-
from chesstransformer.models.tokenizer.move_tokenizer import MoveTokenizer
|
| 9 |
-
|
| 10 |
-
|
| 11 |
-
class LichessPuzzleDataset(Dataset):
|
| 12 |
-
"""Dataset for Lichess chess puzzles.
|
| 13 |
-
|
| 14 |
-
Each puzzle consists of:
|
| 15 |
-
- A FEN position (the position where the puzzle starts)
|
| 16 |
-
- A sequence of moves (the solution to the puzzle)
|
| 17 |
-
- Rating and themes
|
| 18 |
-
|
| 19 |
-
The dataset returns position-move pairs where:
|
| 20 |
-
- position: The current board state encoded as 64 tokens
|
| 21 |
-
- move: The next move in the puzzle solution
|
| 22 |
-
- is_white: Whether white is to move
|
| 23 |
-
"""
|
| 24 |
-
|
| 25 |
-
def __init__(
|
| 26 |
-
self,
|
| 27 |
-
puzzle_path: str,
|
| 28 |
-
min_rating: int = None,
|
| 29 |
-
max_rating: int = None,
|
| 30 |
-
themes: list[str] = None,
|
| 31 |
-
max_puzzles: int = None,
|
| 32 |
-
):
|
| 33 |
-
"""
|
| 34 |
-
Args:
|
| 35 |
-
puzzle_path: Path to the .csv.zst puzzle file
|
| 36 |
-
min_rating: Minimum puzzle rating to include
|
| 37 |
-
max_rating: Maximum puzzle rating to include
|
| 38 |
-
themes: List of themes to filter by (e.g., ['mate', 'mateIn2'])
|
| 39 |
-
max_puzzles: Maximum number of puzzles to load (for testing)
|
| 40 |
-
"""
|
| 41 |
-
self.position_tokenizer = PostionTokenizer()
|
| 42 |
-
self.move_tokenizer = MoveTokenizer()
|
| 43 |
-
self.puzzles = []
|
| 44 |
-
|
| 45 |
-
print(f"Loading puzzles from {puzzle_path}...")
|
| 46 |
-
|
| 47 |
-
# Load and filter puzzles
|
| 48 |
-
with zstd.open(puzzle_path, "rt", encoding="utf-8") as f:
|
| 49 |
-
reader = csv.DictReader(f)
|
| 50 |
-
for i, row in enumerate(reader):
|
| 51 |
-
if max_puzzles and i >= max_puzzles:
|
| 52 |
-
break
|
| 53 |
-
|
| 54 |
-
# Parse puzzle data
|
| 55 |
-
puzzle_rating = int(row["Rating"])
|
| 56 |
-
puzzle_themes = row["Themes"].split()
|
| 57 |
-
|
| 58 |
-
# Apply filters
|
| 59 |
-
if min_rating and puzzle_rating < min_rating:
|
| 60 |
-
continue
|
| 61 |
-
if max_rating and puzzle_rating > max_rating:
|
| 62 |
-
continue
|
| 63 |
-
if themes and not any(theme in puzzle_themes for theme in themes):
|
| 64 |
-
continue
|
| 65 |
-
|
| 66 |
-
# Parse moves (space-separated UCI moves)
|
| 67 |
-
moves = row["Moves"].split()
|
| 68 |
-
if len(moves) < 2: # Need at least opponent move + our response
|
| 69 |
-
continue
|
| 70 |
-
|
| 71 |
-
self.puzzles.append(
|
| 72 |
-
{
|
| 73 |
-
"puzzle_id": row["PuzzleId"],
|
| 74 |
-
"fen": row["FEN"],
|
| 75 |
-
"moves": moves,
|
| 76 |
-
"rating": puzzle_rating,
|
| 77 |
-
"themes": puzzle_themes,
|
| 78 |
-
"game_url": row["GameUrl"],
|
| 79 |
-
}
|
| 80 |
-
)
|
| 81 |
-
|
| 82 |
-
if (i + 1) % 10000 == 0:
|
| 83 |
-
print(f"Loaded {i + 1} puzzles...")
|
| 84 |
-
|
| 85 |
-
print(f"Total puzzles loaded: {len(self.puzzles)}")
|
| 86 |
-
|
| 87 |
-
# Calculate total position-move pairs
|
| 88 |
-
self.total_pairs = sum(len(p["moves"]) - 1 for p in self.puzzles)
|
| 89 |
-
print(f"Total position-move pairs: {self.total_pairs}")
|
| 90 |
-
|
| 91 |
-
def __len__(self):
|
| 92 |
-
"""Return total number of position-move pairs across all puzzles."""
|
| 93 |
-
return self.total_pairs
|
| 94 |
-
|
| 95 |
-
def __getitem__(self, idx):
|
| 96 |
-
"""
|
| 97 |
-
Returns a position-move pair from the puzzle dataset.
|
| 98 |
-
|
| 99 |
-
The dataset is indexed as a flat sequence of all position-move pairs
|
| 100 |
-
across all puzzles. This means we need to:
|
| 101 |
-
1. Find which puzzle this index belongs to
|
| 102 |
-
2. Find which move within that puzzle
|
| 103 |
-
3. Replay the game to get the board state
|
| 104 |
-
4. Return the encoded position + next move
|
| 105 |
-
"""
|
| 106 |
-
# Find the puzzle and move index
|
| 107 |
-
cumulative = 0
|
| 108 |
-
for puzzle in self.puzzles:
|
| 109 |
-
puzzle_pairs = len(puzzle["moves"]) - 1
|
| 110 |
-
if idx < cumulative + puzzle_pairs:
|
| 111 |
-
# Found the right puzzle
|
| 112 |
-
move_idx = idx - cumulative
|
| 113 |
-
break
|
| 114 |
-
cumulative += puzzle_pairs
|
| 115 |
-
else:
|
| 116 |
-
raise IndexError(f"Index {idx} out of range")
|
| 117 |
-
|
| 118 |
-
# Set up the board from FEN
|
| 119 |
-
board = chess.Board(puzzle["fen"])
|
| 120 |
-
|
| 121 |
-
# Apply moves up to (but not including) the target move
|
| 122 |
-
for i in range(move_idx + 1): # +1 because first move is opponent's
|
| 123 |
-
move = chess.Move.from_uci(puzzle["moves"][i])
|
| 124 |
-
board.push(move)
|
| 125 |
-
|
| 126 |
-
# Get the target move
|
| 127 |
-
target_move = puzzle["moves"][move_idx + 1]
|
| 128 |
-
|
| 129 |
-
# Encode the position
|
| 130 |
-
position_tokens = self.position_tokenizer.encode(board)
|
| 131 |
-
|
| 132 |
-
# Encode the move
|
| 133 |
-
try:
|
| 134 |
-
move_token = self.move_tokenizer.encode(target_move)
|
| 135 |
-
except ValueError:
|
| 136 |
-
# If move not in vocabulary, skip this sample
|
| 137 |
-
# This shouldn't happen often with proper vocab
|
| 138 |
-
print(f"Warning: Move {target_move} not in vocabulary for puzzle {puzzle['puzzle_id']}")
|
| 139 |
-
move_token = 0 # Use a default
|
| 140 |
-
|
| 141 |
-
# Get legal moves mask for loss calculation
|
| 142 |
-
legal_moves = list(board.legal_moves)
|
| 143 |
-
legal_moves_mask = torch.zeros(self.move_tokenizer.vocab_size, dtype=torch.float32)
|
| 144 |
-
for move in legal_moves:
|
| 145 |
-
try:
|
| 146 |
-
move_id = self.move_tokenizer.encode(move.uci())
|
| 147 |
-
legal_moves_mask[move_id] = 1.0
|
| 148 |
-
except ValueError:
|
| 149 |
-
continue
|
| 150 |
-
|
| 151 |
-
return {
|
| 152 |
-
"position": torch.tensor(position_tokens, dtype=torch.long),
|
| 153 |
-
"move": torch.tensor(move_token, dtype=torch.long),
|
| 154 |
-
"is_white": board.turn == chess.WHITE,
|
| 155 |
-
"puzzle_id": puzzle["puzzle_id"],
|
| 156 |
-
"puzzle_rating": puzzle["rating"],
|
| 157 |
-
"legal_moves_mask": legal_moves_mask,
|
| 158 |
-
"themes": puzzle["themes"],
|
| 159 |
-
}
|
| 160 |
-
|
| 161 |
-
def get_puzzle_by_id(self, puzzle_id: str):
|
| 162 |
-
"""Get a complete puzzle by its ID."""
|
| 163 |
-
for puzzle in self.puzzles:
|
| 164 |
-
if puzzle["puzzle_id"] == puzzle_id:
|
| 165 |
-
return puzzle
|
| 166 |
-
return None
|
| 167 |
-
|
| 168 |
-
|
| 169 |
-
class LichessPuzzleFullSolutionDataset(Dataset):
|
| 170 |
-
"""Alternative puzzle dataset that returns complete puzzles (not individual moves).
|
| 171 |
-
|
| 172 |
-
Useful for evaluation where you want to test solving the entire puzzle.
|
| 173 |
-
"""
|
| 174 |
-
|
| 175 |
-
def __init__(
|
| 176 |
-
self,
|
| 177 |
-
puzzle_path: str,
|
| 178 |
-
min_rating: int = None,
|
| 179 |
-
max_rating: int = None,
|
| 180 |
-
themes: list[str] = None,
|
| 181 |
-
max_puzzles: int = None,
|
| 182 |
-
):
|
| 183 |
-
"""
|
| 184 |
-
Args:
|
| 185 |
-
puzzle_path: Path to the .csv.zst puzzle file
|
| 186 |
-
min_rating: Minimum puzzle rating to include
|
| 187 |
-
max_rating: Maximum puzzle rating to include
|
| 188 |
-
themes: List of themes to filter by
|
| 189 |
-
max_puzzles: Maximum number of puzzles to load
|
| 190 |
-
"""
|
| 191 |
-
self.position_tokenizer = PostionTokenizer()
|
| 192 |
-
self.move_tokenizer = MoveTokenizer()
|
| 193 |
-
self.puzzles = []
|
| 194 |
-
|
| 195 |
-
print(f"Loading puzzles from {puzzle_path}...")
|
| 196 |
-
|
| 197 |
-
with zstd.open(puzzle_path, "rt", encoding="utf-8") as f:
|
| 198 |
-
reader = csv.DictReader(f)
|
| 199 |
-
for i, row in enumerate(reader):
|
| 200 |
-
if max_puzzles and i >= max_puzzles:
|
| 201 |
-
break
|
| 202 |
-
|
| 203 |
-
puzzle_rating = int(row["Rating"])
|
| 204 |
-
puzzle_themes = row["Themes"].split()
|
| 205 |
-
|
| 206 |
-
if min_rating and puzzle_rating < min_rating:
|
| 207 |
-
continue
|
| 208 |
-
if max_rating and puzzle_rating > max_rating:
|
| 209 |
-
continue
|
| 210 |
-
if themes and not any(theme in puzzle_themes for theme in themes):
|
| 211 |
-
continue
|
| 212 |
-
|
| 213 |
-
moves = row["Moves"].split()
|
| 214 |
-
if len(moves) < 2:
|
| 215 |
-
continue
|
| 216 |
-
|
| 217 |
-
self.puzzles.append(
|
| 218 |
-
{
|
| 219 |
-
"puzzle_id": row["PuzzleId"],
|
| 220 |
-
"fen": row["FEN"],
|
| 221 |
-
"moves": moves,
|
| 222 |
-
"rating": puzzle_rating,
|
| 223 |
-
"themes": puzzle_themes,
|
| 224 |
-
"game_url": row["GameUrl"],
|
| 225 |
-
}
|
| 226 |
-
)
|
| 227 |
-
|
| 228 |
-
print(f"Total puzzles loaded: {len(self.puzzles)}")
|
| 229 |
-
|
| 230 |
-
def __len__(self):
|
| 231 |
-
return len(self.puzzles)
|
| 232 |
-
|
| 233 |
-
def __getitem__(self, idx):
|
| 234 |
-
"""Returns a complete puzzle with starting position and full solution."""
|
| 235 |
-
puzzle = self.puzzles[idx]
|
| 236 |
-
|
| 237 |
-
# Starting position
|
| 238 |
-
board = chess.Board(puzzle["fen"])
|
| 239 |
-
position_tokens = self.position_tokenizer.encode(board)
|
| 240 |
-
|
| 241 |
-
# Encode solution moves
|
| 242 |
-
solution_tokens = []
|
| 243 |
-
for move_uci in puzzle["moves"]:
|
| 244 |
-
try:
|
| 245 |
-
move_token = self.move_tokenizer.encode(move_uci)
|
| 246 |
-
solution_tokens.append(move_token)
|
| 247 |
-
except ValueError:
|
| 248 |
-
solution_tokens.append(0)
|
| 249 |
-
|
| 250 |
-
return {
|
| 251 |
-
"puzzle_id": puzzle["puzzle_id"],
|
| 252 |
-
"position": torch.tensor(position_tokens, dtype=torch.long),
|
| 253 |
-
"solution": torch.tensor(solution_tokens, dtype=torch.long),
|
| 254 |
-
"fen": puzzle["fen"],
|
| 255 |
-
"moves_uci": puzzle["moves"],
|
| 256 |
-
"rating": puzzle["rating"],
|
| 257 |
-
"themes": puzzle["themes"],
|
| 258 |
-
"is_white": board.turn == chess.WHITE,
|
| 259 |
-
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
chesstransformer/datasets/shard_format.py
DELETED
|
@@ -1,129 +0,0 @@
|
|
| 1 |
-
"""On-disk record format for the flat, memory-mappable training shards.
|
| 2 |
-
|
| 3 |
-
Shared by ``scripts/build_shards.py`` (writer), ``FlatShardDataset`` (reader) and
|
| 4 |
-
``scripts/verify_shards.py`` (equivalence harness), so the layout is defined exactly
|
| 5 |
-
once. See ``doc/dataset_shards.md`` §3 for the design rationale and the measurements
|
| 6 |
-
behind the choices below.
|
| 7 |
-
|
| 8 |
-
One sample is one fixed-width row. Only the nine fields ``pos2move_v2_trainer.py``
|
| 9 |
-
actually reads are stored; everything else ``HDF5ChessDataset`` returns is dropped.
|
| 10 |
-
|
| 11 |
-
The legal-move mask is the whole design problem: ``(64, 73)`` bool is 4,672 bytes and
|
| 12 |
-
would dominate the record 20:1. It is stored instead as a fixed-width list of at most
|
| 13 |
-
``LEGAL_CAP`` flat ``from_square * 73 + plane`` indices (129 bytes), which is what makes
|
| 14 |
-
one memmap row per sample — and therefore trivial rank/worker slicing — possible. The
|
| 15 |
-
mask is rebuilt on GPU by :func:`~chesstransformer.datasets.flat_shard_dataset.scatter_legal_planes`.
|
| 16 |
-
|
| 17 |
-
Fixed-width was chosen over CSR (~72 B/sample, no cap) deliberately: CSR is ~28 % smaller
|
| 18 |
-
and is what Megatron's ``IndexedDataset`` does, but it needs a custom ragged collate, and
|
| 19 |
-
the 1.4 GB it saves is on a dataset that already fits in page cache. The cap is not a
|
| 20 |
-
correctness risk because the *target* move's index is always written first (see
|
| 21 |
-
``LEGAL_CAP`` below), so clipping can only ever drop alternative legal moves, never the
|
| 22 |
-
label. Observed legal-move counts over 400 real positions: mean 33.9, p99.9 = 53, max 53.
|
| 23 |
-
"""
|
| 24 |
-
|
| 25 |
-
import numpy as np
|
| 26 |
-
|
| 27 |
-
NUM_ACTION_PLANES = 73
|
| 28 |
-
NUM_SQUARES = 64
|
| 29 |
-
|
| 30 |
-
#: Number of legal-move indices stored per sample. Sits well above the observed max of
|
| 31 |
-
#: 53. The builder counts overflows and prints the rate; if it ever climbs above ~0.01 %
|
| 32 |
-
#: raise this to 96 and rebuild.
|
| 33 |
-
LEGAL_CAP = 64
|
| 34 |
-
|
| 35 |
-
#: Flat action index space is ``[0, 4672)``. Unused slots in ``legal_idx`` are filled with
|
| 36 |
-
#: ``PAD_ACTION`` (== 4672), one past the end, so a GPU ``scatter_`` can write padding into
|
| 37 |
-
#: a scratch column that is then sliced off. Writing padding as 0 instead would silently
|
| 38 |
-
#: mark square a1 / plane 0 legal on every single sample.
|
| 39 |
-
PAD_ACTION = NUM_SQUARES * NUM_ACTION_PLANES
|
| 40 |
-
|
| 41 |
-
#: Packed record layout. Offsets are explicit rather than auto-computed so that both
|
| 42 |
-
#: ``uint16`` fields land on even offsets — an unaligned memmap view of ``legal_idx``
|
| 43 |
-
#: would force numpy into a slow per-element copy on every batch gather.
|
| 44 |
-
#:
|
| 45 |
-
#: ========== ====== =========================================================
|
| 46 |
-
#: offset dtype field
|
| 47 |
-
#: ========== ====== =========================================================
|
| 48 |
-
#: 0-63 u1×64 position board tokens, vocab 0-12
|
| 49 |
-
#: 64-191 u2×64 legal_idx flat legal actions, target first, PAD-filled
|
| 50 |
-
#: 192 u1 legal_cnt number of valid entries in legal_idx
|
| 51 |
-
#: 193 u1 from_square target move source square, 0-63
|
| 52 |
-
#: 194 u1 action_plane target move AlphaZero plane, 0-72
|
| 53 |
-
#: 195 u1 castling_rights 4-bit mask, 0-15
|
| 54 |
-
#: 196 u1 en_passant_file 0-7 = file a-h, 8 = none
|
| 55 |
-
#: 197 u1 is_white side to move
|
| 56 |
-
#: 198 i1 result 0 = draw, 1 = white win, 2 = black win (as in the HDF5)
|
| 57 |
-
#: 199 u1 (pad) keeps move_number 2-byte aligned
|
| 58 |
-
#: 200-201 u2 move_number ply index, max observed 601
|
| 59 |
-
#: ========== ====== =========================================================
|
| 60 |
-
RECORD_DTYPE = np.dtype(
|
| 61 |
-
{
|
| 62 |
-
"names": [
|
| 63 |
-
"position",
|
| 64 |
-
"legal_idx",
|
| 65 |
-
"legal_cnt",
|
| 66 |
-
"from_square",
|
| 67 |
-
"action_plane",
|
| 68 |
-
"castling_rights",
|
| 69 |
-
"en_passant_file",
|
| 70 |
-
"is_white",
|
| 71 |
-
"result",
|
| 72 |
-
"move_number",
|
| 73 |
-
],
|
| 74 |
-
"formats": [
|
| 75 |
-
(np.uint8, NUM_SQUARES),
|
| 76 |
-
(np.uint16, LEGAL_CAP),
|
| 77 |
-
np.uint8,
|
| 78 |
-
np.uint8,
|
| 79 |
-
np.uint8,
|
| 80 |
-
np.uint8,
|
| 81 |
-
np.uint8,
|
| 82 |
-
np.uint8,
|
| 83 |
-
np.int8,
|
| 84 |
-
np.uint16,
|
| 85 |
-
],
|
| 86 |
-
"offsets": [0, 64, 192, 193, 194, 195, 196, 197, 198, 200],
|
| 87 |
-
"itemsize": 202,
|
| 88 |
-
}
|
| 89 |
-
)
|
| 90 |
-
|
| 91 |
-
RECORD_BYTES = RECORD_DTYPE.itemsize
|
| 92 |
-
|
| 93 |
-
#: Bumped whenever the layout above changes in a way that makes old shards unreadable.
|
| 94 |
-
#: ``FlatShardDataset`` refuses to load a directory built by a different version.
|
| 95 |
-
FORMAT_VERSION = 1
|
| 96 |
-
|
| 97 |
-
|
| 98 |
-
def dtype_to_meta() -> dict:
|
| 99 |
-
"""Serialise :data:`RECORD_DTYPE` into ``meta.json``.
|
| 100 |
-
|
| 101 |
-
Stored so a shard directory is self-describing: a reader can reconstruct the exact
|
| 102 |
-
layout it was written with instead of trusting that this module never changed.
|
| 103 |
-
"""
|
| 104 |
-
return {
|
| 105 |
-
"names": list(RECORD_DTYPE.names),
|
| 106 |
-
"formats": [
|
| 107 |
-
(
|
| 108 |
-
[RECORD_DTYPE[n].subdtype[0].str, RECORD_DTYPE[n].subdtype[1][0]]
|
| 109 |
-
if RECORD_DTYPE[n].subdtype
|
| 110 |
-
else RECORD_DTYPE[n].str
|
| 111 |
-
)
|
| 112 |
-
for n in RECORD_DTYPE.names
|
| 113 |
-
],
|
| 114 |
-
"offsets": [RECORD_DTYPE.fields[n][1] for n in RECORD_DTYPE.names],
|
| 115 |
-
"itemsize": RECORD_BYTES,
|
| 116 |
-
}
|
| 117 |
-
|
| 118 |
-
|
| 119 |
-
def dtype_from_meta(meta: dict) -> np.dtype:
|
| 120 |
-
"""Rebuild a record dtype from what ``meta.json`` recorded."""
|
| 121 |
-
formats = [tuple(f) if isinstance(f, list) else f for f in meta["formats"]]
|
| 122 |
-
return np.dtype(
|
| 123 |
-
{
|
| 124 |
-
"names": list(meta["names"]),
|
| 125 |
-
"formats": formats,
|
| 126 |
-
"offsets": list(meta["offsets"]),
|
| 127 |
-
"itemsize": int(meta["itemsize"]),
|
| 128 |
-
}
|
| 129 |
-
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
chesstransformer/distributed_muon.py
DELETED
|
@@ -1,222 +0,0 @@
|
|
| 1 |
-
"""Muon with Newton-Schulz distributed across ranks, plus its schedules.
|
| 2 |
-
|
| 3 |
-
Under plain DDP every rank all-reduces to the *same* gradient and then every rank runs the
|
| 4 |
-
*same* Newton-Schulz iteration on it. That is pure redundant compute: at world size N,
|
| 5 |
-
N-1 copies of the NS work are thrown away, and every rank stores a full momentum buffer.
|
| 6 |
-
|
| 7 |
-
The fix is to partition the work — but the partition axis is the whole point. NS is
|
| 8 |
-
``X @ X.T`` plus a global spectral-norm division, so it needs the *complete* matrix.
|
| 9 |
-
Measured on a 64x32 weight, orthogonalizing per-shard and concatenating gives a **52-58 %
|
| 10 |
-
wrong update** at world size 2-4, and nothing raises: a row-slice of a 2D matrix is still a
|
| 11 |
-
2D matrix. That is the ZeRO-2 trap, and it is why sharding *inside* a tensor is off limits
|
| 12 |
-
here even though it is perfectly safe for elementwise optimizers like AdamW.
|
| 13 |
-
|
| 14 |
-
So this partitions by **whole parameter**, following nanochat's ``MuonAdamW``
|
| 15 |
-
(https://github.com/karpathy/nanochat, ``nanochat/optim.py``): params of a given shape are
|
| 16 |
-
stacked, the K of them are divided into contiguous chunks across ranks, each rank runs NS
|
| 17 |
-
only on the complete matrices it owns, and one ``all_gather`` per shape bucket republishes
|
| 18 |
-
the results. Momentum state exists only for owned params, so it is sharded too.
|
| 19 |
-
|
| 20 |
-
**Communication note, stated honestly.** nanochat does not use DDP: it ``reduce_scatter``s
|
| 21 |
-
gradients inside the optimizer, so grad-sync and state-sharding share one collective and
|
| 22 |
-
the total wire cost equals what DDP's all-reduce would have been. This class runs *on top
|
| 23 |
-
of* DDP, which has already all-reduced the gradients, so the ``all_gather`` here is
|
| 24 |
-
**additional** traffic — bytes DDP did not previously move. It buys NS compute (÷N) and
|
| 25 |
-
momentum memory (÷N) at the cost of one param-sized gather per step. On a fast interconnect
|
| 26 |
-
that is a clear win; on PCIe it may not be. ``scripts/bench_distributed_muon.py`` measures
|
| 27 |
-
it rather than assuming.
|
| 28 |
-
|
| 29 |
-
**Measured so far (gloo/CPU, preset 46m, 121 2D tensors, 16-core box):**
|
| 30 |
-
|
| 31 |
-
=========== =========== ==================
|
| 32 |
-
world size step time momentum state
|
| 33 |
-
=========== =========== ==================
|
| 34 |
-
1 243 ms 177.0 MiB/rank
|
| 35 |
-
2 638 ms 89.0 MiB/rank
|
| 36 |
-
4 491 ms 45.0 MiB/rank
|
| 37 |
-
=========== =========== ==================
|
| 38 |
-
|
| 39 |
-
The memory result is the designed one and is exact: state divides by world size. **The step
|
| 40 |
-
time is currently a regression**, and the honest reading is that this configuration has not
|
| 41 |
-
been shown to be a compute win. Three things confound the CPU numbers -- gloo is far slower
|
| 42 |
-
relative to compute than NCCL, N ranks on one 16-core box contend for the same cores so the
|
| 43 |
-
"NS compute / N" never materialises, and the benchmark is pure optimizer with no forward or
|
| 44 |
-
backward, so the all_gather is 100 % of the measured time instead of a few percent of a real
|
| 45 |
-
step. It still should not be reported as a win until multi-GPU says so.
|
| 46 |
-
|
| 47 |
-
The structural fix is the fused variant, which is why nanochat does not use DDP at all: it
|
| 48 |
-
``reduce_scatter``s gradients inside the optimizer, so grad-sync and state-sharding share one
|
| 49 |
-
collective and the total wire cost equals DDP's all-reduce -- the ``all_gather`` stops being
|
| 50 |
-
*extra*. Doing that here means excluding the Muon params from DDP's reducer
|
| 51 |
-
(``DistributedDataParallel._set_params_and_buffers_to_ignore_for_model``) and doing the
|
| 52 |
-
reduce-scatter in ``step()``. That is the next step, and it is what turns this from "memory
|
| 53 |
-
win, compute unproven" into a pure win.
|
| 54 |
-
"""
|
| 55 |
-
|
| 56 |
-
import math
|
| 57 |
-
|
| 58 |
-
import torch
|
| 59 |
-
import torch.distributed as dist
|
| 60 |
-
|
| 61 |
-
# Reuse torch's own kernels rather than reimplementing the algorithm: it makes this
|
| 62 |
-
# optimizer numerically identical to torch.optim.Muon by construction, which is exactly
|
| 63 |
-
# what tests/test_distributed_muon.py asserts. Private, so guarded.
|
| 64 |
-
try:
|
| 65 |
-
from torch.optim._muon import _adjust_lr, _zeropower_via_newtonschulz
|
| 66 |
-
except ImportError as exc: # pragma: no cover
|
| 67 |
-
raise ImportError(
|
| 68 |
-
"DistributedMuon reuses torch.optim._muon internals; needs a torch with "
|
| 69 |
-
"torch.optim.Muon (2.9+)."
|
| 70 |
-
) from exc
|
| 71 |
-
|
| 72 |
-
#: Params smaller than this stay replicated: every rank computes them redundantly and no
|
| 73 |
-
#: collective is issued. For Pos2MoveV2-large that is 9 of 13 shape buckets holding 0.2M of
|
| 74 |
-
#: 156.4M params — distributing them would cost 9 extra collectives to save nothing.
|
| 75 |
-
DEFAULT_MIN_DISTRIBUTED_NUMEL = 65_536
|
| 76 |
-
|
| 77 |
-
|
| 78 |
-
def muon_momentum(step: int, total_steps: int, warmup: int = 400,
|
| 79 |
-
warmdown_ratio: float = 0.2, lo: float = 0.85,
|
| 80 |
-
hi: float = 0.97, end: float = 0.90) -> float:
|
| 81 |
-
"""Momentum schedule: warm up 0.85 -> 0.97, then down to 0.90 during LR warmdown.
|
| 82 |
-
|
| 83 |
-
Muon's momentum controls how much history the orthogonalized direction carries. Early
|
| 84 |
-
on the gradient is changing fast and a long memory averages over a moving target, so it
|
| 85 |
-
starts low; late in training a slightly *shorter* memory helps the run settle, which is
|
| 86 |
-
why it comes back down rather than staying pinned at the peak. Schedule follows
|
| 87 |
-
nanochat's ``get_muon_momentum``.
|
| 88 |
-
"""
|
| 89 |
-
if step < warmup:
|
| 90 |
-
frac = step / max(1, warmup)
|
| 91 |
-
return (1 - frac) * lo + frac * hi
|
| 92 |
-
warmdown_iters = max(1, round(warmdown_ratio * total_steps))
|
| 93 |
-
warmdown_start = total_steps - warmdown_iters
|
| 94 |
-
if step >= warmdown_start:
|
| 95 |
-
progress = min(1.0, (step - warmdown_start) / warmdown_iters)
|
| 96 |
-
return hi * (1 - progress) + end * progress
|
| 97 |
-
return hi
|
| 98 |
-
|
| 99 |
-
|
| 100 |
-
def muon_weight_decay(step: int, total_steps: int, base_wd: float) -> float:
|
| 101 |
-
"""Cosine-decay weight decay to zero over the run, as nanochat's ``get_weight_decay``."""
|
| 102 |
-
return base_wd * 0.5 * (1 + math.cos(math.pi * min(1.0, step / max(1, total_steps))))
|
| 103 |
-
|
| 104 |
-
|
| 105 |
-
class DistributedMuon(torch.optim.Optimizer):
|
| 106 |
-
"""Drop-in for ``torch.optim.Muon`` that splits NS work and momentum across ranks.
|
| 107 |
-
|
| 108 |
-
Assumes gradients are already synchronized (DDP has all-reduced them), so every rank
|
| 109 |
-
starts from an identical gradient and only the *work* is divided. With no process group
|
| 110 |
-
initialized it degenerates to exactly ``torch.optim.Muon``.
|
| 111 |
-
"""
|
| 112 |
-
|
| 113 |
-
def __init__(self, params, lr=1e-3, weight_decay=0.1, momentum=0.95, nesterov=True,
|
| 114 |
-
ns_coefficients=(3.4445, -4.7750, 2.0315), eps=1e-7, ns_steps=5,
|
| 115 |
-
adjust_lr_fn=None, min_distributed_numel=DEFAULT_MIN_DISTRIBUTED_NUMEL):
|
| 116 |
-
defaults = dict(lr=lr, weight_decay=weight_decay, momentum=momentum,
|
| 117 |
-
nesterov=nesterov, ns_coefficients=ns_coefficients, eps=eps,
|
| 118 |
-
ns_steps=ns_steps, adjust_lr_fn=adjust_lr_fn)
|
| 119 |
-
super().__init__(params, defaults)
|
| 120 |
-
|
| 121 |
-
self.world_size = dist.get_world_size() if dist.is_initialized() else 1
|
| 122 |
-
self.rank = dist.get_rank() if dist.is_initialized() else 0
|
| 123 |
-
self.min_distributed_numel = min_distributed_numel
|
| 124 |
-
self._plan = [self._plan_group(g) for g in self.param_groups]
|
| 125 |
-
|
| 126 |
-
owned = sum(len(b["owned"]) for p in self._plan for b in p["buckets"])
|
| 127 |
-
total = sum(len(g["params"]) for g in self.param_groups)
|
| 128 |
-
self.owned_param_count, self.total_param_count = owned, total
|
| 129 |
-
|
| 130 |
-
def _plan_group(self, group):
|
| 131 |
-
"""Bucket a group's params by shape and assign contiguous chunks to ranks.
|
| 132 |
-
|
| 133 |
-
Contiguous rather than strided so that ``all_gather_into_tensor`` — which lays rank
|
| 134 |
-
i's contribution at output[i*chunk:(i+1)*chunk] — reassembles the bucket in order
|
| 135 |
-
with no extra permutation.
|
| 136 |
-
"""
|
| 137 |
-
by_shape: dict[tuple, list] = {}
|
| 138 |
-
for p in group["params"]:
|
| 139 |
-
if p.ndim != 2:
|
| 140 |
-
raise ValueError(f"DistributedMuon handles 2D params only, got {p.ndim}D")
|
| 141 |
-
by_shape.setdefault(tuple(p.shape), []).append(p)
|
| 142 |
-
|
| 143 |
-
buckets = []
|
| 144 |
-
for shape, plist in sorted(by_shape.items(), key=lambda kv: -kv[1][0].numel() * len(kv[1])):
|
| 145 |
-
k = len(plist)
|
| 146 |
-
distributed = (self.world_size > 1
|
| 147 |
-
and plist[0].numel() >= self.min_distributed_numel)
|
| 148 |
-
if not distributed:
|
| 149 |
-
# Replicated: every rank updates every param, no collective.
|
| 150 |
-
buckets.append(dict(shape=shape, params=plist, owned=plist,
|
| 151 |
-
offset=0, chunk=k, distributed=False))
|
| 152 |
-
continue
|
| 153 |
-
chunk = math.ceil(k / self.world_size)
|
| 154 |
-
start = self.rank * chunk
|
| 155 |
-
owned = plist[start:start + chunk] # may be short on the last rank
|
| 156 |
-
buckets.append(dict(shape=shape, params=plist, owned=owned,
|
| 157 |
-
offset=start, chunk=chunk, distributed=True))
|
| 158 |
-
return dict(buckets=buckets)
|
| 159 |
-
|
| 160 |
-
@torch.no_grad()
|
| 161 |
-
def step(self, closure=None):
|
| 162 |
-
loss = None
|
| 163 |
-
if closure is not None:
|
| 164 |
-
with torch.enable_grad():
|
| 165 |
-
loss = closure()
|
| 166 |
-
|
| 167 |
-
for group, plan in zip(self.param_groups, self._plan):
|
| 168 |
-
lr, wd = group["lr"], group["weight_decay"]
|
| 169 |
-
momentum, nesterov = group["momentum"], group["nesterov"]
|
| 170 |
-
for bucket in plan["buckets"]:
|
| 171 |
-
for p in bucket["owned"]:
|
| 172 |
-
if p.grad is None:
|
| 173 |
-
continue
|
| 174 |
-
self._update_one(p, lr, wd, momentum, nesterov, group)
|
| 175 |
-
for bucket in plan["buckets"]:
|
| 176 |
-
if bucket["distributed"]:
|
| 177 |
-
self._publish(bucket)
|
| 178 |
-
return loss
|
| 179 |
-
|
| 180 |
-
def _update_one(self, p, lr, wd, momentum, nesterov, group):
|
| 181 |
-
"""Exactly torch.optim.Muon's ``_single_tensor_muon`` body, for one param."""
|
| 182 |
-
grad = p.grad
|
| 183 |
-
if grad.ndim != 2:
|
| 184 |
-
raise ValueError("Param gradient must be a 2D matrix")
|
| 185 |
-
state = self.state[p]
|
| 186 |
-
if "momentum_buffer" not in state:
|
| 187 |
-
state["momentum_buffer"] = torch.zeros_like(p)
|
| 188 |
-
buf = state["momentum_buffer"]
|
| 189 |
-
buf.lerp_(grad, 1 - momentum)
|
| 190 |
-
update = grad.lerp(buf, momentum) if nesterov else buf
|
| 191 |
-
update = _zeropower_via_newtonschulz(
|
| 192 |
-
update, group["ns_coefficients"], group["ns_steps"], group["eps"])
|
| 193 |
-
adjusted_lr = _adjust_lr(lr, group["adjust_lr_fn"], p.shape)
|
| 194 |
-
p.mul_(1 - lr * wd)
|
| 195 |
-
p.add_(update, alpha=-adjusted_lr)
|
| 196 |
-
|
| 197 |
-
def _publish(self, bucket):
|
| 198 |
-
"""One all_gather per shape bucket to give every rank the updated params.
|
| 199 |
-
|
| 200 |
-
Zero-padding when K does not divide world_size: the tail slots are gathered and
|
| 201 |
-
discarded, which costs a little bandwidth and keeps the collective's shape uniform.
|
| 202 |
-
"""
|
| 203 |
-
plist, chunk, shape = bucket["params"], bucket["chunk"], bucket["shape"]
|
| 204 |
-
k, ws = len(plist), self.world_size
|
| 205 |
-
ref = plist[0]
|
| 206 |
-
local = torch.zeros(chunk, *shape, dtype=ref.dtype, device=ref.device)
|
| 207 |
-
for i, p in enumerate(bucket["owned"]):
|
| 208 |
-
local[i].copy_(p)
|
| 209 |
-
out = torch.empty(chunk * ws, *shape, dtype=ref.dtype, device=ref.device)
|
| 210 |
-
dist.all_gather_into_tensor(out, local)
|
| 211 |
-
for i in range(k):
|
| 212 |
-
plist[i].copy_(out[i])
|
| 213 |
-
|
| 214 |
-
def sharding_summary(self) -> str:
|
| 215 |
-
parts = []
|
| 216 |
-
for plan in self._plan:
|
| 217 |
-
for b in plan["buckets"]:
|
| 218 |
-
tag = "dist" if b["distributed"] else "repl"
|
| 219 |
-
parts.append(f"{tuple(b['shape'])}x{len(b['params'])}:{tag}")
|
| 220 |
-
return (f"DistributedMuon rank {self.rank}/{self.world_size}: owns "
|
| 221 |
-
f"{self.owned_param_count}/{self.total_param_count} params | "
|
| 222 |
-
+ " ".join(parts))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
chesstransformer/introspection/__init__.py
DELETED
|
@@ -1,5 +0,0 @@
|
|
| 1 |
-
"""Model introspection: attention capture, logit lens, occlusion saliency."""
|
| 2 |
-
|
| 3 |
-
from chesstransformer.introspection.analyze import PositionAnalyzer, load_model
|
| 4 |
-
|
| 5 |
-
__all__ = ["PositionAnalyzer", "load_model"]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
chesstransformer/introspection/analyze.py
DELETED
|
@@ -1,450 +0,0 @@
|
|
| 1 |
-
"""Per-position introspection of a Pos2MoveV2 network.
|
| 2 |
-
|
| 3 |
-
Everything here is read-only with respect to the model: attention maps are
|
| 4 |
-
recomputed from the captured layer inputs using the module's own projections,
|
| 5 |
-
so the numbers match what SDPA saw during the real forward pass.
|
| 6 |
-
|
| 7 |
-
Produced per position:
|
| 8 |
-
* policy over legal moves (probability, raw logit, action plane)
|
| 9 |
-
* value head output, plus the value/policy read off *every* layer
|
| 10 |
-
(logit lens) so you can see at which depth the model makes up its mind
|
| 11 |
-
* full attention tensor (L, H, 67, 67)
|
| 12 |
-
* attention rollout (Abnar & Zuidema 2020) over the layer stack
|
| 13 |
-
* occlusion saliency: remove one piece, measure the swing in value and in
|
| 14 |
-
the probability of the move the model wanted to play
|
| 15 |
-
* 1-ply child values, i.e. what the value head says about the position
|
| 16 |
-
*after* each legal move — the fastest way to spot policy/value disagreement
|
| 17 |
-
"""
|
| 18 |
-
|
| 19 |
-
from __future__ import annotations
|
| 20 |
-
|
| 21 |
-
import json
|
| 22 |
-
import math
|
| 23 |
-
import zlib
|
| 24 |
-
from base64 import b64encode
|
| 25 |
-
from dataclasses import dataclass
|
| 26 |
-
from pathlib import Path
|
| 27 |
-
|
| 28 |
-
import chess
|
| 29 |
-
import numpy as np
|
| 30 |
-
import torch
|
| 31 |
-
from safetensors import safe_open
|
| 32 |
-
|
| 33 |
-
from chesstransformer.models.tokenizer import PostionTokenizer
|
| 34 |
-
from chesstransformer.models.tokenizer.alphazero_move_encoder import (
|
| 35 |
-
NUM_ACTION_PLANES,
|
| 36 |
-
move_to_action_plane,
|
| 37 |
-
)
|
| 38 |
-
from chesstransformer.models.transformer.pos2move_v2 import Pos2MoveV2
|
| 39 |
-
|
| 40 |
-
BOARD_SQUARES = 64
|
| 41 |
-
CONTEXT_SIZE = 67
|
| 42 |
-
STATE_TOKEN_NAMES = ["castling", "en_passant", "player"]
|
| 43 |
-
|
| 44 |
-
|
| 45 |
-
def load_model(model_dir: str | Path, device: str = "cpu", use_ema: bool = False):
|
| 46 |
-
"""Load a Pos2MoveV2 checkpoint in fp32 (no compile, no bf16).
|
| 47 |
-
|
| 48 |
-
Introspection wants clean numbers more than it wants speed, and the model
|
| 49 |
-
is small enough that fp32 eager is instant either way.
|
| 50 |
-
"""
|
| 51 |
-
model_dir = Path(model_dir)
|
| 52 |
-
|
| 53 |
-
config_path = model_dir / "model_config.json"
|
| 54 |
-
if not config_path.exists():
|
| 55 |
-
config_path = model_dir.parent.parent / "model_config.json"
|
| 56 |
-
if not config_path.exists():
|
| 57 |
-
raise FileNotFoundError(f"No model_config.json found for {model_dir}")
|
| 58 |
-
with open(config_path) as f:
|
| 59 |
-
config = json.load(f)
|
| 60 |
-
|
| 61 |
-
model = Pos2MoveV2(**config)
|
| 62 |
-
|
| 63 |
-
weights = model_dir / "model.safetensors"
|
| 64 |
-
if not weights.exists():
|
| 65 |
-
raise FileNotFoundError(f"No model.safetensors in {model_dir}")
|
| 66 |
-
with safe_open(str(weights), framework="pt", device="cpu") as f:
|
| 67 |
-
state_dict = {k: f.get_tensor(k) for k in f.keys()}
|
| 68 |
-
state_dict = {k.replace("_orig_mod.", ""): v for k, v in state_dict.items()}
|
| 69 |
-
model.load_state_dict(state_dict)
|
| 70 |
-
|
| 71 |
-
if use_ema:
|
| 72 |
-
ema_path = model_dir / "ema_state.pt"
|
| 73 |
-
if not ema_path.exists():
|
| 74 |
-
raise FileNotFoundError(f"No ema_state.pt in {model_dir}")
|
| 75 |
-
ema_state = torch.load(str(ema_path), map_location="cpu", weights_only=True)
|
| 76 |
-
ema_state = {k.replace("_orig_mod.", ""): v for k, v in ema_state.items()}
|
| 77 |
-
model.load_state_dict(ema_state, strict=False)
|
| 78 |
-
|
| 79 |
-
model.eval()
|
| 80 |
-
return model.float().to(device), config
|
| 81 |
-
|
| 82 |
-
|
| 83 |
-
def encode_board(board: chess.Board, tokenizer: PostionTokenizer):
|
| 84 |
-
"""Same encoding the bots use: 64 piece tokens + castling/ep/player."""
|
| 85 |
-
tokens = tokenizer.encode(board)
|
| 86 |
-
castling = 0
|
| 87 |
-
if board.has_kingside_castling_rights(chess.WHITE):
|
| 88 |
-
castling |= 1
|
| 89 |
-
if board.has_queenside_castling_rights(chess.WHITE):
|
| 90 |
-
castling |= 2
|
| 91 |
-
if board.has_kingside_castling_rights(chess.BLACK):
|
| 92 |
-
castling |= 4
|
| 93 |
-
if board.has_queenside_castling_rights(chess.BLACK):
|
| 94 |
-
castling |= 8
|
| 95 |
-
ep_file = chess.square_file(board.ep_square) if board.has_legal_en_passant() else 8
|
| 96 |
-
return tokens, int(board.turn), castling, ep_file
|
| 97 |
-
|
| 98 |
-
|
| 99 |
-
@dataclass
|
| 100 |
-
class _Capture:
|
| 101 |
-
"""Residual stream and attention input for one layer."""
|
| 102 |
-
|
| 103 |
-
attn_input: torch.Tensor # normed input handed to the attention module
|
| 104 |
-
output: torch.Tensor # residual stream leaving the layer
|
| 105 |
-
|
| 106 |
-
|
| 107 |
-
class PositionAnalyzer:
|
| 108 |
-
def __init__(
|
| 109 |
-
self,
|
| 110 |
-
model: Pos2MoveV2,
|
| 111 |
-
device: str = "cpu",
|
| 112 |
-
max_moves: int = 64,
|
| 113 |
-
saliency: bool = True,
|
| 114 |
-
):
|
| 115 |
-
self.model = model
|
| 116 |
-
self.device = device
|
| 117 |
-
self.tokenizer = PostionTokenizer()
|
| 118 |
-
self.max_moves = max_moves
|
| 119 |
-
self.saliency = saliency
|
| 120 |
-
self.n_layers = len(model.transformer_layers)
|
| 121 |
-
self.n_heads = model.transformer_layers[0].attn.num_heads
|
| 122 |
-
|
| 123 |
-
# ── raw forward helpers ────────────────────────────────────────────
|
| 124 |
-
@torch.no_grad()
|
| 125 |
-
def _forward_batch(self, boards: list[chess.Board]):
|
| 126 |
-
enc = [encode_board(b, self.tokenizer) for b in boards]
|
| 127 |
-
tokens = torch.tensor([e[0] for e in enc], dtype=torch.long, device=self.device)
|
| 128 |
-
player = torch.tensor([e[1] for e in enc], dtype=torch.long, device=self.device)
|
| 129 |
-
castling = torch.tensor([e[2] for e in enc], dtype=torch.long, device=self.device)
|
| 130 |
-
ep = torch.tensor([e[3] for e in enc], dtype=torch.long, device=self.device)
|
| 131 |
-
return self.model(tokens, player, castling, ep)
|
| 132 |
-
|
| 133 |
-
@torch.no_grad()
|
| 134 |
-
def _forward_tokens(self, tokens, player, castling, ep):
|
| 135 |
-
return self.model(tokens, player, castling, ep)
|
| 136 |
-
|
| 137 |
-
@torch.no_grad()
|
| 138 |
-
def _forward_with_capture(self, board: chess.Board):
|
| 139 |
-
"""One forward pass, keeping every layer's input and output."""
|
| 140 |
-
captures: list[_Capture] = []
|
| 141 |
-
pending: dict[int, torch.Tensor] = {}
|
| 142 |
-
handles = []
|
| 143 |
-
|
| 144 |
-
def make_pre_hook(idx):
|
| 145 |
-
def hook(_module, args):
|
| 146 |
-
pending[idx] = args[0].detach()
|
| 147 |
-
|
| 148 |
-
return hook
|
| 149 |
-
|
| 150 |
-
def make_post_hook(idx):
|
| 151 |
-
def hook(_module, _args, output):
|
| 152 |
-
captures.append(_Capture(attn_input=pending[idx], output=output.detach()))
|
| 153 |
-
|
| 154 |
-
return hook
|
| 155 |
-
|
| 156 |
-
for i, layer in enumerate(self.model.transformer_layers):
|
| 157 |
-
handles.append(layer.attn.register_forward_pre_hook(make_pre_hook(i)))
|
| 158 |
-
handles.append(layer.register_forward_hook(make_post_hook(i)))
|
| 159 |
-
|
| 160 |
-
try:
|
| 161 |
-
tokens, player, castling, ep = encode_board(board, self.tokenizer)
|
| 162 |
-
out = self._forward_tokens(
|
| 163 |
-
torch.tensor([tokens], dtype=torch.long, device=self.device),
|
| 164 |
-
torch.tensor([player], dtype=torch.long, device=self.device),
|
| 165 |
-
torch.tensor([castling], dtype=torch.long, device=self.device),
|
| 166 |
-
torch.tensor([ep], dtype=torch.long, device=self.device),
|
| 167 |
-
)
|
| 168 |
-
finally:
|
| 169 |
-
for h in handles:
|
| 170 |
-
h.remove()
|
| 171 |
-
|
| 172 |
-
return out, captures
|
| 173 |
-
|
| 174 |
-
@torch.no_grad()
|
| 175 |
-
def _attention_probs(self, layer_idx: int, attn_input: torch.Tensor) -> torch.Tensor:
|
| 176 |
-
"""Recompute softmax attention for one layer: (H, 67, 67).
|
| 177 |
-
|
| 178 |
-
Mirrors ChessGroupedQueryAttention.forward up to the softmax — SDPA
|
| 179 |
-
adds the chess-relation bias to the *scaled* scores, so we do too.
|
| 180 |
-
"""
|
| 181 |
-
attn = self.model.transformer_layers[layer_idx].attn
|
| 182 |
-
x = attn_input
|
| 183 |
-
b, t, _ = x.shape
|
| 184 |
-
|
| 185 |
-
q = attn.q_proj(x).view(b, t, attn.num_heads, attn.head_dim).transpose(1, 2)
|
| 186 |
-
k = attn.k_proj(x).view(b, t, attn.num_kv_groups, attn.head_dim).transpose(1, 2)
|
| 187 |
-
q = attn.q_norm(q)
|
| 188 |
-
k = attn.k_norm(k)
|
| 189 |
-
k = k.repeat_interleave(attn.group_size, dim=1)
|
| 190 |
-
|
| 191 |
-
scores = (q @ k.transpose(-2, -1)) / math.sqrt(attn.head_dim)
|
| 192 |
-
scores = scores + attn.bias_table[:, attn.rel_idx].unsqueeze(0)
|
| 193 |
-
return torch.softmax(scores, dim=-1)[0]
|
| 194 |
-
|
| 195 |
-
# ── analysis ───────────────────────────────────────────────────────
|
| 196 |
-
def analyze(self, board: chess.Board, include_attention: bool = True) -> dict:
|
| 197 |
-
(move_logits, value), captures = self._forward_with_capture(board)
|
| 198 |
-
logits = move_logits[0].float() # (64, 73)
|
| 199 |
-
value = float(value[0].item())
|
| 200 |
-
|
| 201 |
-
result: dict = {
|
| 202 |
-
"fen": board.fen(),
|
| 203 |
-
"turn": "w" if board.turn else "b",
|
| 204 |
-
"fullmove": board.fullmove_number,
|
| 205 |
-
"check": board.is_check(),
|
| 206 |
-
"pieces": [
|
| 207 |
-
(board.piece_at(sq).symbol() if board.piece_at(sq) else None)
|
| 208 |
-
for sq in range(64)
|
| 209 |
-
],
|
| 210 |
-
"value": value,
|
| 211 |
-
}
|
| 212 |
-
|
| 213 |
-
legal = list(board.legal_moves)
|
| 214 |
-
result["legal_count"] = len(legal)
|
| 215 |
-
if not legal:
|
| 216 |
-
result["policy"] = {"moves": [], "entropy": 0.0, "top1_prob": 0.0}
|
| 217 |
-
result["terminal"] = board.result()
|
| 218 |
-
return result
|
| 219 |
-
|
| 220 |
-
result.update(self._policy_block(board, logits, legal))
|
| 221 |
-
result.update(self._depth_block(captures, logits, board, legal))
|
| 222 |
-
|
| 223 |
-
if include_attention:
|
| 224 |
-
result["attention"] = self._attention_block(captures)
|
| 225 |
-
|
| 226 |
-
if self.saliency:
|
| 227 |
-
result["saliency"] = self._saliency_block(board, value, result)
|
| 228 |
-
|
| 229 |
-
return result
|
| 230 |
-
|
| 231 |
-
def _policy_block(self, board: chess.Board, logits: torch.Tensor, legal) -> dict:
|
| 232 |
-
planes = [
|
| 233 |
-
move_to_action_plane(m.from_square, m.to_square, m.promotion) for m in legal
|
| 234 |
-
]
|
| 235 |
-
idx = torch.tensor(
|
| 236 |
-
[m.from_square * NUM_ACTION_PLANES + p for m, p in zip(legal, planes)],
|
| 237 |
-
dtype=torch.long,
|
| 238 |
-
)
|
| 239 |
-
flat = logits.reshape(-1).cpu()
|
| 240 |
-
legal_logits = flat[idx]
|
| 241 |
-
probs = torch.softmax(legal_logits, dim=0)
|
| 242 |
-
|
| 243 |
-
# Illegal mass: how much of the raw 64x73 softmax the model spends on
|
| 244 |
-
# moves that do not exist. A cheap sanity signal for a broken head.
|
| 245 |
-
full_probs = torch.softmax(flat, dim=0)
|
| 246 |
-
legal_mass = float(full_probs[idx].sum())
|
| 247 |
-
|
| 248 |
-
order = torch.argsort(probs, descending=True)
|
| 249 |
-
moves = []
|
| 250 |
-
for rank, i in enumerate(order.tolist()):
|
| 251 |
-
m = legal[i]
|
| 252 |
-
moves.append(
|
| 253 |
-
{
|
| 254 |
-
"uci": m.uci(),
|
| 255 |
-
"san": board.san(m),
|
| 256 |
-
"from": m.from_square,
|
| 257 |
-
"to": m.to_square,
|
| 258 |
-
"plane": planes[i],
|
| 259 |
-
"prob": float(probs[i]),
|
| 260 |
-
"logit": float(legal_logits[i]),
|
| 261 |
-
"rank": rank,
|
| 262 |
-
"capture": board.is_capture(m),
|
| 263 |
-
"check": board.gives_check(m),
|
| 264 |
-
}
|
| 265 |
-
)
|
| 266 |
-
|
| 267 |
-
p = probs[probs > 0]
|
| 268 |
-
entropy = float(-(p * p.log()).sum())
|
| 269 |
-
return {
|
| 270 |
-
"policy": {
|
| 271 |
-
"moves": moves[: self.max_moves],
|
| 272 |
-
"n_moves": len(moves),
|
| 273 |
-
"entropy": entropy,
|
| 274 |
-
"max_entropy": math.log(len(legal)),
|
| 275 |
-
"top1_prob": moves[0]["prob"],
|
| 276 |
-
"legal_mass": legal_mass,
|
| 277 |
-
}
|
| 278 |
-
}
|
| 279 |
-
|
| 280 |
-
@torch.no_grad()
|
| 281 |
-
def _depth_block(self, captures, final_logits, board, legal) -> dict:
|
| 282 |
-
"""Logit lens: decode value + policy from every layer's residual."""
|
| 283 |
-
norm = self.model.final_norm
|
| 284 |
-
value_by_layer = []
|
| 285 |
-
top_by_layer = []
|
| 286 |
-
write_norms = []
|
| 287 |
-
|
| 288 |
-
planes = [
|
| 289 |
-
move_to_action_plane(m.from_square, m.to_square, m.promotion) for m in legal
|
| 290 |
-
]
|
| 291 |
-
idx = torch.tensor(
|
| 292 |
-
[m.from_square * NUM_ACTION_PLANES + p for m, p in zip(legal, planes)],
|
| 293 |
-
dtype=torch.long,
|
| 294 |
-
device=self.device,
|
| 295 |
-
)
|
| 296 |
-
final_probs = torch.softmax(final_logits.reshape(-1)[idx.cpu()], dim=0)
|
| 297 |
-
final_top = int(torch.argmax(final_probs))
|
| 298 |
-
|
| 299 |
-
prev = None
|
| 300 |
-
for cap in captures:
|
| 301 |
-
x = norm(cap.output)
|
| 302 |
-
state = x[:, -3:, :].mean(dim=1)
|
| 303 |
-
value_by_layer.append(float(self.model.value_head(state)[0].item()))
|
| 304 |
-
|
| 305 |
-
lens_logits = self.model.move_head(x[:, :64, :])[0].reshape(-1)[idx]
|
| 306 |
-
lens_probs = torch.softmax(lens_logits.float(), dim=0)
|
| 307 |
-
best = int(torch.argmax(lens_probs))
|
| 308 |
-
top_by_layer.append(
|
| 309 |
-
{
|
| 310 |
-
"uci": legal[best].uci(),
|
| 311 |
-
"san": board.san(legal[best]),
|
| 312 |
-
"prob": float(lens_probs[best]),
|
| 313 |
-
"final_move_prob": float(lens_probs[final_top]),
|
| 314 |
-
"agrees": best == final_top,
|
| 315 |
-
}
|
| 316 |
-
)
|
| 317 |
-
|
| 318 |
-
out = cap.output
|
| 319 |
-
write_norms.append(
|
| 320 |
-
float((out - prev).norm().item()) if prev is not None else float(out.norm().item())
|
| 321 |
-
)
|
| 322 |
-
prev = out
|
| 323 |
-
|
| 324 |
-
return {
|
| 325 |
-
"depth": {
|
| 326 |
-
"value_by_layer": value_by_layer,
|
| 327 |
-
"top_by_layer": top_by_layer,
|
| 328 |
-
"residual_write_norm": write_norms,
|
| 329 |
-
}
|
| 330 |
-
}
|
| 331 |
-
|
| 332 |
-
@torch.no_grad()
|
| 333 |
-
def _attention_block(self, captures) -> dict:
|
| 334 |
-
"""Full (L, H, 67, 67) attention, u8-quantised per row, deflated."""
|
| 335 |
-
maps = []
|
| 336 |
-
for i, cap in enumerate(captures):
|
| 337 |
-
maps.append(self._attention_probs(i, cap.attn_input).cpu())
|
| 338 |
-
attn = torch.stack(maps) # (L, H, 67, 67)
|
| 339 |
-
|
| 340 |
-
rollout = self._rollout(attn)
|
| 341 |
-
|
| 342 |
-
# u8 per row, companded through a square root: attention rows are long
|
| 343 |
-
# tails of small weights, and linear u8 would quantise those to 20-30%
|
| 344 |
-
# relative error. sqrt spreads the error evenly across magnitudes.
|
| 345 |
-
row_max = attn.amax(dim=-1, keepdim=True).clamp(min=1e-9)
|
| 346 |
-
quant = ((attn / row_max).sqrt() * 255).round().clamp(0, 255).to(torch.uint8).numpy()
|
| 347 |
-
payload = zlib.compress(quant.tobytes(), 6)
|
| 348 |
-
|
| 349 |
-
return {
|
| 350 |
-
"shape": list(attn.shape),
|
| 351 |
-
"scale": row_max.squeeze(-1).flatten().numpy().astype(np.float32).tolist(),
|
| 352 |
-
"data": b64encode(payload).decode("ascii"),
|
| 353 |
-
"rollout": [round(v, 5) for v in rollout.tolist()],
|
| 354 |
-
}
|
| 355 |
-
|
| 356 |
-
@staticmethod
|
| 357 |
-
def _rollout(attn: torch.Tensor) -> torch.Tensor:
|
| 358 |
-
"""Attention rollout: how much of each token the output actually sees.
|
| 359 |
-
|
| 360 |
-
Head-averaged, residual-corrected (0.5*A + 0.5*I), multiplied through
|
| 361 |
-
the stack, then read from the three game-state token rows — those are
|
| 362 |
-
what the value head consumes.
|
| 363 |
-
"""
|
| 364 |
-
a = attn.mean(dim=1) # (L, T, T)
|
| 365 |
-
eye = torch.eye(a.shape[-1])
|
| 366 |
-
joint = None
|
| 367 |
-
for layer in a:
|
| 368 |
-
m = 0.5 * layer + 0.5 * eye
|
| 369 |
-
m = m / m.sum(dim=-1, keepdim=True)
|
| 370 |
-
joint = m if joint is None else m @ joint
|
| 371 |
-
return joint[-3:, :].mean(dim=0)
|
| 372 |
-
|
| 373 |
-
@torch.no_grad()
|
| 374 |
-
def _saliency_block(self, board: chess.Board, base_value: float, result: dict) -> dict:
|
| 375 |
-
"""Occlusion: delete one piece at a time, measure what moves.
|
| 376 |
-
|
| 377 |
-
Kings are left in place (an occluded king is not a chess position the
|
| 378 |
-
model has ever seen). Everything runs as one batch.
|
| 379 |
-
"""
|
| 380 |
-
top = result["policy"]["moves"][0]
|
| 381 |
-
top_idx = top["from"] * NUM_ACTION_PLANES + top["plane"]
|
| 382 |
-
|
| 383 |
-
squares = [
|
| 384 |
-
sq
|
| 385 |
-
for sq in range(64)
|
| 386 |
-
if board.piece_at(sq) is not None and board.piece_at(sq).piece_type != chess.KING
|
| 387 |
-
]
|
| 388 |
-
value_delta = [0.0] * 64
|
| 389 |
-
policy_delta = [0.0] * 64
|
| 390 |
-
if not squares:
|
| 391 |
-
return {"value_delta": value_delta, "policy_delta": policy_delta, "squares": []}
|
| 392 |
-
|
| 393 |
-
tokens, player, castling, ep = encode_board(board, self.tokenizer)
|
| 394 |
-
base = torch.tensor(tokens, dtype=torch.long).repeat(len(squares), 1)
|
| 395 |
-
for row, sq in enumerate(squares):
|
| 396 |
-
base[row, sq] = 0
|
| 397 |
-
n = len(squares)
|
| 398 |
-
move_logits, values = self._forward_tokens(
|
| 399 |
-
base.to(self.device),
|
| 400 |
-
torch.full((n,), player, dtype=torch.long, device=self.device),
|
| 401 |
-
torch.full((n,), castling, dtype=torch.long, device=self.device),
|
| 402 |
-
torch.full((n,), ep, dtype=torch.long, device=self.device),
|
| 403 |
-
)
|
| 404 |
-
|
| 405 |
-
# Probability of the top move under the *full* 64x73 softmax: occlusion
|
| 406 |
-
# changes the legal move set, so a legal-masked comparison would not be
|
| 407 |
-
# measuring the same quantity across rows.
|
| 408 |
-
flat = move_logits.float().reshape(n, -1).cpu()
|
| 409 |
-
probs = torch.softmax(flat, dim=1)[:, top_idx]
|
| 410 |
-
|
| 411 |
-
ref_logits, _ = self._forward_batch([board])
|
| 412 |
-
ref_prob = float(torch.softmax(ref_logits[0].float().reshape(-1), dim=0)[top_idx])
|
| 413 |
-
|
| 414 |
-
for row, sq in enumerate(squares):
|
| 415 |
-
value_delta[sq] = float(values[row].item()) - base_value
|
| 416 |
-
policy_delta[sq] = float(probs[row]) - ref_prob
|
| 417 |
-
|
| 418 |
-
return {
|
| 419 |
-
"value_delta": [round(v, 5) for v in value_delta],
|
| 420 |
-
"policy_delta": [round(v, 5) for v in policy_delta],
|
| 421 |
-
"squares": squares,
|
| 422 |
-
"top_move": top["uci"],
|
| 423 |
-
"ref_prob": ref_prob,
|
| 424 |
-
}
|
| 425 |
-
|
| 426 |
-
@torch.no_grad()
|
| 427 |
-
def child_values(self, board: chess.Board, moves: list[dict]) -> None:
|
| 428 |
-
"""Fill in the value of the position *after* each candidate move.
|
| 429 |
-
|
| 430 |
-
Stored from the mover's point of view (the raw head is side-to-move
|
| 431 |
-
POV, so it gets negated) — directly comparable to ``value``.
|
| 432 |
-
"""
|
| 433 |
-
if not moves:
|
| 434 |
-
return
|
| 435 |
-
boards = []
|
| 436 |
-
terminal = {}
|
| 437 |
-
for i, m in enumerate(moves):
|
| 438 |
-
b = board.copy(stack=False)
|
| 439 |
-
b.push(chess.Move.from_uci(m["uci"]))
|
| 440 |
-
if b.is_checkmate():
|
| 441 |
-
terminal[i] = 1.0
|
| 442 |
-
elif b.is_stalemate() or b.is_insufficient_material():
|
| 443 |
-
terminal[i] = 0.0
|
| 444 |
-
boards.append(b)
|
| 445 |
-
|
| 446 |
-
_, values = self._forward_batch(boards)
|
| 447 |
-
for i, m in enumerate(moves):
|
| 448 |
-
m["child_value"] = (
|
| 449 |
-
terminal[i] if i in terminal else -float(values[i].item())
|
| 450 |
-
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
chesstransformer/introspection/viewer.html
DELETED
|
@@ -1,860 +0,0 @@
|
|
| 1 |
-
<title>Pos2MoveV2 Glass Box</title>
|
| 2 |
-
<meta name="viewport" content="width=device-width, initial-scale=1">
|
| 3 |
-
<link rel="preconnect" href="https://fonts.googleapis.com">
|
| 4 |
-
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
|
| 5 |
-
<link rel="stylesheet" href="https://fonts.googleapis.com/css2?family=Archivo:wdth,wght@62..125,400..700&family=JetBrains+Mono:wght@400..600&display=swap">
|
| 6 |
-
|
| 7 |
-
<style>
|
| 8 |
-
:root {
|
| 9 |
-
--bg: #e9edec;
|
| 10 |
-
--panel: #f8faf9;
|
| 11 |
-
--panel-2: #f0f3f2;
|
| 12 |
-
--ink: #101a1c;
|
| 13 |
-
--ink-2: #47585b;
|
| 14 |
-
--ink-3: #7c8b8d;
|
| 15 |
-
--line: #d3dbd9;
|
| 16 |
-
--line-strong: #b9c4c2;
|
| 17 |
-
--accent: #8a6208;
|
| 18 |
-
--pos: #0d6b66;
|
| 19 |
-
--neg: #a4382a;
|
| 20 |
-
--neutral: #93a0a1;
|
| 21 |
-
--seq: #2b3a8f;
|
| 22 |
-
--sq-light: #dad5c8;
|
| 23 |
-
--sq-dark: #b0a996;
|
| 24 |
-
--piece-w: #fbf8f1;
|
| 25 |
-
--piece-b: #171c1e;
|
| 26 |
-
--shadow: 0 1px 2px rgba(16,26,28,.06), 0 8px 24px rgba(16,26,28,.06);
|
| 27 |
-
--radius: 10px;
|
| 28 |
-
}
|
| 29 |
-
@media (prefers-color-scheme: dark) {
|
| 30 |
-
:root:not([data-theme="light"]) {
|
| 31 |
-
--bg: #10171a;
|
| 32 |
-
--panel: #172023;
|
| 33 |
-
--panel-2: #1d282b;
|
| 34 |
-
--ink: #e8efee;
|
| 35 |
-
--ink-2: #a7b7b8;
|
| 36 |
-
--ink-3: #75878a;
|
| 37 |
-
--line: #27353a;
|
| 38 |
-
--line-strong: #3a4b50;
|
| 39 |
-
--accent: #d6a63c;
|
| 40 |
-
--pos: #3fb3a6;
|
| 41 |
-
--neg: #e0705c;
|
| 42 |
-
--neutral: #6f8083;
|
| 43 |
-
--seq: #8fa2ff;
|
| 44 |
-
--sq-light: #414c4f;
|
| 45 |
-
--sq-dark: #2d383b;
|
| 46 |
-
--piece-w: #f2f5f2;
|
| 47 |
-
--piece-b: #0b1012;
|
| 48 |
-
--shadow: 0 1px 2px rgba(0,0,0,.4), 0 10px 30px rgba(0,0,0,.28);
|
| 49 |
-
}
|
| 50 |
-
}
|
| 51 |
-
:root[data-theme="dark"] {
|
| 52 |
-
--bg: #10171a;
|
| 53 |
-
--panel: #172023;
|
| 54 |
-
--panel-2: #1d282b;
|
| 55 |
-
--ink: #e8efee;
|
| 56 |
-
--ink-2: #a7b7b8;
|
| 57 |
-
--ink-3: #75878a;
|
| 58 |
-
--line: #27353a;
|
| 59 |
-
--line-strong: #3a4b50;
|
| 60 |
-
--accent: #d6a63c;
|
| 61 |
-
--pos: #3fb3a6;
|
| 62 |
-
--neg: #e0705c;
|
| 63 |
-
--neutral: #6f8083;
|
| 64 |
-
--seq: #8fa2ff;
|
| 65 |
-
--sq-light: #414c4f;
|
| 66 |
-
--sq-dark: #2d383b;
|
| 67 |
-
--piece-w: #f2f5f2;
|
| 68 |
-
--piece-b: #0b1012;
|
| 69 |
-
--shadow: 0 1px 2px rgba(0,0,0,.4), 0 10px 30px rgba(0,0,0,.28);
|
| 70 |
-
}
|
| 71 |
-
|
| 72 |
-
* { box-sizing: border-box; }
|
| 73 |
-
|
| 74 |
-
body {
|
| 75 |
-
margin: 0;
|
| 76 |
-
background: var(--bg);
|
| 77 |
-
color: var(--ink);
|
| 78 |
-
font-family: Archivo, "Helvetica Neue", Arial, sans-serif;
|
| 79 |
-
font-size: 15px;
|
| 80 |
-
line-height: 1.5;
|
| 81 |
-
-webkit-font-smoothing: antialiased;
|
| 82 |
-
}
|
| 83 |
-
|
| 84 |
-
.wrap { max-width: 1400px; margin: 0 auto; padding: 28px 24px 64px; }
|
| 85 |
-
|
| 86 |
-
/* ── header ─────────────────────────────────────────────── */
|
| 87 |
-
header { display: flex; flex-wrap: wrap; align-items: flex-end; gap: 20px; justify-content: space-between; padding-bottom: 18px; border-bottom: 1px solid var(--line); }
|
| 88 |
-
h1 {
|
| 89 |
-
margin: 0;
|
| 90 |
-
font-size: clamp(26px, 3.4vw, 40px);
|
| 91 |
-
font-variation-settings: "wdth" 112, "wght" 700;
|
| 92 |
-
letter-spacing: -.02em;
|
| 93 |
-
line-height: 1.02;
|
| 94 |
-
text-wrap: balance;
|
| 95 |
-
}
|
| 96 |
-
h1 .sub { display: block; font-size: 13px; font-variation-settings: "wdth" 88, "wght" 500; letter-spacing: .14em; text-transform: uppercase; color: var(--accent); margin-bottom: 8px; }
|
| 97 |
-
.meta-grid { display: flex; flex-wrap: wrap; gap: 6px 22px; font-family: "JetBrains Mono", ui-monospace, monospace; font-size: 11.5px; color: var(--ink-3); }
|
| 98 |
-
.meta-grid b { color: var(--ink-2); font-weight: 500; }
|
| 99 |
-
|
| 100 |
-
/* ── generic panel ──────────────────────────────────────── */
|
| 101 |
-
.panel { background: var(--panel); border: 1px solid var(--line); border-radius: var(--radius); box-shadow: var(--shadow); }
|
| 102 |
-
.panel > h2, .head { display: flex; align-items: baseline; justify-content: space-between; gap: 12px; margin: 0; padding: 12px 16px; border-bottom: 1px solid var(--line); font-size: 11px; letter-spacing: .13em; text-transform: uppercase; font-variation-settings: "wdth" 88, "wght" 600; color: var(--ink-2); }
|
| 103 |
-
.head .note { font-family: "JetBrains Mono", ui-monospace, monospace; font-size: 10.5px; letter-spacing: 0; text-transform: none; color: var(--ink-3); }
|
| 104 |
-
|
| 105 |
-
/* ── position strip ─────────────────────────────────────── */
|
| 106 |
-
.strip { display: flex; gap: 6px; overflow-x: auto; padding: 14px 0 4px; }
|
| 107 |
-
.ply {
|
| 108 |
-
flex: 0 0 auto; border: 1px solid var(--line); background: var(--panel); color: var(--ink-2);
|
| 109 |
-
border-radius: 999px; padding: 5px 13px; cursor: pointer;
|
| 110 |
-
font-family: "JetBrains Mono", ui-monospace, monospace; font-size: 12px;
|
| 111 |
-
}
|
| 112 |
-
.ply:hover { border-color: var(--line-strong); color: var(--ink); }
|
| 113 |
-
.ply[aria-current="true"] { background: var(--accent); border-color: var(--accent); color: var(--bg); }
|
| 114 |
-
|
| 115 |
-
/* ── main layout ────────────────────────────────────────── */
|
| 116 |
-
.main { display: grid; grid-template-columns: minmax(360px, 520px) minmax(340px, 1fr); gap: 20px; margin-top: 20px; align-items: start; }
|
| 117 |
-
@media (max-width: 980px) { .main { grid-template-columns: 1fr; } }
|
| 118 |
-
|
| 119 |
-
/* ── board ──────────────────────────────────────────────── */
|
| 120 |
-
.board-shell { padding: 16px; }
|
| 121 |
-
@media (min-width: 981px) { .board-shell { position: sticky; top: 20px; } }
|
| 122 |
-
.board-frame { position: relative; }
|
| 123 |
-
.board { position: relative; display: grid; grid-template-columns: repeat(8, 1fr); aspect-ratio: 1; border-radius: 6px; overflow: hidden; border: 1px solid var(--line-strong); }
|
| 124 |
-
.sq { position: relative; display: flex; align-items: center; justify-content: center; cursor: pointer; }
|
| 125 |
-
.sq.light { background: var(--sq-light); }
|
| 126 |
-
.sq.dark { background: var(--sq-dark); }
|
| 127 |
-
.sq .heat { position: absolute; inset: 0; pointer-events: none; }
|
| 128 |
-
.sq .glyph { position: relative; font-size: min(6.2vw, 40px); line-height: 1; user-select: none; pointer-events: none; font-family: "Noto Sans Symbols 2", "DejaVu Sans", "Segoe UI Symbol", serif; }
|
| 129 |
-
.sq .glyph.w { color: var(--piece-w); -webkit-text-stroke: 1.1px var(--piece-b); }
|
| 130 |
-
.sq .glyph.b { color: var(--piece-b); -webkit-text-stroke: 1.1px rgba(250,250,245,.55); }
|
| 131 |
-
.sq .val { position: absolute; bottom: 2px; right: 3px; font-family: "JetBrains Mono", ui-monospace, monospace; font-size: 9px; color: var(--ink); opacity: .72; pointer-events: none; font-variant-numeric: tabular-nums; }
|
| 132 |
-
.sq.sel { outline: 2px solid var(--accent); outline-offset: -2px; }
|
| 133 |
-
.sq.lastmove::after { content: ""; position: absolute; inset: 0; border: 2px solid var(--accent); opacity: .45; pointer-events: none; }
|
| 134 |
-
.sq .rank-lbl, .sq .file-lbl { position: absolute; font-family: "JetBrains Mono", ui-monospace, monospace; font-size: 9px; color: var(--ink-2); opacity: .6; pointer-events: none; }
|
| 135 |
-
.sq .rank-lbl { top: 2px; left: 3px; }
|
| 136 |
-
.sq .file-lbl { bottom: 1px; left: 3px; }
|
| 137 |
-
#arrows { position: absolute; inset: 0; width: 100%; height: 100%; pointer-events: none; }
|
| 138 |
-
|
| 139 |
-
.modes { display: flex; flex-wrap: wrap; gap: 6px; margin-top: 14px; }
|
| 140 |
-
.mode {
|
| 141 |
-
border: 1px solid var(--line); background: var(--panel-2); color: var(--ink-2);
|
| 142 |
-
border-radius: 7px; padding: 6px 11px; cursor: pointer; font-size: 12.5px;
|
| 143 |
-
font-variation-settings: "wdth" 92, "wght" 500;
|
| 144 |
-
}
|
| 145 |
-
.mode:hover { border-color: var(--line-strong); color: var(--ink); }
|
| 146 |
-
.mode[aria-pressed="true"] { background: var(--accent); border-color: var(--accent); color: var(--bg); }
|
| 147 |
-
.scalebar { display: flex; align-items: center; gap: 9px; margin-top: 12px; font-family: "JetBrains Mono", ui-monospace, monospace; font-size: 10.5px; color: var(--ink-3); }
|
| 148 |
-
.ramp { flex: 1; height: 7px; border-radius: 4px; border: 1px solid var(--line); }
|
| 149 |
-
.fen { margin-top: 12px; font-family: "JetBrains Mono", ui-monospace, monospace; font-size: 10.5px; color: var(--ink-3); word-break: break-all; }
|
| 150 |
-
.hint { margin-top: 10px; font-size: 12px; color: var(--ink-3); }
|
| 151 |
-
|
| 152 |
-
/* ── stat tiles ─────────────────────────────────────────── */
|
| 153 |
-
.tiles { display: grid; grid-template-columns: repeat(auto-fit, minmax(168px, 1fr)); gap: 12px; }
|
| 154 |
-
.tile { background: var(--panel); border: 1px solid var(--line); border-radius: var(--radius); padding: 13px 15px; box-shadow: var(--shadow); }
|
| 155 |
-
.tile .lbl { font-size: 10px; letter-spacing: .13em; text-transform: uppercase; color: var(--ink-3); font-variation-settings: "wdth" 88, "wght" 600; }
|
| 156 |
-
.tile .big { font-size: 26px; font-variation-settings: "wdth" 105, "wght" 700; letter-spacing: -.02em; font-variant-numeric: tabular-nums; margin-top: 2px; }
|
| 157 |
-
.tile .small { font-family: "JetBrains Mono", ui-monospace, monospace; font-size: 11px; color: var(--ink-3); font-variant-numeric: tabular-nums; }
|
| 158 |
-
.meter { height: 6px; border-radius: 3px; background: var(--panel-2); border: 1px solid var(--line); margin-top: 8px; position: relative; overflow: hidden; }
|
| 159 |
-
.meter i { position: absolute; top: 0; bottom: 0; display: block; }
|
| 160 |
-
.meter .mid { position: absolute; top: -1px; bottom: -1px; left: 50%; width: 1px; background: var(--line-strong); }
|
| 161 |
-
|
| 162 |
-
/* ── move table ─────────────────────────────────────────── */
|
| 163 |
-
.tablewrap { overflow-x: auto; }
|
| 164 |
-
table { width: 100%; border-collapse: collapse; font-size: 13px; }
|
| 165 |
-
th { text-align: left; font-size: 10px; letter-spacing: .11em; text-transform: uppercase; color: var(--ink-3); font-variation-settings: "wdth" 88, "wght" 600; padding: 9px 12px; border-bottom: 1px solid var(--line); white-space: nowrap; }
|
| 166 |
-
td { padding: 6px 12px; border-bottom: 1px solid var(--line); vertical-align: middle; font-variant-numeric: tabular-nums; }
|
| 167 |
-
tbody tr { cursor: pointer; }
|
| 168 |
-
tbody tr:hover { background: var(--panel-2); }
|
| 169 |
-
tbody tr.played td { box-shadow: inset 2px 0 0 var(--accent); }
|
| 170 |
-
td.san { font-family: "JetBrains Mono", ui-monospace, monospace; font-weight: 600; white-space: nowrap; }
|
| 171 |
-
td.num { font-family: "JetBrains Mono", ui-monospace, monospace; font-size: 12px; }
|
| 172 |
-
.bar { position: relative; height: 9px; min-width: 60px; background: var(--panel-2); border-radius: 4px; }
|
| 173 |
-
.bar i { position: absolute; left: 0; top: 0; bottom: 0; background: var(--seq); border-radius: 0 4px 4px 0; }
|
| 174 |
-
.chip { display: inline-block; padding: 1px 7px; border-radius: 999px; font-family: "JetBrains Mono", ui-monospace, monospace; font-size: 11px; border: 1px solid var(--line-strong); }
|
| 175 |
-
|
| 176 |
-
/* ── charts ─────────────────────────────────────────────── */
|
| 177 |
-
.chartbox { padding: 14px 16px 8px; }
|
| 178 |
-
svg.chart { width: 100%; height: auto; display: block; overflow: visible; }
|
| 179 |
-
.axis text { font-family: "JetBrains Mono", ui-monospace, monospace; font-size: 9px; fill: var(--ink-3); }
|
| 180 |
-
.grid line { stroke: var(--line); stroke-width: 1; }
|
| 181 |
-
.zero { stroke: var(--line-strong); stroke-width: 1; stroke-dasharray: 3 3; }
|
| 182 |
-
.vline { stroke: var(--pos); stroke-width: 2; fill: none; stroke-linejoin: round; }
|
| 183 |
-
.tip { position: fixed; z-index: 40; pointer-events: none; background: var(--panel); border: 1px solid var(--line-strong); border-radius: 8px; padding: 7px 10px; font-family: "JetBrains Mono", ui-monospace, monospace; font-size: 11.5px; color: var(--ink); box-shadow: var(--shadow); opacity: 0; transition: opacity .1s; max-width: 260px; }
|
| 184 |
-
.tip.on { opacity: 1; }
|
| 185 |
-
|
| 186 |
-
/* ── attention explorer ─────────────────────────────────── */
|
| 187 |
-
.attn-section { margin-top: 22px; }
|
| 188 |
-
.attn-grid { display: grid; grid-template-columns: minmax(300px, 480px) 1fr; gap: 20px; align-items: start; }
|
| 189 |
-
@media (max-width: 980px) { .attn-grid { grid-template-columns: 1fr; } }
|
| 190 |
-
.controls { display: flex; flex-wrap: wrap; gap: 14px; align-items: center; padding: 12px 16px; border-bottom: 1px solid var(--line); }
|
| 191 |
-
.controls label { font-size: 11px; letter-spacing: .1em; text-transform: uppercase; color: var(--ink-3); font-variation-settings: "wdth" 88, "wght" 600; }
|
| 192 |
-
input[type=range] { accent-color: var(--accent); width: 190px; }
|
| 193 |
-
select, button.ghost { font-family: inherit; font-size: 12.5px; color: var(--ink); background: var(--panel-2); border: 1px solid var(--line); border-radius: 7px; padding: 5px 9px; }
|
| 194 |
-
canvas { display: block; image-rendering: pixelated; width: 100%; height: auto; border-radius: 6px; }
|
| 195 |
-
.heads { display: grid; grid-template-columns: repeat(auto-fill, minmax(84px, 1fr)); gap: 10px; padding: 14px 16px; }
|
| 196 |
-
.headcell { border: 1px solid var(--line); border-radius: 7px; padding: 5px; cursor: pointer; background: var(--panel-2); }
|
| 197 |
-
.headcell[aria-pressed="true"] { border-color: var(--accent); box-shadow: 0 0 0 1px var(--accent); }
|
| 198 |
-
.headcell figcaption { font-family: "JetBrains Mono", ui-monospace, monospace; font-size: 10px; color: var(--ink-3); text-align: center; margin-top: 4px; }
|
| 199 |
-
.biasrow { display: flex; gap: 3px; padding: 0 16px 14px; }
|
| 200 |
-
.biasrow div { flex: 1; text-align: center; font-family: "JetBrains Mono", ui-monospace, monospace; font-size: 9.5px; color: var(--ink-3); }
|
| 201 |
-
.biasrow .cell { height: 26px; border-radius: 4px; border: 1px solid var(--line); margin-bottom: 3px; }
|
| 202 |
-
|
| 203 |
-
details.reading { margin-top: 22px; }
|
| 204 |
-
details.reading summary { cursor: pointer; font-size: 11px; letter-spacing: .13em; text-transform: uppercase; color: var(--ink-2); font-variation-settings: "wdth" 88, "wght" 600; padding: 12px 0; }
|
| 205 |
-
.reading .cols { display: grid; grid-template-columns: repeat(auto-fit, minmax(260px, 1fr)); gap: 18px; }
|
| 206 |
-
.reading p { margin: 0 0 8px; color: var(--ink-2); font-size: 13.5px; max-width: 62ch; }
|
| 207 |
-
.reading code { font-family: "JetBrains Mono", ui-monospace, monospace; font-size: 12px; background: var(--panel-2); border: 1px solid var(--line); border-radius: 4px; padding: 1px 5px; }
|
| 208 |
-
.reading h3 { font-size: 13px; margin: 0 0 6px; font-variation-settings: "wdth" 100, "wght" 700; }
|
| 209 |
-
:focus-visible { outline: 2px solid var(--accent); outline-offset: 2px; }
|
| 210 |
-
@media (prefers-reduced-motion: reduce) { * { transition: none !important; animation: none !important; } }
|
| 211 |
-
</style>
|
| 212 |
-
|
| 213 |
-
<div class="wrap">
|
| 214 |
-
<header>
|
| 215 |
-
<h1><span class="sub" id="eyebrow">Model introspection</span><span id="title">Glass Box</span></h1>
|
| 216 |
-
<div class="meta-grid" id="meta"></div>
|
| 217 |
-
</header>
|
| 218 |
-
|
| 219 |
-
<div class="strip" id="strip"></div>
|
| 220 |
-
|
| 221 |
-
<div class="main">
|
| 222 |
-
<section class="panel board-shell">
|
| 223 |
-
<div class="board-frame">
|
| 224 |
-
<div class="board" id="board"></div>
|
| 225 |
-
<svg id="arrows" viewBox="0 0 800 800" aria-hidden="true"></svg>
|
| 226 |
-
</div>
|
| 227 |
-
<div class="modes" id="modes"></div>
|
| 228 |
-
<div class="scalebar" id="scalebar"></div>
|
| 229 |
-
<div class="hint" id="overlay-hint"></div>
|
| 230 |
-
<div class="fen" id="fen"></div>
|
| 231 |
-
</section>
|
| 232 |
-
|
| 233 |
-
<section style="display:grid; gap:16px;">
|
| 234 |
-
<div class="tiles" id="tiles"></div>
|
| 235 |
-
|
| 236 |
-
<div class="panel">
|
| 237 |
-
<div class="head"><span>Legal moves — policy vs. value</span><span class="note" id="moves-note"></span></div>
|
| 238 |
-
<div class="tablewrap"><table id="moves"></table></div>
|
| 239 |
-
</div>
|
| 240 |
-
|
| 241 |
-
<div class="panel chartbox">
|
| 242 |
-
<div class="head" style="margin:-14px -16px 10px; padding:12px 16px;"><span>Where the answer forms</span><span class="note" id="depth-note"></span></div>
|
| 243 |
-
<svg class="chart" id="depthchart" viewBox="0 0 720 260" preserveAspectRatio="xMidYMid meet"></svg>
|
| 244 |
-
</div>
|
| 245 |
-
</section>
|
| 246 |
-
</div>
|
| 247 |
-
|
| 248 |
-
<section class="panel attn-section" id="attn-panel">
|
| 249 |
-
<div class="controls">
|
| 250 |
-
<span style="font-size:11px;letter-spacing:.13em;text-transform:uppercase;font-variation-settings:'wdth' 88,'wght' 600;color:var(--ink-2)">Attention</span>
|
| 251 |
-
<label for="layer">Layer</label>
|
| 252 |
-
<input type="range" id="layer" min="0" max="15" value="15">
|
| 253 |
-
<span class="note" id="layer-val" style="font-family:'JetBrains Mono',monospace;font-size:12px"></span>
|
| 254 |
-
<label for="head">Head</label>
|
| 255 |
-
<select id="head"></select>
|
| 256 |
-
<label for="attnmode">Read</label>
|
| 257 |
-
<select id="attnmode">
|
| 258 |
-
<option value="row">row — where a token looks</option>
|
| 259 |
-
<option value="col">column — who looks at a token</option>
|
| 260 |
-
</select>
|
| 261 |
-
<span class="note" id="attn-status" style="color:var(--ink-3);font-family:'JetBrains Mono',monospace;font-size:11px"></span>
|
| 262 |
-
</div>
|
| 263 |
-
<div class="attn-grid">
|
| 264 |
-
<div style="padding:16px">
|
| 265 |
-
<canvas id="heatmap" width="67" height="67"></canvas>
|
| 266 |
-
<div class="scalebar"><span>0</span><span class="ramp" id="attn-ramp"></span><span id="attn-max">max</span></div>
|
| 267 |
-
<div class="hint">Rows are queries (the token doing the looking), columns are keys. The last three rows/columns are the castling, en-passant and side-to-move tokens. Click a cell to select that square on the board.</div>
|
| 268 |
-
</div>
|
| 269 |
-
<div>
|
| 270 |
-
<div class="head" style="border-top:0"><span>All heads, layer <span id="hl-layer"></span></span><span class="note">click to select</span></div>
|
| 271 |
-
<div class="heads" id="heads"></div>
|
| 272 |
-
<div class="head"><span>Relation bias — head <span id="hb-head"></span></span><span class="note">learned, position-independent</span></div>
|
| 273 |
-
<div class="biasrow" id="biasrow"></div>
|
| 274 |
-
</div>
|
| 275 |
-
</div>
|
| 276 |
-
</section>
|
| 277 |
-
|
| 278 |
-
<details class="reading">
|
| 279 |
-
<summary>How to read this / how it was made</summary>
|
| 280 |
-
<div class="cols">
|
| 281 |
-
<div>
|
| 282 |
-
<h3>Policy vs. value disagreement</h3>
|
| 283 |
-
<p>Every legal move carries its policy probability and the <em>child value</em> — the value head's verdict on the position after that move, from the mover's side. A move the policy loves but the value dislikes (or vice versa) is where search and network fight each other.</p>
|
| 284 |
-
</div>
|
| 285 |
-
<div>
|
| 286 |
-
<h3>Where the answer forms</h3>
|
| 287 |
-
<p>The value and policy are decoded from <em>every</em> layer's residual stream through the final norm and the real heads (a logit lens). If the top move is already locked in by layer 6 and nothing moves after, the last ten layers are not doing policy work.</p>
|
| 288 |
-
</div>
|
| 289 |
-
<div>
|
| 290 |
-
<h3>Occlusion saliency</h3>
|
| 291 |
-
<p>Each non-king piece is deleted in turn and the position re-evaluated. The square shows the swing in value, or in the probability of the move the model wanted. Big swings mark the pieces the verdict actually rests on.</p>
|
| 292 |
-
</div>
|
| 293 |
-
<div>
|
| 294 |
-
<h3>Attention and rollout</h3>
|
| 295 |
-
<p>Attention is recomputed from each layer's captured input using the model's own projections and its chess-relation bias, then checked against the real forward pass. Rollout multiplies the head-averaged, residual-corrected maps through the stack and reads the game-state token rows.</p>
|
| 296 |
-
</div>
|
| 297 |
-
</div>
|
| 298 |
-
<p style="margin-top:10px"><code id="repro"></code></p>
|
| 299 |
-
</details>
|
| 300 |
-
</div>
|
| 301 |
-
|
| 302 |
-
<div class="tip" id="tip"></div>
|
| 303 |
-
|
| 304 |
-
<script id="dump" type="application/json">__PAYLOAD__</script>
|
| 305 |
-
<script>
|
| 306 |
-
"use strict";
|
| 307 |
-
const DUMP = JSON.parse(document.getElementById("dump").textContent);
|
| 308 |
-
const META = DUMP.meta;
|
| 309 |
-
const POS = DUMP.positions;
|
| 310 |
-
const L = META.n_layers, H = META.n_heads, T = 67;
|
| 311 |
-
const GLYPH = { p: "♟", n: "♞", b: "♝", r: "♜", q: "♛", k: "♚" };
|
| 312 |
-
const FILES = "abcdefgh";
|
| 313 |
-
const STATE_NAMES = ["castling", "en-passant", "side-to-move"];
|
| 314 |
-
|
| 315 |
-
const state = {
|
| 316 |
-
pos: 0,
|
| 317 |
-
mode: "policy",
|
| 318 |
-
layer: L - 1,
|
| 319 |
-
head: -1, // -1 = mean over heads
|
| 320 |
-
attnMode: "row",
|
| 321 |
-
square: null, // selected token index 0..66
|
| 322 |
-
hoverMove: null,
|
| 323 |
-
};
|
| 324 |
-
const attnCache = new Map(); // pos index -> {q: Uint8Array, scale: Float32Array}
|
| 325 |
-
|
| 326 |
-
const css = (n) => getComputedStyle(document.documentElement).getPropertyValue(n).trim();
|
| 327 |
-
const sqName = (i) => (i < 64 ? FILES[i % 8] + (Math.floor(i / 8) + 1) : STATE_NAMES[i - 64]);
|
| 328 |
-
const fmt = (x, d = 3) => (x >= 0 ? "+" : "−") + Math.abs(x).toFixed(d);
|
| 329 |
-
const pct = (x) => (100 * x).toFixed(1) + "%";
|
| 330 |
-
|
| 331 |
-
/* ── colour ramps ──────────────────────────────────────────
|
| 332 |
-
sequential (attention, one hue, light→dark) and diverging
|
| 333 |
-
(value / saliency, two poles with a neutral middle). */
|
| 334 |
-
function mix(a, b, t) { return a.map((v, i) => Math.round(v + (b[i] - v) * t)); }
|
| 335 |
-
function rgb(c, alpha) { return `rgba(${c[0]},${c[1]},${c[2]},${alpha == null ? 1 : alpha})`; }
|
| 336 |
-
function hex2rgb(h) { h = h.replace("#", ""); return [0, 2, 4].map((i) => parseInt(h.slice(i, i + 2), 16)); }
|
| 337 |
-
function seqColor(t) { // 0..1
|
| 338 |
-
const lo = hex2rgb(css("--panel-2")), hi = hex2rgb(css("--seq"));
|
| 339 |
-
return mix(lo, hi, Math.max(0, Math.min(1, Math.pow(t, 0.45))));
|
| 340 |
-
}
|
| 341 |
-
function divColor(t) { // -1..1
|
| 342 |
-
const mid = hex2rgb(css("--neutral"));
|
| 343 |
-
const pole = hex2rgb(t >= 0 ? css("--pos") : css("--neg"));
|
| 344 |
-
return mix(mid, pole, Math.min(1, Math.abs(t)));
|
| 345 |
-
}
|
| 346 |
-
|
| 347 |
-
/* ── attention decoding ────────────────────────────────── */
|
| 348 |
-
async function inflate(b64) {
|
| 349 |
-
const bin = atob(b64);
|
| 350 |
-
const bytes = new Uint8Array(bin.length);
|
| 351 |
-
for (let i = 0; i < bin.length; i++) bytes[i] = bin.charCodeAt(i);
|
| 352 |
-
if (typeof DecompressionStream === "undefined") throw new Error("no DecompressionStream");
|
| 353 |
-
const stream = new Blob([bytes]).stream().pipeThrough(new DecompressionStream("deflate"));
|
| 354 |
-
return new Uint8Array(await new Response(stream).arrayBuffer());
|
| 355 |
-
}
|
| 356 |
-
async function getAttn(i) {
|
| 357 |
-
if (attnCache.has(i)) return attnCache.get(i);
|
| 358 |
-
const a = POS[i].attention;
|
| 359 |
-
if (!a) return null;
|
| 360 |
-
const q = await inflate(a.data);
|
| 361 |
-
const rec = { q, scale: Float32Array.from(a.scale) };
|
| 362 |
-
attnCache.set(i, rec);
|
| 363 |
-
return rec;
|
| 364 |
-
}
|
| 365 |
-
// attention[layer][head][from][to], de-quantised with the per-row scale.
|
| 366 |
-
// The stored byte is sqrt-companded, so square it on the way back out.
|
| 367 |
-
function attnAt(rec, l, h, i, j) {
|
| 368 |
-
const row = (l * H + h) * T + i;
|
| 369 |
-
const u = rec.q[row * T + j] / 255;
|
| 370 |
-
return u * u * rec.scale[row];
|
| 371 |
-
}
|
| 372 |
-
function headMeanAt(rec, l, i, j) {
|
| 373 |
-
let s = 0;
|
| 374 |
-
for (let h = 0; h < H; h++) s += attnAt(rec, l, h, i, j);
|
| 375 |
-
return s / H;
|
| 376 |
-
}
|
| 377 |
-
|
| 378 |
-
/* ── board ─────────────────────────────────────────────── */
|
| 379 |
-
const boardEl = document.getElementById("board");
|
| 380 |
-
const cells = [];
|
| 381 |
-
for (let row = 0; row < 8; row++) {
|
| 382 |
-
for (let col = 0; col < 8; col++) {
|
| 383 |
-
const sq = (7 - row) * 8 + col;
|
| 384 |
-
const d = document.createElement("div");
|
| 385 |
-
d.className = "sq " + ((row + col) % 2 === 0 ? "light" : "dark");
|
| 386 |
-
d.dataset.sq = sq;
|
| 387 |
-
d.innerHTML =
|
| 388 |
-
'<span class="heat"></span>' +
|
| 389 |
-
(col === 0 ? `<span class="rank-lbl">${8 - row}</span>` : "") +
|
| 390 |
-
(row === 7 ? `<span class="file-lbl">${FILES[col]}</span>` : "") +
|
| 391 |
-
'<span class="glyph"></span><span class="val"></span>';
|
| 392 |
-
d.addEventListener("click", () => {
|
| 393 |
-
state.square = state.square === sq ? null : sq;
|
| 394 |
-
render();
|
| 395 |
-
});
|
| 396 |
-
boardEl.appendChild(d);
|
| 397 |
-
cells[sq] = d;
|
| 398 |
-
}
|
| 399 |
-
}
|
| 400 |
-
|
| 401 |
-
const MODES = [
|
| 402 |
-
{ id: "policy", label: "Policy", hint: "Arrow width and opacity follow the move probability. The dashed arrow is the move actually played in the game." },
|
| 403 |
-
{ id: "attention", label: "Attention", hint: "Attention for the selected layer/head. Click a square to pin a query token; with nothing selected the board shows attention received from the three game-state tokens." },
|
| 404 |
-
{ id: "rollout", label: "Rollout", hint: "Attention rollout through the whole stack, read from the game-state tokens the value head consumes." },
|
| 405 |
-
{ id: "value", label: "Value saliency", hint: "Delete this piece, and the value moves by this much. Teal = the model likes the position more without it." },
|
| 406 |
-
{ id: "policysal", label: "Move saliency", hint: "Delete this piece, and the probability of the model's chosen move changes by this much." },
|
| 407 |
-
];
|
| 408 |
-
const modesEl = document.getElementById("modes");
|
| 409 |
-
MODES.forEach((m) => {
|
| 410 |
-
const b = document.createElement("button");
|
| 411 |
-
b.className = "mode"; b.textContent = m.label; b.dataset.mode = m.id;
|
| 412 |
-
b.addEventListener("click", () => { state.mode = m.id; render(); });
|
| 413 |
-
modesEl.appendChild(b);
|
| 414 |
-
});
|
| 415 |
-
|
| 416 |
-
/* ── overlays ──────────────────────────────────────────── */
|
| 417 |
-
function overlayValues(p, rec) {
|
| 418 |
-
// Returns {v: Float64Array(64), kind: "seq"|"div", max, label}
|
| 419 |
-
const out = new Float64Array(64);
|
| 420 |
-
if (state.mode === "rollout" && p.attention) {
|
| 421 |
-
for (let i = 0; i < 64; i++) out[i] = p.attention.rollout[i];
|
| 422 |
-
return { v: out, kind: "seq", max: Math.max(...out), label: "rollout weight" };
|
| 423 |
-
}
|
| 424 |
-
if (state.mode === "value" && p.saliency) {
|
| 425 |
-
for (let i = 0; i < 64; i++) out[i] = p.saliency.value_delta[i];
|
| 426 |
-
return { v: out, kind: "div", max: Math.max(...out.map(Math.abs)), label: "Δ value" };
|
| 427 |
-
}
|
| 428 |
-
if (state.mode === "policysal" && p.saliency) {
|
| 429 |
-
for (let i = 0; i < 64; i++) out[i] = p.saliency.policy_delta[i];
|
| 430 |
-
return { v: out, kind: "div", max: Math.max(...out.map(Math.abs)), label: "Δ p(" + p.saliency.top_move + ")" };
|
| 431 |
-
}
|
| 432 |
-
if (state.mode === "attention" && rec) {
|
| 433 |
-
const l = state.layer;
|
| 434 |
-
const get = (i, j) => (state.head < 0 ? headMeanAt(rec, l, i, j) : attnAt(rec, l, state.head, i, j));
|
| 435 |
-
if (state.square !== null) {
|
| 436 |
-
for (let i = 0; i < 64; i++) out[i] = state.attnMode === "row" ? get(state.square, i) : get(i, state.square);
|
| 437 |
-
} else {
|
| 438 |
-
for (let i = 0; i < 64; i++) { let s = 0; for (let t = 64; t < 67; t++) s += get(t, i); out[i] = s / 3; }
|
| 439 |
-
}
|
| 440 |
-
return { v: out, kind: "seq", max: Math.max(...out), label: "attention" };
|
| 441 |
-
}
|
| 442 |
-
return null;
|
| 443 |
-
}
|
| 444 |
-
|
| 445 |
-
function paintBoard(p, rec) {
|
| 446 |
-
const ov = overlayValues(p, rec);
|
| 447 |
-
const lastFrom = p.last_move ? sqIndex(p.last_move.slice(0, 2)) : null;
|
| 448 |
-
const lastTo = p.last_move ? sqIndex(p.last_move.slice(2, 4)) : null;
|
| 449 |
-
|
| 450 |
-
for (let sq = 0; sq < 64; sq++) {
|
| 451 |
-
const cell = cells[sq];
|
| 452 |
-
const glyph = cell.querySelector(".glyph");
|
| 453 |
-
const piece = p.pieces[sq];
|
| 454 |
-
if (piece) {
|
| 455 |
-
glyph.textContent = GLYPH[piece.toLowerCase()];
|
| 456 |
-
glyph.className = "glyph " + (piece === piece.toUpperCase() ? "w" : "b");
|
| 457 |
-
} else { glyph.textContent = ""; glyph.className = "glyph"; }
|
| 458 |
-
|
| 459 |
-
const heat = cell.querySelector(".heat");
|
| 460 |
-
const val = cell.querySelector(".val");
|
| 461 |
-
if (ov && ov.max > 0) {
|
| 462 |
-
const x = ov.v[sq];
|
| 463 |
-
if (ov.kind === "seq") {
|
| 464 |
-
heat.style.background = rgb(seqColor(x / ov.max), Math.min(0.88, 0.12 + 0.88 * (x / ov.max)));
|
| 465 |
-
} else {
|
| 466 |
-
const t = x / ov.max;
|
| 467 |
-
heat.style.background = Math.abs(t) < 0.02 ? "transparent" : rgb(divColor(t), Math.min(0.85, Math.abs(t) * 0.9));
|
| 468 |
-
}
|
| 469 |
-
val.textContent = ov.kind === "div"
|
| 470 |
-
? (Math.abs(x) >= ov.max * 0.25 ? (x >= 0 ? "+" : "−") + Math.abs(x).toFixed(2).slice(1) : "")
|
| 471 |
-
: (x >= ov.max * 0.3 ? (100 * x).toFixed(0) + "%" : "");
|
| 472 |
-
} else { heat.style.background = "transparent"; val.textContent = ""; }
|
| 473 |
-
|
| 474 |
-
cell.classList.toggle("sel", state.square === sq);
|
| 475 |
-
cell.classList.toggle("lastmove", sq === lastFrom || sq === lastTo);
|
| 476 |
-
}
|
| 477 |
-
|
| 478 |
-
const bar = document.getElementById("scalebar");
|
| 479 |
-
if (ov && ov.max > 0) {
|
| 480 |
-
const stops = ov.kind === "seq"
|
| 481 |
-
? [0, .25, .5, .75, 1].map((t) => rgb(seqColor(t))).join(",")
|
| 482 |
-
: [-1, -.5, 0, .5, 1].map((t) => rgb(divColor(t))).join(",");
|
| 483 |
-
bar.innerHTML = `<span>${ov.kind === "div" ? fmt(-ov.max, 2) : "0"}</span>
|
| 484 |
-
<span class="ramp" style="background:linear-gradient(90deg,${stops})"></span>
|
| 485 |
-
<span>${ov.kind === "div" ? fmt(ov.max, 2) : (100 * ov.max).toFixed(1) + "%"}</span>
|
| 486 |
-
<span style="margin-left:6px">${ov.label}</span>`;
|
| 487 |
-
} else bar.innerHTML = "";
|
| 488 |
-
}
|
| 489 |
-
|
| 490 |
-
function sqIndex(name) { return (parseInt(name[1], 10) - 1) * 8 + FILES.indexOf(name[0]); }
|
| 491 |
-
function sqCenter(sq) { const c = sq % 8, r = Math.floor(sq / 8); return [c * 100 + 50, (7 - r) * 100 + 50]; }
|
| 492 |
-
|
| 493 |
-
function paintArrows(p) {
|
| 494 |
-
const svg = document.getElementById("arrows");
|
| 495 |
-
svg.innerHTML = "";
|
| 496 |
-
if (state.mode !== "policy" && state.hoverMove === null) return;
|
| 497 |
-
const moves = state.hoverMove !== null
|
| 498 |
-
? [p.policy.moves[state.hoverMove]]
|
| 499 |
-
: p.policy.moves.slice(0, 6);
|
| 500 |
-
const defs = document.createElementNS("http://www.w3.org/2000/svg", "defs");
|
| 501 |
-
svg.appendChild(defs);
|
| 502 |
-
moves.forEach((m, i) => {
|
| 503 |
-
const [x1, y1] = sqCenter(m.from), [x2, y2] = sqCenter(m.to);
|
| 504 |
-
const opacity = state.hoverMove !== null ? 0.95 : Math.max(0.18, Math.min(0.92, 0.25 + m.prob));
|
| 505 |
-
const w = 6 + 20 * Math.sqrt(m.prob);
|
| 506 |
-
const color = i === 0 || state.hoverMove !== null ? css("--accent") : css("--seq");
|
| 507 |
-
const dx = x2 - x1, dy = y2 - y1, len = Math.hypot(dx, dy);
|
| 508 |
-
const ux = dx / len, uy = dy / len;
|
| 509 |
-
const hx = x2 - ux * 26, hy = y2 - uy * 26; // stop short for the head
|
| 510 |
-
const line = document.createElementNS("http://www.w3.org/2000/svg", "path");
|
| 511 |
-
line.setAttribute("d", `M ${x1 + ux * 16} ${y1 + uy * 16} L ${hx} ${hy}`);
|
| 512 |
-
line.setAttribute("stroke", color);
|
| 513 |
-
line.setAttribute("stroke-width", w);
|
| 514 |
-
line.setAttribute("stroke-linecap", "round");
|
| 515 |
-
line.setAttribute("opacity", opacity);
|
| 516 |
-
line.setAttribute("fill", "none");
|
| 517 |
-
svg.appendChild(line);
|
| 518 |
-
const head = document.createElementNS("http://www.w3.org/2000/svg", "path");
|
| 519 |
-
const px = -uy, py = ux, s = w * 1.5;
|
| 520 |
-
head.setAttribute("d", `M ${x2 - ux * 4} ${y2 - uy * 4} L ${hx + px * s} ${hy + py * s} L ${hx - px * s} ${hy - py * s} Z`);
|
| 521 |
-
head.setAttribute("fill", color);
|
| 522 |
-
head.setAttribute("opacity", opacity);
|
| 523 |
-
svg.appendChild(head);
|
| 524 |
-
});
|
| 525 |
-
const topUci = p.policy.moves[0] && p.policy.moves[0].uci;
|
| 526 |
-
if (p.played_uci && p.played_uci !== topUci && state.hoverMove === null) {
|
| 527 |
-
const [x1, y1] = sqCenter(sqIndex(p.played_uci.slice(0, 2)));
|
| 528 |
-
const [x2, y2] = sqCenter(sqIndex(p.played_uci.slice(2, 4)));
|
| 529 |
-
const dash = document.createElementNS("http://www.w3.org/2000/svg", "path");
|
| 530 |
-
dash.setAttribute("d", `M ${x1} ${y1} L ${x2} ${y2}`);
|
| 531 |
-
dash.setAttribute("stroke", css("--ink"));
|
| 532 |
-
dash.setAttribute("stroke-width", "3");
|
| 533 |
-
dash.setAttribute("stroke-dasharray", "10 8");
|
| 534 |
-
dash.setAttribute("opacity", ".65");
|
| 535 |
-
dash.setAttribute("fill", "none");
|
| 536 |
-
svg.appendChild(dash);
|
| 537 |
-
}
|
| 538 |
-
}
|
| 539 |
-
|
| 540 |
-
/* ── tiles ─────────────────────────────────────────────── */
|
| 541 |
-
// The move the value head would pick if it, not the policy, chose. When this
|
| 542 |
-
// disagrees with the policy's move, search will pull away from the prior —
|
| 543 |
-
// and that gap is usually where the interesting bugs live.
|
| 544 |
-
function valuePick(p) {
|
| 545 |
-
const scored = p.policy.moves.filter((m) => m.child_value != null);
|
| 546 |
-
if (scored.length < 2) return null;
|
| 547 |
-
let best = scored[0];
|
| 548 |
-
for (const m of scored) if (m.child_value > best.child_value) best = m;
|
| 549 |
-
const same = best.uci === p.policy.moves[0].uci;
|
| 550 |
-
return {
|
| 551 |
-
lbl: "Value head's pick",
|
| 552 |
-
big: best.san,
|
| 553 |
-
small: same ? "agrees with the policy" : `policy rank #${best.rank + 1} · ${pct(best.prob)} · ${fmt(best.child_value, 3)}`,
|
| 554 |
-
meter: { from: 0, to: best.prob, color: same ? css("--pos") : css("--neg") },
|
| 555 |
-
};
|
| 556 |
-
}
|
| 557 |
-
|
| 558 |
-
function paintTiles(p) {
|
| 559 |
-
const top = p.policy.moves[0];
|
| 560 |
-
const ent = p.policy.entropy, maxEnt = p.policy.max_entropy || 1;
|
| 561 |
-
const played = p.played_san;
|
| 562 |
-
const rankOfPlayed = played ? p.policy.moves.findIndex((m) => m.san === played) : -1;
|
| 563 |
-
const vPct = (p.value + 1) / 2;
|
| 564 |
-
const cmp = p.compare;
|
| 565 |
-
const tiles = [
|
| 566 |
-
{ lbl: "Model's move", big: top ? top.san : "—", small: top ? pct(top.prob) + " of the legal mass" + (cmp ? " · other: " + cmp.top1 : "") : "",
|
| 567 |
-
meter: { from: 0, to: top ? top.prob : 0, color: css("--seq") } },
|
| 568 |
-
{ lbl: "Value (side to move)", big: top ? fmt(p.value, 3) : fmt(p.value, 3),
|
| 569 |
-
small: (p.turn === "w" ? "White" : "Black") + " to move" + (cmp ? " · other: " + fmt(cmp.value, 3) : ""),
|
| 570 |
-
meter: { from: Math.min(0.5, vPct), to: Math.max(0.5, vPct), color: p.value >= 0 ? css("--pos") : css("--neg"), mid: true } },
|
| 571 |
-
{ lbl: "Policy entropy", big: ent.toFixed(2) + " b", small: "of " + maxEnt.toFixed(2) + " b over " + p.legal_count + " moves",
|
| 572 |
-
meter: { from: 0, to: ent / maxEnt, color: css("--accent") } },
|
| 573 |
-
valuePick(p),
|
| 574 |
-
played ? { lbl: "Move played", big: played, small: rankOfPlayed >= 0 ? "model rank #" + (rankOfPlayed + 1) + " · " + pct(p.policy.moves[rankOfPlayed].prob) : "not in the legal list",
|
| 575 |
-
meter: { from: 0, to: rankOfPlayed >= 0 ? p.policy.moves[rankOfPlayed].prob : 0, color: css("--accent") } } : null,
|
| 576 |
-
].filter(Boolean);
|
| 577 |
-
|
| 578 |
-
document.getElementById("tiles").innerHTML = tiles.map((t) => `
|
| 579 |
-
<div class="tile">
|
| 580 |
-
<div class="lbl">${t.lbl}</div>
|
| 581 |
-
<div class="big">${t.big}</div>
|
| 582 |
-
<div class="small">${t.small}</div>
|
| 583 |
-
<div class="meter">${t.meter.mid ? '<span class="mid"></span>' : ""}
|
| 584 |
-
<i style="left:${100 * t.meter.from}%;width:${100 * (t.meter.to - t.meter.from)}%;background:${t.meter.color}"></i>
|
| 585 |
-
</div>
|
| 586 |
-
</div>`).join("");
|
| 587 |
-
}
|
| 588 |
-
|
| 589 |
-
/* ── move table ────────────────────────────────────────── */
|
| 590 |
-
function paintMoves(p) {
|
| 591 |
-
const hasCmp = p.policy.moves.some((m) => m.compare_prob != null);
|
| 592 |
-
const rows = p.policy.moves.slice(0, 18);
|
| 593 |
-
const head = `<thead><tr><th>#</th><th>Move</th><th style="width:34%">Policy</th><th>p</th>
|
| 594 |
-
<th>Child value</th><th>Value gain</th>${hasCmp ? "<th>Other model</th>" : ""}<th></th></tr></thead>`;
|
| 595 |
-
const body = rows.map((m, i) => {
|
| 596 |
-
const cv = m.child_value;
|
| 597 |
-
const tension = cv == null ? null : cv - p.policy.moves[0].child_value;
|
| 598 |
-
return `<tr data-i="${i}" class="${p.played_san && m.san === p.played_san ? "played" : ""}">
|
| 599 |
-
<td class="num" style="color:var(--ink-3)">${i + 1}</td>
|
| 600 |
-
<td class="san">${m.san}</td>
|
| 601 |
-
<td><div class="bar"><i style="width:${Math.max(1.5, 100 * m.prob)}%"></i></div></td>
|
| 602 |
-
<td class="num">${(100 * m.prob).toFixed(1)}</td>
|
| 603 |
-
<td class="num"><span class="chip" style="color:${cv == null ? "var(--ink-3)" : cv >= 0 ? "var(--pos)" : "var(--neg)"}">${cv == null ? "—" : fmt(cv, 3)}</span></td>
|
| 604 |
-
<td class="num" style="color:${tension > 0.05 ? "var(--pos)" : "var(--ink-3)"}">${tension == null ? "" : (tension > 0 ? "+" : "") + tension.toFixed(3)}</td>
|
| 605 |
-
${hasCmp ? `<td class="num" style="color:var(--ink-3)">${m.compare_prob == null ? "—" : (100 * m.compare_prob).toFixed(1)}</td>` : ""}
|
| 606 |
-
<td class="num" style="color:var(--ink-3)">${sqName(m.from)}→${sqName(m.to)}</td>
|
| 607 |
-
</tr>`;
|
| 608 |
-
}).join("");
|
| 609 |
-
const t = document.getElementById("moves");
|
| 610 |
-
t.innerHTML = head + "<tbody>" + body + "</tbody>";
|
| 611 |
-
t.querySelectorAll("tbody tr").forEach((tr) => {
|
| 612 |
-
tr.addEventListener("mouseenter", () => { state.hoverMove = +tr.dataset.i; paintArrows(p); });
|
| 613 |
-
tr.addEventListener("mouseleave", () => { state.hoverMove = null; paintArrows(p); });
|
| 614 |
-
tr.addEventListener("click", () => { state.square = +p.policy.moves[+tr.dataset.i].from; render(); });
|
| 615 |
-
});
|
| 616 |
-
document.getElementById("moves-note").textContent =
|
| 617 |
-
`${p.policy.n_moves} legal · ${pct(p.policy.legal_mass)} of the raw 64×73 softmax is legal` +
|
| 618 |
-
(rows.length < p.policy.n_moves ? ` · top ${rows.length} shown` : "");
|
| 619 |
-
}
|
| 620 |
-
|
| 621 |
-
/* ── depth chart ───────────────────────────────────────── */
|
| 622 |
-
function paintDepth(p) {
|
| 623 |
-
const d = p.depth;
|
| 624 |
-
if (!d) return;
|
| 625 |
-
const W = 720, Ht = 260, m = { l: 44, r: 16, t: 16, b: 54 };
|
| 626 |
-
const iw = W - m.l - m.r, ih = Ht - m.t - m.b;
|
| 627 |
-
const vals = d.value_by_layer;
|
| 628 |
-
const lim = Math.max(0.15, ...vals.map((v) => Math.abs(v)), Math.abs(p.value)) * 1.15;
|
| 629 |
-
const x = (i) => m.l + (iw * i) / Math.max(1, vals.length - 1);
|
| 630 |
-
const y = (v) => m.t + ih * (1 - (v + lim) / (2 * lim));
|
| 631 |
-
|
| 632 |
-
let g = `<g class="grid">`;
|
| 633 |
-
[-lim, -lim / 2, 0, lim / 2, lim].forEach((v) => { g += `<line x1="${m.l}" y1="${y(v).toFixed(1)}" x2="${W - m.r}" y2="${y(v).toFixed(1)}"/>`; });
|
| 634 |
-
g += `</g><line class="zero" x1="${m.l}" y1="${y(0)}" x2="${W - m.r}" y2="${y(0)}"/>`;
|
| 635 |
-
g += `<g class="axis">` + [-lim, 0, lim].map((v) => `<text x="${m.l - 8}" y="${y(v) + 3}" text-anchor="end">${v.toFixed(2)}</text>`).join("") + `</g>`;
|
| 636 |
-
|
| 637 |
-
const path = vals.map((v, i) => `${i ? "L" : "M"} ${x(i).toFixed(1)} ${y(v).toFixed(1)}`).join(" ");
|
| 638 |
-
const area = `${path} L ${x(vals.length - 1).toFixed(1)} ${y(0)} L ${m.l} ${y(0)} Z`;
|
| 639 |
-
g += `<path d="${area}" fill="${css("--pos")}" opacity=".12"/>`;
|
| 640 |
-
g += `<path class="vline" d="${path}" stroke="${css("--pos")}"/>`;
|
| 641 |
-
g += `<circle cx="${x(vals.length - 1)}" cy="${y(vals[vals.length - 1])}" r="4.5" fill="${css("--pos")}"/>`;
|
| 642 |
-
g += `<text x="${x(vals.length - 1) - 8}" y="${y(vals[vals.length - 1]) - 10}" text-anchor="end" font-family="JetBrains Mono, monospace" font-size="11" fill="${css("--ink")}">${fmt(p.value, 3)}</text>`;
|
| 643 |
-
|
| 644 |
-
// Agreement strip: does the logit lens at this layer already pick the final move?
|
| 645 |
-
const sy = m.t + ih + 16;
|
| 646 |
-
d.top_by_layer.forEach((t, i) => {
|
| 647 |
-
const w = Math.max(6, iw / vals.length - 3);
|
| 648 |
-
g += `<rect x="${(x(i) - w / 2).toFixed(1)}" y="${sy}" width="${w.toFixed(1)}" height="10" rx="3"
|
| 649 |
-
fill="${t.agrees ? css("--accent") : css("--line-strong")}" opacity="${t.agrees ? 0.9 : 0.6}"/>`;
|
| 650 |
-
g += `<rect class="hitbox" data-i="${i}" x="${(x(i) - w / 2 - 2).toFixed(1)}" y="${m.t}" width="${(w + 4).toFixed(1)}" height="${ih + 30}" fill="transparent"/>`;
|
| 651 |
-
});
|
| 652 |
-
g += `<g class="axis">` + vals.map((_, i) =>
|
| 653 |
-
(i % 2 === 0 || i === vals.length - 1) ? `<text x="${x(i)}" y="${sy + 26}" text-anchor="middle">${i + 1}</text>` : "").join("") + `</g>`;
|
| 654 |
-
g += `<text class="axis" x="${m.l}" y="${Ht - 4}" font-family="JetBrains Mono, monospace" font-size="9" fill="${css("--ink-3")}">layer → · value read from each layer (line), lens already picks the final move (gold)</text>`;
|
| 655 |
-
|
| 656 |
-
const svg = document.getElementById("depthchart");
|
| 657 |
-
svg.innerHTML = g;
|
| 658 |
-
svg.querySelectorAll(".hitbox").forEach((r) => {
|
| 659 |
-
r.addEventListener("mousemove", (e) => {
|
| 660 |
-
const i = +r.dataset.i, t = d.top_by_layer[i];
|
| 661 |
-
showTip(e, `layer ${i + 1}<br>value ${fmt(vals[i], 3)}<br>lens top: ${t.san} ${pct(t.prob)}<br>final move here: ${pct(t.final_move_prob)}<br>write norm ${d.residual_write_norm[i].toFixed(1)}`);
|
| 662 |
-
});
|
| 663 |
-
r.addEventListener("mouseleave", hideTip);
|
| 664 |
-
});
|
| 665 |
-
|
| 666 |
-
const firstAgree = d.top_by_layer.findIndex((t, i) => d.top_by_layer.slice(i).every((u) => u.agrees));
|
| 667 |
-
document.getElementById("depth-note").textContent =
|
| 668 |
-
firstAgree >= 0 ? `final move locked in from layer ${firstAgree + 1} of ${L}` : "the final move is never stable across layers";
|
| 669 |
-
}
|
| 670 |
-
|
| 671 |
-
const tipEl = document.getElementById("tip");
|
| 672 |
-
function showTip(e, html) {
|
| 673 |
-
tipEl.innerHTML = html; tipEl.classList.add("on");
|
| 674 |
-
const r = tipEl.getBoundingClientRect();
|
| 675 |
-
tipEl.style.left = Math.min(window.innerWidth - r.width - 10, e.clientX + 14) + "px";
|
| 676 |
-
tipEl.style.top = Math.max(8, e.clientY - r.height - 12) + "px";
|
| 677 |
-
}
|
| 678 |
-
function hideTip() { tipEl.classList.remove("on"); }
|
| 679 |
-
|
| 680 |
-
/* ── attention explorer ────────────────────────────────── */
|
| 681 |
-
function paintHeatmap(rec) {
|
| 682 |
-
const cv = document.getElementById("heatmap");
|
| 683 |
-
const ctx = cv.getContext("2d");
|
| 684 |
-
if (!rec) { ctx.clearRect(0, 0, T, T); return; }
|
| 685 |
-
const img = ctx.createImageData(T, T);
|
| 686 |
-
const get = (i, j) => (state.head < 0 ? headMeanAt(rec, state.layer, i, j) : attnAt(rec, state.layer, state.head, i, j));
|
| 687 |
-
let max = 0;
|
| 688 |
-
const buf = new Float64Array(T * T);
|
| 689 |
-
for (let i = 0; i < T; i++) for (let j = 0; j < T; j++) { const v = get(i, j); buf[i * T + j] = v; if (v > max) max = v; }
|
| 690 |
-
for (let i = 0; i < T; i++) for (let j = 0; j < T; j++) {
|
| 691 |
-
const c = seqColor(buf[i * T + j] / (max || 1));
|
| 692 |
-
const o = (i * T + j) * 4;
|
| 693 |
-
img.data[o] = c[0]; img.data[o + 1] = c[1]; img.data[o + 2] = c[2]; img.data[o + 3] = 255;
|
| 694 |
-
}
|
| 695 |
-
ctx.putImageData(img, 0, 0);
|
| 696 |
-
document.getElementById("attn-max").textContent = max.toFixed(3);
|
| 697 |
-
document.getElementById("attn-ramp").style.background =
|
| 698 |
-
`linear-gradient(90deg,${[0, .25, .5, .75, 1].map((t) => rgb(seqColor(t))).join(",")})`;
|
| 699 |
-
}
|
| 700 |
-
|
| 701 |
-
function paintHeads(rec) {
|
| 702 |
-
const wrap = document.getElementById("heads");
|
| 703 |
-
wrap.innerHTML = "";
|
| 704 |
-
document.getElementById("hl-layer").textContent = state.layer + 1;
|
| 705 |
-
if (!rec) return;
|
| 706 |
-
const opts = [{ h: -1, name: "mean" }].concat(Array.from({ length: H }, (_, h) => ({ h, name: "h" + h })));
|
| 707 |
-
opts.forEach(({ h, name }) => {
|
| 708 |
-
const fig = document.createElement("figure");
|
| 709 |
-
fig.className = "headcell"; fig.style.margin = "0";
|
| 710 |
-
fig.setAttribute("aria-pressed", state.head === h);
|
| 711 |
-
const cv = document.createElement("canvas");
|
| 712 |
-
cv.width = T; cv.height = T;
|
| 713 |
-
const ctx = cv.getContext("2d");
|
| 714 |
-
const img = ctx.createImageData(T, T);
|
| 715 |
-
let max = 0; const buf = new Float64Array(T * T);
|
| 716 |
-
for (let i = 0; i < T; i++) for (let j = 0; j < T; j++) {
|
| 717 |
-
const v = h < 0 ? headMeanAt(rec, state.layer, i, j) : attnAt(rec, state.layer, h, i, j);
|
| 718 |
-
buf[i * T + j] = v; if (v > max) max = v;
|
| 719 |
-
}
|
| 720 |
-
for (let i = 0; i < T * T; i++) {
|
| 721 |
-
const c = seqColor(buf[i] / (max || 1)), o = i * 4;
|
| 722 |
-
img.data[o] = c[0]; img.data[o + 1] = c[1]; img.data[o + 2] = c[2]; img.data[o + 3] = 255;
|
| 723 |
-
}
|
| 724 |
-
ctx.putImageData(img, 0, 0);
|
| 725 |
-
fig.appendChild(cv);
|
| 726 |
-
const cap = document.createElement("figcaption");
|
| 727 |
-
cap.textContent = name;
|
| 728 |
-
fig.appendChild(cap);
|
| 729 |
-
fig.addEventListener("click", () => { state.head = h; render(); });
|
| 730 |
-
wrap.appendChild(fig);
|
| 731 |
-
});
|
| 732 |
-
}
|
| 733 |
-
|
| 734 |
-
function paintBias() {
|
| 735 |
-
const row = document.getElementById("biasrow");
|
| 736 |
-
const names = META.relation_names || [];
|
| 737 |
-
const bias = META.relation_bias;
|
| 738 |
-
document.getElementById("hb-head").textContent = state.head < 0 ? "mean" : state.head;
|
| 739 |
-
if (!bias) { row.innerHTML = ""; return; }
|
| 740 |
-
const vals = state.head < 0
|
| 741 |
-
? names.map((_, k) => bias[state.layer].reduce((s, h) => s + h[k], 0) / H)
|
| 742 |
-
: bias[state.layer][state.head];
|
| 743 |
-
const max = Math.max(...vals.map(Math.abs), 1e-6);
|
| 744 |
-
row.innerHTML = vals.map((v, k) => `
|
| 745 |
-
<div title="${names[k]}: ${fmt(v, 3)}">
|
| 746 |
-
<div class="cell" style="background:${rgb(divColor(v / max), Math.min(0.9, 0.15 + Math.abs(v / max)))}"></div>
|
| 747 |
-
${names[k]}<br><span style="color:var(--ink-2)">${v.toFixed(2)}</span>
|
| 748 |
-
</div>`).join("");
|
| 749 |
-
}
|
| 750 |
-
|
| 751 |
-
/* ── position strip, header ────────────────────────────── */
|
| 752 |
-
function paintStrip() {
|
| 753 |
-
const strip = document.getElementById("strip");
|
| 754 |
-
if (POS.length < 2) { strip.style.display = "none"; return; }
|
| 755 |
-
strip.innerHTML = POS.map((p, i) => {
|
| 756 |
-
const label = p.ply != null ? `${Math.floor(p.ply / 2) + 1}${p.ply % 2 === 0 ? "." : "…"} ${p.played_san || "—"}` : `#${i + 1}`;
|
| 757 |
-
return `<button class="ply" data-i="${i}" aria-current="${i === state.pos}">${label}</button>`;
|
| 758 |
-
}).join("");
|
| 759 |
-
strip.querySelectorAll(".ply").forEach((b) =>
|
| 760 |
-
b.addEventListener("click", () => { state.pos = +b.dataset.i; state.square = null; render(); }));
|
| 761 |
-
}
|
| 762 |
-
|
| 763 |
-
function paintHeader() {
|
| 764 |
-
document.getElementById("title").textContent = META.title || "Glass Box";
|
| 765 |
-
const bits = [
|
| 766 |
-
["model", (META.model_dir || "").split("/").pop() + (META.ema ? " (EMA)" : "")],
|
| 767 |
-
["shape", `${META.n_layers}L · ${META.n_heads}H · d${META.config ? META.config.embed_dim : "?"}`],
|
| 768 |
-
META.compare_dir ? ["vs", META.compare_dir.split("/").pop()] : null,
|
| 769 |
-
["attn check", "Δ " + Number(META.attention_check).toExponential(1)],
|
| 770 |
-
["generated", META.generated],
|
| 771 |
-
].filter(Boolean);
|
| 772 |
-
document.getElementById("meta").innerHTML = bits.map(([k, v]) => `<span>${k} <b>${v}</b></span>`).join("");
|
| 773 |
-
document.getElementById("repro").textContent =
|
| 774 |
-
`python scripts/introspect_model.py --model-dir ${META.model_dir} --fen "<fen>" --out out/think.html`;
|
| 775 |
-
}
|
| 776 |
-
|
| 777 |
-
/* ── render ────────────────────────────────────────────── */
|
| 778 |
-
let renderToken = 0;
|
| 779 |
-
async function render() {
|
| 780 |
-
const p = POS[state.pos];
|
| 781 |
-
const my = ++renderToken;
|
| 782 |
-
paintStrip();
|
| 783 |
-
paintTiles(p);
|
| 784 |
-
paintMoves(p);
|
| 785 |
-
paintDepth(p);
|
| 786 |
-
paintBias();
|
| 787 |
-
document.getElementById("fen").textContent = p.fen;
|
| 788 |
-
document.getElementById("layer-val").textContent = `${state.layer + 1}/${L}`;
|
| 789 |
-
document.getElementById("overlay-hint").textContent =
|
| 790 |
-
(MODES.find((m) => m.id === state.mode) || {}).hint +
|
| 791 |
-
(state.square !== null ? ` · selected: ${sqName(state.square)}` : "");
|
| 792 |
-
modesEl.querySelectorAll(".mode").forEach((b) => b.setAttribute("aria-pressed", b.dataset.mode === state.mode));
|
| 793 |
-
document.getElementById("head").value = String(state.head);
|
| 794 |
-
document.getElementById("attnmode").value = state.attnMode;
|
| 795 |
-
|
| 796 |
-
let rec = null;
|
| 797 |
-
if (p.attention) {
|
| 798 |
-
const status = document.getElementById("attn-status");
|
| 799 |
-
if (!attnCache.has(state.pos)) status.textContent = "decoding…";
|
| 800 |
-
try { rec = await getAttn(state.pos); status.textContent = ""; }
|
| 801 |
-
catch (e) { status.textContent = "attention unavailable: " + e.message; }
|
| 802 |
-
} else {
|
| 803 |
-
document.getElementById("attn-panel").style.display = "none";
|
| 804 |
-
}
|
| 805 |
-
if (my !== renderToken) return;
|
| 806 |
-
paintBoard(p, rec);
|
| 807 |
-
paintArrows(p);
|
| 808 |
-
paintHeatmap(rec);
|
| 809 |
-
paintHeads(rec);
|
| 810 |
-
}
|
| 811 |
-
|
| 812 |
-
/* ── controls ──────────────────────────────────────────── */
|
| 813 |
-
const layerEl = document.getElementById("layer");
|
| 814 |
-
layerEl.max = String(L - 1); layerEl.value = String(state.layer);
|
| 815 |
-
layerEl.addEventListener("input", () => { state.layer = +layerEl.value; render(); });
|
| 816 |
-
|
| 817 |
-
const headEl = document.getElementById("head");
|
| 818 |
-
headEl.innerHTML = `<option value="-1">mean of ${H}</option>` +
|
| 819 |
-
Array.from({ length: H }, (_, h) => `<option value="${h}">head ${h}</option>`).join("");
|
| 820 |
-
headEl.addEventListener("change", () => { state.head = +headEl.value; render(); });
|
| 821 |
-
|
| 822 |
-
document.getElementById("attnmode").addEventListener("change", (e) => { state.attnMode = e.target.value; render(); });
|
| 823 |
-
|
| 824 |
-
document.getElementById("heatmap").addEventListener("click", async (e) => {
|
| 825 |
-
const r = e.target.getBoundingClientRect();
|
| 826 |
-
const j = Math.floor(((e.clientX - r.left) / r.width) * T);
|
| 827 |
-
const i = Math.floor(((e.clientY - r.top) / r.height) * T);
|
| 828 |
-
state.square = state.attnMode === "row" ? i : j;
|
| 829 |
-
if (state.square > 63) state.square = null;
|
| 830 |
-
state.mode = "attention";
|
| 831 |
-
render();
|
| 832 |
-
});
|
| 833 |
-
document.getElementById("heatmap").addEventListener("mousemove", async (e) => {
|
| 834 |
-
const rec = attnCache.get(state.pos);
|
| 835 |
-
if (!rec) return;
|
| 836 |
-
const r = e.target.getBoundingClientRect();
|
| 837 |
-
const j = Math.floor(((e.clientX - r.left) / r.width) * T);
|
| 838 |
-
const i = Math.floor(((e.clientY - r.top) / r.height) * T);
|
| 839 |
-
if (i < 0 || j < 0 || i >= T || j >= T) return hideTip();
|
| 840 |
-
const v = state.head < 0 ? headMeanAt(rec, state.layer, i, j) : attnAt(rec, state.layer, state.head, i, j);
|
| 841 |
-
showTip(e, `${sqName(i)} → ${sqName(j)}<br>${v.toFixed(4)}`);
|
| 842 |
-
});
|
| 843 |
-
document.getElementById("heatmap").addEventListener("mouseleave", hideTip);
|
| 844 |
-
|
| 845 |
-
window.addEventListener("keydown", (e) => {
|
| 846 |
-
if (e.target.tagName === "SELECT" || e.target.tagName === "INPUT") return;
|
| 847 |
-
const k = e.key;
|
| 848 |
-
if (k === "ArrowRight" && state.pos < POS.length - 1) { state.pos++; state.square = null; render(); }
|
| 849 |
-
else if (k === "ArrowLeft" && state.pos > 0) { state.pos--; state.square = null; render(); }
|
| 850 |
-
else if (k === "ArrowUp" && state.layer < L - 1) { state.layer++; layerEl.value = state.layer; render(); }
|
| 851 |
-
else if (k === "ArrowDown" && state.layer > 0) { state.layer--; layerEl.value = state.layer; render(); }
|
| 852 |
-
else if (k === "]" && state.head < H - 1) { state.head++; render(); }
|
| 853 |
-
else if (k === "[" && state.head >= 0) { state.head--; render(); }
|
| 854 |
-
else return;
|
| 855 |
-
e.preventDefault();
|
| 856 |
-
});
|
| 857 |
-
|
| 858 |
-
paintHeader();
|
| 859 |
-
render();
|
| 860 |
-
</script>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
chesstransformer/models/__init__.py
DELETED
|
@@ -1 +0,0 @@
|
|
| 1 |
-
"""This module contains all the models."""
|
|
|
|
|
|
chesstransformer/models/tokenizer/__init__.py
DELETED
|
@@ -1,93 +0,0 @@
|
|
| 1 |
-
from tokenizers import Tokenizer
|
| 2 |
-
from tokenizers import models, pre_tokenizers, trainers, processors, normalizers
|
| 3 |
-
from .move_tokenizer import MoveTokenizer
|
| 4 |
-
from .position_tokenizer import PostionTokenizer
|
| 5 |
-
|
| 6 |
-
|
| 7 |
-
def create_bpe_tokenizer(
|
| 8 |
-
vocab_size: int = 2000,
|
| 9 |
-
) -> tuple[Tokenizer, trainers.BpeTrainer]:
|
| 10 |
-
"""
|
| 11 |
-
Create a BPE tokenizer for chess moves.
|
| 12 |
-
|
| 13 |
-
The tokenizer uses Byte-Pair Encoding (BPE) with ByteLevel pre-tokenization
|
| 14 |
-
to handle the unique structure of chess moves. The <STEP> token is used to
|
| 15 |
-
explicitly separate each move, ensuring clear move boundaries.
|
| 16 |
-
|
| 17 |
-
Special tokens included:
|
| 18 |
-
- <PAD>: Padding token
|
| 19 |
-
- <START>: Start of sequence token
|
| 20 |
-
- <END>: End of sequence token
|
| 21 |
-
- <STEP>: Move separator token
|
| 22 |
-
- <1-0>: White wins
|
| 23 |
-
- <0-1>: Black wins
|
| 24 |
-
- <1/2-1/2>: Draw
|
| 25 |
-
- <UNK>: Unknown token
|
| 26 |
-
|
| 27 |
-
Args:
|
| 28 |
-
vocab_size: Size of the vocabulary (default: 2000)
|
| 29 |
-
|
| 30 |
-
Returns:
|
| 31 |
-
A Tokenizer object and trainer
|
| 32 |
-
"""
|
| 33 |
-
|
| 34 |
-
# Initialize a tokenizer with BPE model
|
| 35 |
-
encoder = Tokenizer(models.BPE(unk_token="<UNK>"))
|
| 36 |
-
encoder.normalizer = normalizers.Sequence([]) # No normalization needed for chess moves
|
| 37 |
-
|
| 38 |
-
# remove the white space pre-tokenizer
|
| 39 |
-
encoder.pre_tokenizer = pre_tokenizers.Whitespace()
|
| 40 |
-
|
| 41 |
-
# Setup trainer with special tokens, including space
|
| 42 |
-
trainer = trainers.BpeTrainer(
|
| 43 |
-
vocab_size=vocab_size,
|
| 44 |
-
special_tokens=[
|
| 45 |
-
"<PAD>",
|
| 46 |
-
"<START>",
|
| 47 |
-
"<END>",
|
| 48 |
-
"<1-0>",
|
| 49 |
-
"<0-1>",
|
| 50 |
-
"<1/2-1/2>",
|
| 51 |
-
"<UNK>",
|
| 52 |
-
"<STEP>",
|
| 53 |
-
],
|
| 54 |
-
show_progress=True,
|
| 55 |
-
min_frequency=2,
|
| 56 |
-
initial_alphabet=[
|
| 57 |
-
"a",
|
| 58 |
-
"b",
|
| 59 |
-
"c",
|
| 60 |
-
"d",
|
| 61 |
-
"e",
|
| 62 |
-
"f",
|
| 63 |
-
"g",
|
| 64 |
-
"h",
|
| 65 |
-
"1",
|
| 66 |
-
"2",
|
| 67 |
-
"3",
|
| 68 |
-
"4",
|
| 69 |
-
"5",
|
| 70 |
-
"6",
|
| 71 |
-
"7",
|
| 72 |
-
"8",
|
| 73 |
-
"q",
|
| 74 |
-
"k",
|
| 75 |
-
"r",
|
| 76 |
-
"b",
|
| 77 |
-
"n", # Piece notations
|
| 78 |
-
],
|
| 79 |
-
)
|
| 80 |
-
|
| 81 |
-
# Add post-processing to add special tokens
|
| 82 |
-
encoder.post_processor = processors.TemplateProcessing(
|
| 83 |
-
single="<START> $A <END>",
|
| 84 |
-
special_tokens=[
|
| 85 |
-
("<START>", 1),
|
| 86 |
-
("<END>", 2),
|
| 87 |
-
],
|
| 88 |
-
)
|
| 89 |
-
|
| 90 |
-
return encoder, trainer
|
| 91 |
-
|
| 92 |
-
|
| 93 |
-
__all__ = ["create_bpe_tokenizer", "MoveTokenizer", "PostionTokenizer"]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
chesstransformer/models/tokenizer/alphazero_move_encoder.py
DELETED
|
@@ -1,127 +0,0 @@
|
|
| 1 |
-
"""AlphaZero-style action encoding for chess moves.
|
| 2 |
-
|
| 3 |
-
Encodes moves as (from_square, action_plane) pairs using the 8×8×73
|
| 4 |
-
representation from the AlphaZero paper (Silver et al., 2018).
|
| 5 |
-
|
| 6 |
-
The 73 action planes decompose as:
|
| 7 |
-
0–55: Queen-type moves — 8 directions × 7 distances
|
| 8 |
-
56–63: Knight moves — 8 L-shaped jumps
|
| 9 |
-
64–72: Underpromotions — 3 directions × 3 piece types (knight/bishop/rook)
|
| 10 |
-
|
| 11 |
-
Queen promotions are encoded as regular queen-type moves (distance-1 to the
|
| 12 |
-
last rank). Underpromotions get their own dedicated planes.
|
| 13 |
-
"""
|
| 14 |
-
|
| 15 |
-
import chess
|
| 16 |
-
import torch
|
| 17 |
-
|
| 18 |
-
NUM_ACTION_PLANES = 73
|
| 19 |
-
|
| 20 |
-
# Direction vectors (file_delta, rank_delta) for queen-type moves
|
| 21 |
-
QUEEN_DIRECTIONS = [
|
| 22 |
-
(0, 1), # N
|
| 23 |
-
(1, 1), # NE
|
| 24 |
-
(1, 0), # E
|
| 25 |
-
(1, -1), # SE
|
| 26 |
-
(0, -1), # S
|
| 27 |
-
(-1, -1), # SW
|
| 28 |
-
(-1, 0), # W
|
| 29 |
-
(-1, 1), # NW
|
| 30 |
-
]
|
| 31 |
-
|
| 32 |
-
# Knight jump offsets (file_delta, rank_delta)
|
| 33 |
-
KNIGHT_JUMPS = [
|
| 34 |
-
(1, 2), (2, 1), (2, -1), (1, -2),
|
| 35 |
-
(-1, -2), (-2, -1), (-2, 1), (-1, 2),
|
| 36 |
-
]
|
| 37 |
-
|
| 38 |
-
# Underpromotion piece types (queen promotion is a regular queen-type move)
|
| 39 |
-
UNDERPROMO_PIECES = [chess.KNIGHT, chess.BISHOP, chess.ROOK]
|
| 40 |
-
|
| 41 |
-
# Pre-compute knight jump lookup for O(1) detection
|
| 42 |
-
_KNIGHT_JUMP_INDEX = {(df, dr): i for i, (df, dr) in enumerate(KNIGHT_JUMPS)}
|
| 43 |
-
|
| 44 |
-
|
| 45 |
-
def move_to_action_plane(from_sq: int, to_sq: int, promotion: int | None = None) -> int:
|
| 46 |
-
"""Convert a chess move to its action plane index (0–72).
|
| 47 |
-
|
| 48 |
-
Args:
|
| 49 |
-
from_sq: Source square (0–63, python-chess ordering).
|
| 50 |
-
to_sq: Destination square (0–63).
|
| 51 |
-
promotion: Promotion piece type (chess.QUEEN, chess.KNIGHT, etc.) or None.
|
| 52 |
-
|
| 53 |
-
Returns:
|
| 54 |
-
Action plane index in [0, 72].
|
| 55 |
-
"""
|
| 56 |
-
df = (to_sq % 8) - (from_sq % 8)
|
| 57 |
-
dr = (to_sq // 8) - (from_sq // 8)
|
| 58 |
-
|
| 59 |
-
# ── Underpromotion ───────────────────────────────────────────────────
|
| 60 |
-
if promotion is not None and promotion != chess.QUEEN:
|
| 61 |
-
# Direction: file delta ∈ {-1, 0, +1} → index 0, 1, 2
|
| 62 |
-
dir_idx = df + 1
|
| 63 |
-
piece_idx = UNDERPROMO_PIECES.index(promotion)
|
| 64 |
-
return 64 + dir_idx * 3 + piece_idx
|
| 65 |
-
|
| 66 |
-
# ── Knight move ──────────────────────────────────────────────────────
|
| 67 |
-
knight_idx = _KNIGHT_JUMP_INDEX.get((df, dr))
|
| 68 |
-
if knight_idx is not None:
|
| 69 |
-
return 56 + knight_idx
|
| 70 |
-
|
| 71 |
-
# ── Queen-type move (includes queen promotions) ──────────────────────
|
| 72 |
-
distance = max(abs(df), abs(dr))
|
| 73 |
-
dir_f = (df // distance) if df != 0 else 0
|
| 74 |
-
dir_r = (dr // distance) if dr != 0 else 0
|
| 75 |
-
dir_idx = QUEEN_DIRECTIONS.index((dir_f, dir_r))
|
| 76 |
-
return dir_idx * 7 + (distance - 1)
|
| 77 |
-
|
| 78 |
-
|
| 79 |
-
def action_to_move(from_sq: int, plane: int) -> tuple[int, int | None]:
|
| 80 |
-
"""Decode an action plane back to (to_square, promotion_piece | None).
|
| 81 |
-
|
| 82 |
-
For queen-type moves landing on promotion ranks, the caller must check
|
| 83 |
-
whether the source piece is a pawn and add ``chess.QUEEN`` promotion.
|
| 84 |
-
|
| 85 |
-
Args:
|
| 86 |
-
from_sq: Source square (0–63).
|
| 87 |
-
plane: Action plane index (0–72).
|
| 88 |
-
|
| 89 |
-
Returns:
|
| 90 |
-
(to_sq, promotion_piece) — promotion_piece is set only for underpromotions.
|
| 91 |
-
"""
|
| 92 |
-
from_file = from_sq % 8
|
| 93 |
-
from_rank = from_sq // 8
|
| 94 |
-
|
| 95 |
-
if plane < 56:
|
| 96 |
-
# Queen-type move
|
| 97 |
-
dir_idx = plane // 7
|
| 98 |
-
distance = plane % 7 + 1
|
| 99 |
-
df, dr = QUEEN_DIRECTIONS[dir_idx]
|
| 100 |
-
to_file = from_file + df * distance
|
| 101 |
-
to_rank = from_rank + dr * distance
|
| 102 |
-
return to_rank * 8 + to_file, None
|
| 103 |
-
|
| 104 |
-
if plane < 64:
|
| 105 |
-
# Knight move
|
| 106 |
-
df, dr = KNIGHT_JUMPS[plane - 56]
|
| 107 |
-
to_file = from_file + df
|
| 108 |
-
to_rank = from_rank + dr
|
| 109 |
-
return to_rank * 8 + to_file, None
|
| 110 |
-
|
| 111 |
-
# Underpromotion
|
| 112 |
-
promo_idx = plane - 64
|
| 113 |
-
dir_idx = promo_idx // 3
|
| 114 |
-
piece_idx = promo_idx % 3
|
| 115 |
-
df = dir_idx - 1 # 0→-1, 1→0, 2→+1
|
| 116 |
-
to_file = from_file + df
|
| 117 |
-
to_rank = 7 if from_rank == 6 else 0 # white promotes to rank 8, black to rank 1
|
| 118 |
-
return to_rank * 8 + to_file, UNDERPROMO_PIECES[piece_idx]
|
| 119 |
-
|
| 120 |
-
|
| 121 |
-
def build_legal_moves_planes(board: chess.Board) -> torch.Tensor:
|
| 122 |
-
"""Build a (64, 73) boolean mask of legal moves in action-plane format."""
|
| 123 |
-
mask = torch.zeros(64, NUM_ACTION_PLANES, dtype=torch.bool)
|
| 124 |
-
for move in board.legal_moves:
|
| 125 |
-
plane = move_to_action_plane(move.from_square, move.to_square, move.promotion)
|
| 126 |
-
mask[move.from_square, plane] = True
|
| 127 |
-
return mask
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
chesstransformer/models/tokenizer/move_tokenizer.py
DELETED
|
@@ -1,38 +0,0 @@
|
|
| 1 |
-
import json
|
| 2 |
-
from pathlib import Path
|
| 3 |
-
|
| 4 |
-
DATA_DIR = Path(__file__).parents[2] / "data"
|
| 5 |
-
|
| 6 |
-
|
| 7 |
-
class MoveTokenizer:
|
| 8 |
-
def __init__(self, vocab_path=DATA_DIR / "tokenizer_models/chess_moves_vocab.json"):
|
| 9 |
-
with open(vocab_path, "r") as f:
|
| 10 |
-
self.vocab = json.load(f)
|
| 11 |
-
self.inv_vocab = {v: k for k, v in self.vocab.items()}
|
| 12 |
-
|
| 13 |
-
def encode(self, move: str) -> int:
|
| 14 |
-
"""Encode a chess move string into a token ID.
|
| 15 |
-
Args:
|
| 16 |
-
move: A string representing a chess move in UCI format (e.g., "e2e4", "g1f3").
|
| 17 |
-
Returns:
|
| 18 |
-
An integer token ID representing the move.
|
| 19 |
-
"""
|
| 20 |
-
if move not in self.vocab:
|
| 21 |
-
raise ValueError(f"Move '{move}' not in vocabulary.")
|
| 22 |
-
return self.vocab[move]
|
| 23 |
-
|
| 24 |
-
def decode(self, token_id: int) -> str:
|
| 25 |
-
"""Decode a token ID back into a chess move string.
|
| 26 |
-
Args:
|
| 27 |
-
token_id: An integer token ID representing the move.
|
| 28 |
-
Returns:
|
| 29 |
-
A string representing a chess move in UCI format.
|
| 30 |
-
"""
|
| 31 |
-
if token_id not in self.inv_vocab:
|
| 32 |
-
raise ValueError(f"Token ID '{token_id}' not in vocabulary.")
|
| 33 |
-
return self.inv_vocab[token_id]
|
| 34 |
-
|
| 35 |
-
@property
|
| 36 |
-
def vocab_size(self) -> int:
|
| 37 |
-
"""Return the size of the vocabulary."""
|
| 38 |
-
return len(self.vocab)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
chesstransformer/models/tokenizer/position_tokenizer.py
DELETED
|
@@ -1,88 +0,0 @@
|
|
| 1 |
-
import chess
|
| 2 |
-
|
| 3 |
-
_PIECE_TOKEN = {
|
| 4 |
-
(chess.PAWN, chess.WHITE): 1, (chess.KNIGHT, chess.WHITE): 2,
|
| 5 |
-
(chess.BISHOP, chess.WHITE): 3, (chess.ROOK, chess.WHITE): 4,
|
| 6 |
-
(chess.QUEEN, chess.WHITE): 5, (chess.KING, chess.WHITE): 6,
|
| 7 |
-
(chess.PAWN, chess.BLACK): 7, (chess.KNIGHT, chess.BLACK): 8,
|
| 8 |
-
(chess.BISHOP, chess.BLACK): 9, (chess.ROOK, chess.BLACK): 10,
|
| 9 |
-
(chess.QUEEN, chess.BLACK): 11, (chess.KING, chess.BLACK): 12,
|
| 10 |
-
}
|
| 11 |
-
|
| 12 |
-
|
| 13 |
-
class PostionTokenizer:
|
| 14 |
-
"""A simple tokenizer for chess board positions.
|
| 15 |
-
Each square on the board is represented by a token ID based on the piece occupying it.
|
| 16 |
-
Empty squares are represented by the token ID 0.
|
| 17 |
-
The board is represented as an 8x8 grid, flattened into a list of 64 tokens.
|
| 18 |
-
"""
|
| 19 |
-
|
| 20 |
-
def __init__(self):
|
| 21 |
-
self.vocab = {
|
| 22 |
-
"P": 1,
|
| 23 |
-
"N": 2,
|
| 24 |
-
"B": 3,
|
| 25 |
-
"R": 4,
|
| 26 |
-
"Q": 5,
|
| 27 |
-
"K": 6,
|
| 28 |
-
"p": 7,
|
| 29 |
-
"n": 8,
|
| 30 |
-
"b": 9,
|
| 31 |
-
"r": 10,
|
| 32 |
-
"q": 11,
|
| 33 |
-
"k": 12,
|
| 34 |
-
".": 0,
|
| 35 |
-
}
|
| 36 |
-
self.inv_vocab = {v: k for k, v in self.vocab.items()}
|
| 37 |
-
|
| 38 |
-
def encode(self, board: chess.Board) -> list[int]:
|
| 39 |
-
"""Encode a chess.Board object into a list of token IDs.
|
| 40 |
-
The board is represented as an 8x8 grid, flattened into a list of 64 tokens.
|
| 41 |
-
|
| 42 |
-
Args:
|
| 43 |
-
board: A chess.Board object
|
| 44 |
-
|
| 45 |
-
Returns:
|
| 46 |
-
List of token IDs representing the board position
|
| 47 |
-
"""
|
| 48 |
-
result = [0] * 64
|
| 49 |
-
for sq, piece in board.piece_map().items():
|
| 50 |
-
result[sq] = _PIECE_TOKEN[(piece.piece_type, piece.color)]
|
| 51 |
-
return result
|
| 52 |
-
|
| 53 |
-
def decode(self, token_ids: list[int]) -> chess.Board:
|
| 54 |
-
"""Decode a list of token IDs back into a chess board string.
|
| 55 |
-
Args:
|
| 56 |
-
token_ids: List of token IDs representing the board position
|
| 57 |
-
Returns:
|
| 58 |
-
A chess.Board object
|
| 59 |
-
"""
|
| 60 |
-
chars = [self.inv_vocab[token_id] for token_id in token_ids]
|
| 61 |
-
rows = ["".join(chars[i * 8 : (i + 1) * 8]) for i in range(8)][::-1] # Reverse to get the correct order
|
| 62 |
-
board_str = "\n".join(rows)
|
| 63 |
-
board = self._ascii2board(board_str)
|
| 64 |
-
return board
|
| 65 |
-
|
| 66 |
-
def _ascii2board(self, ascii_board: str) -> chess.Board:
|
| 67 |
-
"""Convert an ASCII board representation back to a chess.Board object.
|
| 68 |
-
Args:
|
| 69 |
-
ascii_board: String representation of the board (8 lines of 8 characters)
|
| 70 |
-
Returns:
|
| 71 |
-
A chess.Board object
|
| 72 |
-
"""
|
| 73 |
-
board = chess.Board.empty()
|
| 74 |
-
rows = ascii_board.split("\n")
|
| 75 |
-
for rank in range(8):
|
| 76 |
-
file = 0
|
| 77 |
-
for char in rows[7 - rank]: # Reverse the order of ranks
|
| 78 |
-
if char in self.vocab and char != ".":
|
| 79 |
-
square = chess.square(file, rank)
|
| 80 |
-
piece = chess.Piece.from_symbol(char)
|
| 81 |
-
board.set_piece_at(square, piece)
|
| 82 |
-
file += 1
|
| 83 |
-
return board
|
| 84 |
-
|
| 85 |
-
@property
|
| 86 |
-
def vocab_size(self) -> int:
|
| 87 |
-
"""Return the size of the vocabulary."""
|
| 88 |
-
return len(self.vocab)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
chesstransformer/models/tokenizer/train_bpe_tokenizer.py
DELETED
|
@@ -1,94 +0,0 @@
|
|
| 1 |
-
from pathlib import Path
|
| 2 |
-
import argparse
|
| 3 |
-
|
| 4 |
-
from chesstransformer.models.tokenizer import create_bpe_tokenizer
|
| 5 |
-
|
| 6 |
-
|
| 7 |
-
def arg_parser():
|
| 8 |
-
parser = argparse.ArgumentParser(description="Train a BPE tokenizer for chess moves.")
|
| 9 |
-
parser.add_argument(
|
| 10 |
-
"--vocab_size",
|
| 11 |
-
type=int,
|
| 12 |
-
default=2000,
|
| 13 |
-
help="Size of the vocabulary (default: 2000)",
|
| 14 |
-
)
|
| 15 |
-
parser.add_argument(
|
| 16 |
-
"--output_dir",
|
| 17 |
-
type=str,
|
| 18 |
-
help="Directory to save the trained tokenizer",
|
| 19 |
-
required=True,
|
| 20 |
-
)
|
| 21 |
-
parser.add_argument(
|
| 22 |
-
"--dataset_path",
|
| 23 |
-
type=str,
|
| 24 |
-
help="Path to the lichess dataset file (PGN format)",
|
| 25 |
-
required=True,
|
| 26 |
-
)
|
| 27 |
-
|
| 28 |
-
parser.add_argument(
|
| 29 |
-
"--num_samples",
|
| 30 |
-
type=int,
|
| 31 |
-
default=100000,
|
| 32 |
-
help="Number of games to sample for training (default: 100000)",
|
| 33 |
-
)
|
| 34 |
-
|
| 35 |
-
parser.add_argument(
|
| 36 |
-
"--notation",
|
| 37 |
-
type=str,
|
| 38 |
-
choices=["uci", "san"],
|
| 39 |
-
default="uci",
|
| 40 |
-
help="Notation format for moves (default: uci)",
|
| 41 |
-
)
|
| 42 |
-
|
| 43 |
-
args = parser.parse_args()
|
| 44 |
-
return args
|
| 45 |
-
|
| 46 |
-
|
| 47 |
-
def main():
|
| 48 |
-
args = arg_parser()
|
| 49 |
-
vocab_size = args.vocab_size
|
| 50 |
-
output_dir = Path(args.output_dir)
|
| 51 |
-
dataset_path = Path(args.dataset_path)
|
| 52 |
-
num_samples = args.num_samples
|
| 53 |
-
notation = args.notation
|
| 54 |
-
|
| 55 |
-
output_dir.mkdir(parents=True, exist_ok=True)
|
| 56 |
-
|
| 57 |
-
print(f"Creating BPE tokenizer with vocab size {vocab_size}...")
|
| 58 |
-
tokenizer, trainer = create_bpe_tokenizer(vocab_size=vocab_size)
|
| 59 |
-
|
| 60 |
-
print(f"Loading dataset from {dataset_path}...")
|
| 61 |
-
from chesstransformer.datasets.lichess import extract_games_optimized
|
| 62 |
-
|
| 63 |
-
games = extract_games_optimized(dataset_path, num_games=num_samples)
|
| 64 |
-
|
| 65 |
-
print(f"Extracting moves in {notation} notation...")
|
| 66 |
-
print("Using <STEP> token to separate moves...")
|
| 67 |
-
if notation == "uci":
|
| 68 |
-
move_sequences = ("<STEP>".join([move.uci() for move in game.mainline_moves()]) for game in games)
|
| 69 |
-
elif notation == "san":
|
| 70 |
-
move_sequences = (
|
| 71 |
-
"<STEP>".join(
|
| 72 |
-
[
|
| 73 |
-
(lambda board, move: (board.san(move), board.push(move))[0])(board, move)
|
| 74 |
-
for board in [game.board()]
|
| 75 |
-
for move in game.mainline_moves()
|
| 76 |
-
]
|
| 77 |
-
)
|
| 78 |
-
for game in games
|
| 79 |
-
)
|
| 80 |
-
else:
|
| 81 |
-
raise ValueError(f"Unsupported notation: {notation}")
|
| 82 |
-
|
| 83 |
-
print(f"Training tokenizer on {num_samples} games...")
|
| 84 |
-
tokenizer.train_from_iterator(move_sequences, trainer=trainer)
|
| 85 |
-
|
| 86 |
-
tokenizer_path = output_dir / f"bpe_tokenizer_vocab{vocab_size}.json"
|
| 87 |
-
print(f"Saving tokenizer to {tokenizer_path}...")
|
| 88 |
-
tokenizer.save(str(tokenizer_path))
|
| 89 |
-
|
| 90 |
-
print("Tokenizer training complete.")
|
| 91 |
-
|
| 92 |
-
|
| 93 |
-
if __name__ == "__main__":
|
| 94 |
-
main()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
chesstransformer/models/transformer/__init__.py
DELETED
|
File without changes
|
chesstransformer/models/transformer/attention.py
DELETED
|
@@ -1,228 +0,0 @@
|
|
| 1 |
-
"""This module implements attention mechanisms for LLMs."""
|
| 2 |
-
|
| 3 |
-
import torch
|
| 4 |
-
import torch.nn as nn
|
| 5 |
-
import torch.nn.functional as F
|
| 6 |
-
|
| 7 |
-
from chesstransformer.models.transformer.rope import RoPEEmbedding
|
| 8 |
-
|
| 9 |
-
|
| 10 |
-
class MultiHeadAttention(nn.Module):
|
| 11 |
-
def __init__(
|
| 12 |
-
self, d_in, d_out, num_heads, context_length, qkv_bias=False, dropout=0.0, mask_future=True, apply_rope=False
|
| 13 |
-
):
|
| 14 |
-
super().__init__()
|
| 15 |
-
assert d_out % num_heads == 0, "d_out must be divisible by num_heads"
|
| 16 |
-
self.num_heads = num_heads
|
| 17 |
-
self.d_out = d_out
|
| 18 |
-
self.d_head = d_out // num_heads
|
| 19 |
-
self.mask_future = mask_future
|
| 20 |
-
self.W_query = nn.Linear(d_in, d_out, bias=qkv_bias)
|
| 21 |
-
self.W_key = nn.Linear(d_in, d_out, bias=qkv_bias)
|
| 22 |
-
self.W_value = nn.Linear(d_in, d_out, bias=qkv_bias)
|
| 23 |
-
self.out_proj = nn.Linear(d_out, d_out)
|
| 24 |
-
self.dropout = nn.Dropout(dropout)
|
| 25 |
-
self.apply_rope = apply_rope
|
| 26 |
-
|
| 27 |
-
if self.apply_rope:
|
| 28 |
-
self.rope = RoPEEmbedding(dim=self.d_head, max_position_embeddings=context_length)
|
| 29 |
-
|
| 30 |
-
self.register_buffer("mask", torch.triu(torch.ones(context_length, context_length), diagonal=1))
|
| 31 |
-
|
| 32 |
-
def forward(self, x):
|
| 33 |
-
"""
|
| 34 |
-
Computes multi-head self-attention.
|
| 35 |
-
|
| 36 |
-
Args:
|
| 37 |
-
x: Input tensor of shape (batch_size, seq_len, d_in)
|
| 38 |
-
|
| 39 |
-
Returns:
|
| 40 |
-
Output tensor of shape (batch_size, seq_len, d_out)
|
| 41 |
-
"""
|
| 42 |
-
bz, nb_tockens, _ = x.shape
|
| 43 |
-
|
| 44 |
-
# Linear projections
|
| 45 |
-
keys = self.W_key(x)
|
| 46 |
-
values = self.W_value(x)
|
| 47 |
-
queries = self.W_query(x)
|
| 48 |
-
|
| 49 |
-
# Reshape for multi-head attention
|
| 50 |
-
keys = keys.view(bz, nb_tockens, self.num_heads, self.d_head).transpose(1, 2)
|
| 51 |
-
values = values.view(bz, nb_tockens, self.num_heads, self.d_head).transpose(1, 2)
|
| 52 |
-
queries = queries.view(bz, nb_tockens, self.num_heads, self.d_head).transpose(1, 2)
|
| 53 |
-
|
| 54 |
-
if self.apply_rope:
|
| 55 |
-
keys = self.rope(keys)
|
| 56 |
-
queries = self.rope(values)
|
| 57 |
-
|
| 58 |
-
# Compute attention scores
|
| 59 |
-
attention_scores = queries @ keys.transpose(2, 3) # (batch_size, num_heads, seq_len, seq_len)
|
| 60 |
-
|
| 61 |
-
if self.mask_future:
|
| 62 |
-
# Apply mask
|
| 63 |
-
mask_bool = self.mask.bool()[:nb_tockens, :nb_tockens]
|
| 64 |
-
attention_scores = attention_scores.masked_fill(mask_bool, -torch.inf)
|
| 65 |
-
|
| 66 |
-
# Compute attention weights
|
| 67 |
-
attention_weights = F.softmax(attention_scores / keys.shape[-1] ** 0.5, dim=-1)
|
| 68 |
-
attention_weights = self.dropout(attention_weights)
|
| 69 |
-
|
| 70 |
-
# apply attention weights to values
|
| 71 |
-
context_vec = (attention_weights @ values).transpose(1, 2).contiguous()
|
| 72 |
-
|
| 73 |
-
# Merge heads
|
| 74 |
-
context_vec = context_vec.view(bz, nb_tockens, self.d_out)
|
| 75 |
-
context_vec = self.out_proj(context_vec)
|
| 76 |
-
return context_vec
|
| 77 |
-
|
| 78 |
-
|
| 79 |
-
class PyTorchMultiHeadAttention(nn.Module):
|
| 80 |
-
def __init__(self, d_in, d_out, num_heads, dropout=0.0, qkv_bias=False, mask_future=True, apply_rope=False):
|
| 81 |
-
super().__init__()
|
| 82 |
-
|
| 83 |
-
assert d_out % num_heads == 0, "d_out is indivisible by num_heads"
|
| 84 |
-
|
| 85 |
-
self.num_heads = num_heads
|
| 86 |
-
self.head_dim = d_out // num_heads
|
| 87 |
-
self.d_out = d_out
|
| 88 |
-
self.mask_future = mask_future
|
| 89 |
-
self.apply_rope = apply_rope
|
| 90 |
-
|
| 91 |
-
self.qkv = nn.Linear(d_in, 3 * d_out, bias=qkv_bias)
|
| 92 |
-
self.proj = nn.Linear(d_out, d_out)
|
| 93 |
-
self.dropout = dropout
|
| 94 |
-
self.rope = None # Placeholder for RoPE, if needed
|
| 95 |
-
if self.apply_rope:
|
| 96 |
-
self.rope = RoPEEmbedding(dim=self.head_dim)
|
| 97 |
-
|
| 98 |
-
def forward(self, x):
|
| 99 |
-
batch_size, num_tokens, embed_dim = x.shape
|
| 100 |
-
|
| 101 |
-
# (b, num_tokens, embed_dim) --> (b, num_tokens, 3 * embed_dim)
|
| 102 |
-
qkv = self.qkv(x)
|
| 103 |
-
|
| 104 |
-
# (b, num_tokens, 3 * embed_dim) --> (b, num_tokens, 3, num_heads, head_dim)
|
| 105 |
-
qkv = qkv.view(batch_size, num_tokens, 3, self.num_heads, self.head_dim)
|
| 106 |
-
|
| 107 |
-
# (b, num_tokens, 3, num_heads, head_dim) --> (3, b, num_heads, num_tokens, head_dim)
|
| 108 |
-
qkv = qkv.permute(2, 0, 3, 1, 4)
|
| 109 |
-
|
| 110 |
-
# (3, b, num_heads, num_tokens, head_dim) -> 3 times (b, num_heads, num_tokens, head_dim)
|
| 111 |
-
queries, keys, values = qkv
|
| 112 |
-
|
| 113 |
-
if self.apply_rope and self.rope is not None:
|
| 114 |
-
keys = self.rope(keys)
|
| 115 |
-
queries = self.rope(queries)
|
| 116 |
-
|
| 117 |
-
use_dropout = 0.0 if not self.training else self.dropout
|
| 118 |
-
|
| 119 |
-
context_vec = nn.functional.scaled_dot_product_attention(
|
| 120 |
-
queries, keys, values, attn_mask=None, dropout_p=use_dropout, is_causal=self.mask_future
|
| 121 |
-
)
|
| 122 |
-
|
| 123 |
-
# Combine heads, where self.d_out = self.num_heads * self.head_dim
|
| 124 |
-
context_vec = context_vec.transpose(1, 2).contiguous().view(batch_size, num_tokens, self.d_out)
|
| 125 |
-
|
| 126 |
-
context_vec = self.proj(context_vec)
|
| 127 |
-
|
| 128 |
-
return context_vec
|
| 129 |
-
|
| 130 |
-
|
| 131 |
-
class GroupedQueryAttention(nn.Module):
|
| 132 |
-
"""
|
| 133 |
-
Multi-head self-attention with grouped keys and values.
|
| 134 |
-
This implementation allows multiple attention heads to share the same keys and values,
|
| 135 |
-
reducing the number of parameters and computational cost.
|
| 136 |
-
|
| 137 |
-
It works by dividing the attention heads into groups, where each group shares a set of keys and values.
|
| 138 |
-
|
| 139 |
-
Example:
|
| 140 |
-
If there are 8 attention heads and 4 key-value groups, then each group will be shared by 2 attention heads.
|
| 141 |
-
|
| 142 |
-
Args:
|
| 143 |
-
d_in: Input feature dimension
|
| 144 |
-
d_out: Output feature dimension
|
| 145 |
-
num_heads: Number of attention heads
|
| 146 |
-
num_kv_groups: Number of key-value groups (must divide num_heads)
|
| 147 |
-
dropout: Dropout probability
|
| 148 |
-
qkv_bias: Whether to include bias terms in the linear projections
|
| 149 |
-
mask_future: Whether to apply causal masking to prevent attention to future tokens
|
| 150 |
-
apply_rope: Whether to apply Rotary Positional Embeddings (RoPE)
|
| 151 |
-
Returns:
|
| 152 |
-
Output tensor of shape (batch_size, seq_len, d_out)
|
| 153 |
-
"""
|
| 154 |
-
|
| 155 |
-
def __init__(
|
| 156 |
-
self, d_in, d_out, num_heads, num_kv_groups, dropout=0.0, qkv_bias=False, mask_future=True, apply_rope=False
|
| 157 |
-
):
|
| 158 |
-
super().__init__()
|
| 159 |
-
|
| 160 |
-
assert d_out % num_heads == 0, "d_out is indivisible by num_heads"
|
| 161 |
-
assert num_heads % num_kv_groups == 0, "num_heads is indivisible by num_kv_groups"
|
| 162 |
-
|
| 163 |
-
self.num_heads = num_heads
|
| 164 |
-
self.num_kv_groups = num_kv_groups
|
| 165 |
-
self.head_dim = d_out // num_heads
|
| 166 |
-
self.d_out = d_out
|
| 167 |
-
|
| 168 |
-
self.q_proj = nn.Linear(d_in, d_out, bias=qkv_bias)
|
| 169 |
-
self.k_proj = nn.Linear(d_in, num_kv_groups * self.head_dim, bias=qkv_bias)
|
| 170 |
-
self.v_proj = nn.Linear(d_in, num_kv_groups * self.head_dim, bias=qkv_bias)
|
| 171 |
-
self.group_size = num_heads // num_kv_groups
|
| 172 |
-
self.num_kv_groups = num_kv_groups
|
| 173 |
-
|
| 174 |
-
self.proj = nn.Linear(d_out, d_out, bias=False)
|
| 175 |
-
self.dropout = nn.Dropout(dropout)
|
| 176 |
-
|
| 177 |
-
self.mask_future = mask_future
|
| 178 |
-
self.apply_rope = apply_rope
|
| 179 |
-
if self.apply_rope:
|
| 180 |
-
self.rope = RoPEEmbedding(dim=self.head_dim)
|
| 181 |
-
|
| 182 |
-
def forward(self, x):
|
| 183 |
-
batch_size, num_tokens, embed_dim = x.shape
|
| 184 |
-
|
| 185 |
-
queries = self.q_proj(x)
|
| 186 |
-
keys = self.k_proj(x)
|
| 187 |
-
values = self.v_proj(x)
|
| 188 |
-
|
| 189 |
-
queries = queries.view(batch_size, num_tokens, self.num_heads, self.head_dim).transpose(1, 2)
|
| 190 |
-
keys_new = keys.view(batch_size, num_tokens, self.num_kv_groups, self.head_dim).transpose(1, 2)
|
| 191 |
-
values_new = values.view(batch_size, num_tokens, self.num_kv_groups, self.head_dim).transpose(1, 2)
|
| 192 |
-
|
| 193 |
-
keys = keys_new.repeat_interleave(self.group_size, dim=1)
|
| 194 |
-
values = values_new.repeat_interleave(self.group_size, dim=1)
|
| 195 |
-
|
| 196 |
-
if self.apply_rope:
|
| 197 |
-
keys = self.rope(keys)
|
| 198 |
-
queries = self.rope(queries)
|
| 199 |
-
|
| 200 |
-
use_dropout = 0.0 if not self.training else self.dropout.p
|
| 201 |
-
|
| 202 |
-
context_vec = F.scaled_dot_product_attention(
|
| 203 |
-
queries, keys, values, attn_mask=None, dropout_p=use_dropout, is_causal=self.mask_future
|
| 204 |
-
)
|
| 205 |
-
|
| 206 |
-
context_vec = context_vec.transpose(1, 2).contiguous()
|
| 207 |
-
context_vec = context_vec.view(batch_size, num_tokens, self.d_out)
|
| 208 |
-
context_vec = self.proj(context_vec)
|
| 209 |
-
|
| 210 |
-
return context_vec
|
| 211 |
-
|
| 212 |
-
|
| 213 |
-
if __name__ == "__main__":
|
| 214 |
-
# Simple test
|
| 215 |
-
batch_size = 2
|
| 216 |
-
seq_len = 16
|
| 217 |
-
d_in = 64
|
| 218 |
-
d_out = 64
|
| 219 |
-
num_heads = 8
|
| 220 |
-
num_kv_groups = 4
|
| 221 |
-
|
| 222 |
-
x = torch.randn(batch_size, seq_len, d_in)
|
| 223 |
-
|
| 224 |
-
attn = GroupedQueryAttention(d_in, d_out, num_heads, num_kv_groups, apply_rope=True)
|
| 225 |
-
output = attn(x)
|
| 226 |
-
|
| 227 |
-
print("Input shape:", x.shape)
|
| 228 |
-
print("Output shape:", output.shape)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
chesstransformer/models/transformer/gpt.py
DELETED
|
@@ -1,79 +0,0 @@
|
|
| 1 |
-
"""This module implements GPT model architecture."""
|
| 2 |
-
|
| 3 |
-
import torch
|
| 4 |
-
import torch.nn as nn
|
| 5 |
-
from ..transformer.attention import MultiHeadAttention
|
| 6 |
-
from ..transformer.utils import LayerNorm, GELU
|
| 7 |
-
|
| 8 |
-
|
| 9 |
-
class FeedForward(nn.Module):
|
| 10 |
-
def __init__(self, cfg):
|
| 11 |
-
super().__init__()
|
| 12 |
-
self.layers = nn.Sequential(
|
| 13 |
-
nn.Linear(cfg["embed_dim"], 4 * cfg["embed_dim"]),
|
| 14 |
-
GELU(),
|
| 15 |
-
nn.Linear(4 * cfg["embed_dim"], cfg["embed_dim"]),
|
| 16 |
-
)
|
| 17 |
-
|
| 18 |
-
def forward(self, x):
|
| 19 |
-
return self.layers(x)
|
| 20 |
-
|
| 21 |
-
|
| 22 |
-
class TransformerBlock(nn.Module):
|
| 23 |
-
def __init__(self, cfg):
|
| 24 |
-
super().__init__()
|
| 25 |
-
self.att = MultiHeadAttention(
|
| 26 |
-
d_in=cfg["embed_dim"],
|
| 27 |
-
d_out=cfg["embed_dim"],
|
| 28 |
-
num_heads=cfg["num_heads"],
|
| 29 |
-
context_length=cfg["context_size"],
|
| 30 |
-
dropout=cfg["dropout"],
|
| 31 |
-
qkv_bias=cfg["qkv_bias"],
|
| 32 |
-
)
|
| 33 |
-
self.feed_forward = FeedForward(cfg)
|
| 34 |
-
self.norm1 = LayerNorm(cfg["embed_dim"])
|
| 35 |
-
self.norm2 = LayerNorm(cfg["embed_dim"])
|
| 36 |
-
self.dropout_shortcut = nn.Dropout(cfg["dropout"])
|
| 37 |
-
|
| 38 |
-
def forward(self, x):
|
| 39 |
-
shortcut = x
|
| 40 |
-
x = self.norm1(x)
|
| 41 |
-
x = self.att(x)
|
| 42 |
-
x = self.dropout_shortcut(x)
|
| 43 |
-
x = x + shortcut
|
| 44 |
-
|
| 45 |
-
shortcut = x
|
| 46 |
-
x = self.norm2(x)
|
| 47 |
-
x = self.feed_forward(x)
|
| 48 |
-
x = self.dropout_shortcut(x)
|
| 49 |
-
x = x + shortcut
|
| 50 |
-
|
| 51 |
-
return x
|
| 52 |
-
|
| 53 |
-
|
| 54 |
-
class GPT2Model(nn.Module):
|
| 55 |
-
def __init__(self, cfg):
|
| 56 |
-
super().__init__()
|
| 57 |
-
self.token_embedding = nn.Embedding(cfg["vocab_size"], cfg["embed_dim"])
|
| 58 |
-
self.position_embedding = nn.Embedding(cfg["context_size"], cfg["embed_dim"])
|
| 59 |
-
self.dropout_embed = nn.Dropout(cfg["dropout"])
|
| 60 |
-
self.transfomer_blocks = nn.Sequential(*[TransformerBlock(cfg) for _ in range(cfg["num_layers"])])
|
| 61 |
-
self.final_norm = LayerNorm(cfg["embed_dim"])
|
| 62 |
-
self.lm_head = nn.Linear(cfg["embed_dim"], cfg["vocab_size"], bias=False)
|
| 63 |
-
|
| 64 |
-
def forward(self, x):
|
| 65 |
-
bz, sq_len = x.size()
|
| 66 |
-
|
| 67 |
-
token_emb = self.token_embedding(x) # (bz, sq_len, embed_dim)
|
| 68 |
-
|
| 69 |
-
positions = torch.arange(sq_len, device=x.device)
|
| 70 |
-
pos_emb = self.position_embedding(positions) # (sq_len, embed_dim)
|
| 71 |
-
|
| 72 |
-
x = token_emb + pos_emb # (bz, sq_len, embed_dim)
|
| 73 |
-
x = self.dropout_embed(x)
|
| 74 |
-
|
| 75 |
-
x = self.transfomer_blocks(x)
|
| 76 |
-
x = self.final_norm(x)
|
| 77 |
-
|
| 78 |
-
logits = self.lm_head(x) # (bz, sq_len, vocab_size)
|
| 79 |
-
return logits
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
chesstransformer/models/transformer/pos2move_v2.py
DELETED
|
@@ -1,311 +0,0 @@
|
|
| 1 |
-
import torch
|
| 2 |
-
import torch.nn as nn
|
| 3 |
-
import torch.nn.functional as F
|
| 4 |
-
|
| 5 |
-
NUM_ACTION_PLANES = 73
|
| 6 |
-
BOARD_SQUARES = 64
|
| 7 |
-
EXTRA_TOKENS = 3 # castling, en_passant, player
|
| 8 |
-
CONTEXT_SIZE = BOARD_SQUARES + EXTRA_TOKENS # 67
|
| 9 |
-
|
| 10 |
-
|
| 11 |
-
def build_chess_relation_index(context_size: int = CONTEXT_SIZE) -> torch.Tensor:
|
| 12 |
-
"""Precompute (T, T) relation index between every pair of tokens.
|
| 13 |
-
|
| 14 |
-
Categories (8 total):
|
| 15 |
-
0: same square / same token
|
| 16 |
-
1: same file
|
| 17 |
-
2: same rank
|
| 18 |
-
3: same diagonal
|
| 19 |
-
4: knight-reachable
|
| 20 |
-
5: king-adjacent (Chebyshev <= 1)
|
| 21 |
-
6: nearby (Manhattan <= 3)
|
| 22 |
-
7: far
|
| 23 |
-
Game-state tokens (>= 64) get category 0 toward themselves and
|
| 24 |
-
a dedicated "global" category 7 toward all board squares.
|
| 25 |
-
"""
|
| 26 |
-
rel = torch.zeros(context_size, context_size, dtype=torch.long)
|
| 27 |
-
for i in range(context_size):
|
| 28 |
-
for j in range(context_size):
|
| 29 |
-
if i == j:
|
| 30 |
-
rel[i, j] = 0
|
| 31 |
-
continue
|
| 32 |
-
if i >= BOARD_SQUARES or j >= BOARD_SQUARES:
|
| 33 |
-
# Any pair involving a game-state token: "global" bucket
|
| 34 |
-
rel[i, j] = 7
|
| 35 |
-
continue
|
| 36 |
-
fi, ri = i % 8, i // 8
|
| 37 |
-
fj, rj = j % 8, j // 8
|
| 38 |
-
df, dr = abs(fi - fj), abs(ri - rj)
|
| 39 |
-
if df == 0:
|
| 40 |
-
rel[i, j] = 1
|
| 41 |
-
elif dr == 0:
|
| 42 |
-
rel[i, j] = 2
|
| 43 |
-
elif df == dr:
|
| 44 |
-
rel[i, j] = 3
|
| 45 |
-
elif {df, dr} == {1, 2}:
|
| 46 |
-
rel[i, j] = 4
|
| 47 |
-
elif df <= 1 and dr <= 1:
|
| 48 |
-
rel[i, j] = 5
|
| 49 |
-
elif df + dr <= 3:
|
| 50 |
-
rel[i, j] = 6
|
| 51 |
-
else:
|
| 52 |
-
rel[i, j] = 7
|
| 53 |
-
return rel
|
| 54 |
-
|
| 55 |
-
|
| 56 |
-
_RELATION_INDEX: torch.Tensor = build_chess_relation_index()
|
| 57 |
-
|
| 58 |
-
|
| 59 |
-
class ReluSquared(nn.Module):
|
| 60 |
-
def forward(self, x):
|
| 61 |
-
return F.relu(x) ** 2
|
| 62 |
-
|
| 63 |
-
class ChessGroupedQueryAttention(nn.Module):
|
| 64 |
-
def __init__(
|
| 65 |
-
self, d_in, d_out, num_heads, num_kv_groups, context_size, dropout=0.0, qkv_bias=False
|
| 66 |
-
):
|
| 67 |
-
super().__init__()
|
| 68 |
-
|
| 69 |
-
assert d_out % num_heads == 0, "d_out is indivisible by num_heads"
|
| 70 |
-
assert num_heads % num_kv_groups == 0, "num_heads is indivisible by num_kv_groups"
|
| 71 |
-
|
| 72 |
-
self.num_heads = num_heads
|
| 73 |
-
self.num_kv_groups = num_kv_groups
|
| 74 |
-
self.head_dim = d_out // num_heads
|
| 75 |
-
self.d_out = d_out
|
| 76 |
-
|
| 77 |
-
self.q_proj = nn.Linear(d_in, d_out, bias=qkv_bias)
|
| 78 |
-
self.k_proj = nn.Linear(d_in, num_kv_groups * self.head_dim, bias=qkv_bias)
|
| 79 |
-
self.v_proj = nn.Linear(d_in, num_kv_groups * self.head_dim, bias=qkv_bias)
|
| 80 |
-
self.group_size = num_heads // num_kv_groups
|
| 81 |
-
|
| 82 |
-
self.proj = nn.Linear(d_out, d_out, bias=False)
|
| 83 |
-
self.dropout = dropout
|
| 84 |
-
|
| 85 |
-
self.q_norm = nn.RMSNorm(self.head_dim)
|
| 86 |
-
self.k_norm = nn.RMSNorm(self.head_dim)
|
| 87 |
-
|
| 88 |
-
# Chess-geometry relative bias: 8 relation categories per head
|
| 89 |
-
# Massively fewer params than fully-learned (H, T, T) bias.
|
| 90 |
-
self.num_relations = 8
|
| 91 |
-
self.bias_table = nn.Parameter(torch.zeros(num_heads, self.num_relations))
|
| 92 |
-
self.register_buffer("rel_idx", _RELATION_INDEX, persistent=False)
|
| 93 |
-
|
| 94 |
-
|
| 95 |
-
def forward(self, x):
|
| 96 |
-
batch_size, num_tokens, embed_dim = x.shape
|
| 97 |
-
|
| 98 |
-
queries = self.q_proj(x)
|
| 99 |
-
keys = self.k_proj(x)
|
| 100 |
-
values = self.v_proj(x)
|
| 101 |
-
|
| 102 |
-
queries = queries.view(batch_size, num_tokens, self.num_heads, self.head_dim).transpose(1, 2)
|
| 103 |
-
keys_new = keys.view(batch_size, num_tokens, self.num_kv_groups, self.head_dim).transpose(1, 2)
|
| 104 |
-
values_new = values.view(batch_size, num_tokens, self.num_kv_groups, self.head_dim).transpose(1, 2)
|
| 105 |
-
|
| 106 |
-
queries = self.q_norm(queries)
|
| 107 |
-
keys_new = self.k_norm(keys_new)
|
| 108 |
-
|
| 109 |
-
# Expand K/V to full head count. At this tiny sequence length (67) the
|
| 110 |
-
# repeat_interleave is cheaper than SDPA's native enable_gqa path, which
|
| 111 |
-
# falls back to a slower kernel here (measured ~20% slower).
|
| 112 |
-
keys = keys_new.repeat_interleave(self.group_size, dim=1)
|
| 113 |
-
values = values_new.repeat_interleave(self.group_size, dim=1)
|
| 114 |
-
|
| 115 |
-
# Build additive chess-geometry bias from the compact (H, 8) table.
|
| 116 |
-
# Shape: (1, H, T, T) so SDPA broadcasts it across the batch.
|
| 117 |
-
attn_bias = self.bias_table[:, self.rel_idx].unsqueeze(0)
|
| 118 |
-
|
| 119 |
-
context_vec = F.scaled_dot_product_attention(
|
| 120 |
-
queries, keys, values,
|
| 121 |
-
attn_mask=attn_bias,
|
| 122 |
-
dropout_p=self.dropout if self.training else 0.0,
|
| 123 |
-
is_causal=False,
|
| 124 |
-
)
|
| 125 |
-
|
| 126 |
-
context_vec = context_vec.transpose(1, 2).contiguous()
|
| 127 |
-
context_vec = context_vec.view(batch_size, num_tokens, self.d_out)
|
| 128 |
-
context_vec = self.proj(context_vec)
|
| 129 |
-
|
| 130 |
-
return context_vec
|
| 131 |
-
|
| 132 |
-
class MLP(nn.Module):
|
| 133 |
-
def __init__(self, d_in, d_hidden, d_out, dropout=0.0):
|
| 134 |
-
super().__init__()
|
| 135 |
-
self.fc1 = nn.Linear(d_in, d_hidden)
|
| 136 |
-
self.act = ReluSquared()
|
| 137 |
-
self.fc2 = nn.Linear(d_hidden, d_out)
|
| 138 |
-
self.dropout = nn.Dropout(dropout)
|
| 139 |
-
|
| 140 |
-
def forward(self, x):
|
| 141 |
-
x = self.fc1(x)
|
| 142 |
-
x = self.act(x)
|
| 143 |
-
x = self.dropout(x)
|
| 144 |
-
x = self.fc2(x)
|
| 145 |
-
return x
|
| 146 |
-
|
| 147 |
-
class ChessTransformerLayer(nn.Module):
|
| 148 |
-
def __init__(self, dim, nb_heads, qkv_bias, context_size, dropout=0.1, layer_drop=0.0):
|
| 149 |
-
super().__init__()
|
| 150 |
-
self.attn = ChessGroupedQueryAttention(
|
| 151 |
-
d_in=dim,
|
| 152 |
-
d_out=dim,
|
| 153 |
-
num_heads=nb_heads,
|
| 154 |
-
num_kv_groups=nb_heads // 2,
|
| 155 |
-
context_size=context_size,
|
| 156 |
-
dropout=dropout,
|
| 157 |
-
qkv_bias=qkv_bias,
|
| 158 |
-
)
|
| 159 |
-
self.mlp = MLP(dim, dim * 4, dim, dropout=dropout)
|
| 160 |
-
self.norm1 = nn.RMSNorm(dim)
|
| 161 |
-
self.norm2 = nn.RMSNorm(dim)
|
| 162 |
-
self.layer_drop = layer_drop
|
| 163 |
-
|
| 164 |
-
def forward(self, x):
|
| 165 |
-
# Stochastic depth: with prob `layer_drop`, skip the entire layer.
|
| 166 |
-
# Applied independently to attn and mlp branches (sub-layer LayerDrop).
|
| 167 |
-
if self.training and self.layer_drop > 0.0:
|
| 168 |
-
# Per-sample keep mask would be more standard, but per-layer scalar matches your old behavior.
|
| 169 |
-
keep_attn = torch.empty(1, device=x.device, dtype=x.dtype).bernoulli_(1.0 - self.layer_drop)
|
| 170 |
-
keep_mlp = torch.empty(1, device=x.device, dtype=x.dtype).bernoulli_(1.0 - self.layer_drop)
|
| 171 |
-
x = x + keep_attn * self.attn(self.norm1(x))
|
| 172 |
-
x = x + keep_mlp * self.mlp(self.norm2(x))
|
| 173 |
-
else:
|
| 174 |
-
x = x + self.attn(self.norm1(x))
|
| 175 |
-
x = x + self.mlp(self.norm2(x))
|
| 176 |
-
return x
|
| 177 |
-
|
| 178 |
-
class Pos2MoveV2(nn.Module):
|
| 179 |
-
def __init__(
|
| 180 |
-
self,
|
| 181 |
-
vocab_size: int = 13,
|
| 182 |
-
embed_dim: int = 512,
|
| 183 |
-
nb_transformer_layers: int = 2,
|
| 184 |
-
num_heads: int = 8,
|
| 185 |
-
dropout: float = 0.1,
|
| 186 |
-
kvq_bias: bool = False,
|
| 187 |
-
layer_drop: float = 0.0,
|
| 188 |
-
value_bounded: bool = False,
|
| 189 |
-
value_clamp: bool = False,
|
| 190 |
-
):
|
| 191 |
-
super().__init__()
|
| 192 |
-
|
| 193 |
-
self.token_embedding = nn.Embedding(vocab_size, embed_dim)
|
| 194 |
-
self.position_embedding = nn.Embedding(67, embed_dim) # 64 squares + 3 extra tokens
|
| 195 |
-
# Static position indices, kept as a buffer so we don't rebuild the
|
| 196 |
-
# arange (and a host->device copy) on every forward pass.
|
| 197 |
-
self.register_buffer("pos_index", torch.arange(67), persistent=False)
|
| 198 |
-
self.castling_embedding = nn.Embedding(16, embed_dim) # 4-bit bitmask: 0-15
|
| 199 |
-
self.en_passant_embedding = nn.Embedding(9, embed_dim) # 8 files + no en passant (8)
|
| 200 |
-
self.player_embedding = nn.Embedding(2, embed_dim) # White or Black
|
| 201 |
-
|
| 202 |
-
# Linearly increasing layer drop rate: deeper layers drop more.
|
| 203 |
-
# This is the "Deep Networks with Stochastic Depth" schedule.
|
| 204 |
-
drop_rates = [layer_drop * i / max(1, nb_transformer_layers - 1)
|
| 205 |
-
for i in range(nb_transformer_layers)]
|
| 206 |
-
|
| 207 |
-
self.transformer_layers = nn.ModuleList(
|
| 208 |
-
[
|
| 209 |
-
ChessTransformerLayer(
|
| 210 |
-
dim=embed_dim,
|
| 211 |
-
nb_heads=num_heads,
|
| 212 |
-
qkv_bias=kvq_bias,
|
| 213 |
-
context_size=67, # Max sequence length (64 squares + 3 extra tokens)
|
| 214 |
-
dropout=dropout,
|
| 215 |
-
layer_drop=drop_rates[i],
|
| 216 |
-
)
|
| 217 |
-
for i in range(nb_transformer_layers)
|
| 218 |
-
]
|
| 219 |
-
)
|
| 220 |
-
|
| 221 |
-
self.final_norm = nn.RMSNorm(embed_dim)
|
| 222 |
-
|
| 223 |
-
policy_hidden = embed_dim // 2
|
| 224 |
-
self.move_head = nn.Sequential(
|
| 225 |
-
nn.Linear(embed_dim, policy_hidden),
|
| 226 |
-
nn.GELU(),
|
| 227 |
-
nn.Linear(policy_hidden, NUM_ACTION_PLANES),
|
| 228 |
-
)
|
| 229 |
-
|
| 230 |
-
# Deeper value head: D → D//4 → GELU → 1
|
| 231 |
-
# value_bounded appends a tanh so the output cannot leave the target's
|
| 232 |
-
# [-1, 1] range. Kept off by default: the shipped v2.1 weights were
|
| 233 |
-
# trained without it, and squashing them post-hoc changes their scale.
|
| 234 |
-
# The Tanh lands at index 3, so the Linear state-dict keys (0.*, 2.*)
|
| 235 |
-
# are identical either way and checkpoints stay interchangeable.
|
| 236 |
-
value_hidden = embed_dim // 4
|
| 237 |
-
value_layers = [
|
| 238 |
-
nn.Linear(embed_dim, value_hidden),
|
| 239 |
-
nn.GELU(),
|
| 240 |
-
nn.Linear(value_hidden, 1),
|
| 241 |
-
]
|
| 242 |
-
if value_bounded:
|
| 243 |
-
value_layers.append(nn.Tanh())
|
| 244 |
-
elif value_clamp:
|
| 245 |
-
# Minimal alternative to tanh for weights already trained unbounded:
|
| 246 |
-
# a hard clamp leaves every in-range output untouched and only edits
|
| 247 |
-
# the semantically impossible ones (a leaf "better than won"). v2.1
|
| 248 |
-
# emits those on 0.76% of positions, reaching 1.039 -- and since MCTS
|
| 249 |
-
# maximises over leaf values, that tail is precisely what it chases.
|
| 250 |
-
# MEASURED 2026-08-19: on v2.1 this is a NO-OP -- the chosen move was
|
| 251 |
-
# identical on 25/25 opening positions at both 4000 and 8000 sims. A
|
| 252 |
-
# 4% excursion is too small to steer the search; it mattered for the
|
| 253 |
-
# cp-trained head only because that one reached 1.211. Don't re-run.
|
| 254 |
-
# Kept for unbounded heads that overshoot more; prefer value_bounded
|
| 255 |
-
# (tanh) when training a head from scratch.
|
| 256 |
-
value_layers.append(nn.Hardtanh(-1.0, 1.0))
|
| 257 |
-
self.value_head = nn.Sequential(*value_layers)
|
| 258 |
-
|
| 259 |
-
self._init_weights(nb_transformer_layers)
|
| 260 |
-
|
| 261 |
-
def _init_weights(self, nb_layers: int):
|
| 262 |
-
"""GPT-2 style init with depth-scaled residual projections.
|
| 263 |
-
|
| 264 |
-
- All Linear/Embedding: N(0, 0.02)
|
| 265 |
-
- Residual projections (attn out + mlp out): scaled by 1/sqrt(2*N)
|
| 266 |
-
to keep residual stream variance bounded with depth.
|
| 267 |
-
"""
|
| 268 |
-
for module in self.modules():
|
| 269 |
-
if isinstance(module, nn.Linear):
|
| 270 |
-
nn.init.normal_(module.weight, mean=0.0, std=0.02)
|
| 271 |
-
if module.bias is not None:
|
| 272 |
-
nn.init.zeros_(module.bias)
|
| 273 |
-
elif isinstance(module, nn.Embedding):
|
| 274 |
-
nn.init.normal_(module.weight, mean=0.0, std=0.02)
|
| 275 |
-
|
| 276 |
-
# Scale down the projections that write into the residual stream
|
| 277 |
-
scale = (2 * nb_layers) ** -0.5
|
| 278 |
-
for layer in self.transformer_layers:
|
| 279 |
-
with torch.no_grad():
|
| 280 |
-
layer.attn.proj.weight.mul_(scale)
|
| 281 |
-
layer.mlp.fc2.weight.mul_(scale)
|
| 282 |
-
|
| 283 |
-
def forward(self, board_tokens, player_token, castling_token, en_passant_token):
|
| 284 |
-
batch_size = board_tokens.size(0)
|
| 285 |
-
|
| 286 |
-
token_emb = self.token_embedding(board_tokens) # (B, 64, D)
|
| 287 |
-
pos_emb = self.position_embedding(self.pos_index) # (67, D)
|
| 288 |
-
castling_emb = self.castling_embedding(castling_token).unsqueeze(1) # (B, 1, D)
|
| 289 |
-
en_passant_emb = self.en_passant_embedding(en_passant_token).unsqueeze(1) # (B, 1, D)
|
| 290 |
-
player_emb = self.player_embedding(player_token).unsqueeze(1) # (B, 1, D)
|
| 291 |
-
|
| 292 |
-
x = token_emb + pos_emb[:64] # Add positional embedding to board tokens
|
| 293 |
-
x = torch.cat([
|
| 294 |
-
x,
|
| 295 |
-
castling_emb + pos_emb[64],
|
| 296 |
-
en_passant_emb + pos_emb[65],
|
| 297 |
-
player_emb + pos_emb[66],
|
| 298 |
-
], dim=1) # (B, 67, D)
|
| 299 |
-
|
| 300 |
-
for layer in self.transformer_layers:
|
| 301 |
-
x = layer(x)
|
| 302 |
-
|
| 303 |
-
x = self.final_norm(x)
|
| 304 |
-
|
| 305 |
-
state_token = x[:, -3:, :].mean(dim=1) # Average the last 3 game-state tokens: (B, D)
|
| 306 |
-
|
| 307 |
-
board_tokens_out = x[:, :64, :] # (B, 64, D)
|
| 308 |
-
move_logits = self.move_head(board_tokens_out) # (B, 64, NUM_ACTION_PLANES)
|
| 309 |
-
value_pred = self.value_head(state_token) # (B, 1)
|
| 310 |
-
|
| 311 |
-
return move_logits, value_pred
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
chesstransformer/models/transformer/position2move.py
DELETED
|
@@ -1,329 +0,0 @@
|
|
| 1 |
-
import torch
|
| 2 |
-
from torch import nn
|
| 3 |
-
import torch.nn.functional as F
|
| 4 |
-
|
| 5 |
-
from chesstransformer.models.transformer.attention import GroupedQueryAttention
|
| 6 |
-
|
| 7 |
-
|
| 8 |
-
class RMSNorm(nn.Module):
|
| 9 |
-
"""Root Mean Square Layer Normalization.
|
| 10 |
-
|
| 11 |
-
More efficient than LayerNorm as it doesn't compute mean.
|
| 12 |
-
Used in Llama, Mistral, and other modern transformers.
|
| 13 |
-
"""
|
| 14 |
-
|
| 15 |
-
def __init__(self, dim: int, eps: float = 1e-6):
|
| 16 |
-
super().__init__()
|
| 17 |
-
self.eps = eps
|
| 18 |
-
self.weight = nn.Parameter(torch.ones(dim))
|
| 19 |
-
|
| 20 |
-
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
| 21 |
-
# x: (batch, seq, dim)
|
| 22 |
-
rms = torch.rsqrt(x.pow(2).mean(-1, keepdim=True) + self.eps)
|
| 23 |
-
return x * rms * self.weight
|
| 24 |
-
|
| 25 |
-
|
| 26 |
-
class SwiGLUFeedForward(nn.Module):
|
| 27 |
-
"""SwiGLU Feed-Forward Network.
|
| 28 |
-
|
| 29 |
-
Combines Swish activation with Gated Linear Units.
|
| 30 |
-
Better performance than standard GELU FFN.
|
| 31 |
-
Used in Llama 2, PaLM, and other modern transformers.
|
| 32 |
-
"""
|
| 33 |
-
|
| 34 |
-
def __init__(self, embed_dim: int, hidden_dim: int = None, dropout: float = 0.0):
|
| 35 |
-
super().__init__()
|
| 36 |
-
# Default hidden_dim is ~2.67x embed_dim (compensates for extra params from gate)
|
| 37 |
-
hidden_dim = hidden_dim or int(embed_dim * 8 / 3)
|
| 38 |
-
# Round to nearest multiple of 64 for efficiency
|
| 39 |
-
hidden_dim = ((hidden_dim + 63) // 64) * 64
|
| 40 |
-
|
| 41 |
-
self.w1 = nn.Linear(embed_dim, hidden_dim, bias=False) # Up projection
|
| 42 |
-
self.w2 = nn.Linear(hidden_dim, embed_dim, bias=False) # Down projection
|
| 43 |
-
self.w3 = nn.Linear(embed_dim, hidden_dim, bias=False) # Gate projection
|
| 44 |
-
self.dropout = nn.Dropout(dropout)
|
| 45 |
-
|
| 46 |
-
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
| 47 |
-
# SwiGLU: w2(SiLU(w1(x)) * w3(x))
|
| 48 |
-
return self.dropout(self.w2(F.silu(self.w1(x)) * self.w3(x)))
|
| 49 |
-
|
| 50 |
-
|
| 51 |
-
# Keep old FeedForward for backward compatibility
|
| 52 |
-
class FeedForward(nn.Module):
|
| 53 |
-
def __init__(self, embed_dim):
|
| 54 |
-
super().__init__()
|
| 55 |
-
self.layers = nn.Sequential(
|
| 56 |
-
nn.Linear(embed_dim, 4 * embed_dim),
|
| 57 |
-
nn.GELU(approximate="tanh"),
|
| 58 |
-
nn.Linear(4 * embed_dim, embed_dim),
|
| 59 |
-
)
|
| 60 |
-
|
| 61 |
-
def forward(self, x):
|
| 62 |
-
return self.layers(x)
|
| 63 |
-
|
| 64 |
-
|
| 65 |
-
class TransformerBlock(nn.Module):
|
| 66 |
-
def __init__(
|
| 67 |
-
self,
|
| 68 |
-
embed_dim,
|
| 69 |
-
num_heads,
|
| 70 |
-
num_kv_groups,
|
| 71 |
-
context_length,
|
| 72 |
-
dropout,
|
| 73 |
-
qkv_bias,
|
| 74 |
-
mask_future=False,
|
| 75 |
-
apply_rope=True,
|
| 76 |
-
use_swiglu=True,
|
| 77 |
-
):
|
| 78 |
-
super().__init__()
|
| 79 |
-
self.att = GroupedQueryAttention(
|
| 80 |
-
d_in=embed_dim,
|
| 81 |
-
d_out=embed_dim,
|
| 82 |
-
num_heads=num_heads,
|
| 83 |
-
num_kv_groups=num_kv_groups,
|
| 84 |
-
dropout=dropout,
|
| 85 |
-
qkv_bias=qkv_bias,
|
| 86 |
-
mask_future=mask_future,
|
| 87 |
-
apply_rope=apply_rope,
|
| 88 |
-
)
|
| 89 |
-
# Use SwiGLU by default, fallback to standard FFN
|
| 90 |
-
if use_swiglu:
|
| 91 |
-
self.feed_forward = SwiGLUFeedForward(embed_dim, dropout=dropout)
|
| 92 |
-
else:
|
| 93 |
-
self.feed_forward = FeedForward(embed_dim)
|
| 94 |
-
|
| 95 |
-
# Use RMSNorm instead of LayerNorm
|
| 96 |
-
self.norm1 = RMSNorm(embed_dim)
|
| 97 |
-
self.norm2 = RMSNorm(embed_dim)
|
| 98 |
-
self.dropout_shortcut = nn.Dropout(dropout)
|
| 99 |
-
|
| 100 |
-
def forward(self, x):
|
| 101 |
-
shortcut = x
|
| 102 |
-
x = self.norm1(x)
|
| 103 |
-
x = self.att(x)
|
| 104 |
-
x = self.dropout_shortcut(x)
|
| 105 |
-
x = x + shortcut
|
| 106 |
-
|
| 107 |
-
shortcut = x
|
| 108 |
-
x = self.norm2(x)
|
| 109 |
-
x = self.feed_forward(x)
|
| 110 |
-
x = self.dropout_shortcut(x)
|
| 111 |
-
x = x + shortcut
|
| 112 |
-
|
| 113 |
-
return x
|
| 114 |
-
|
| 115 |
-
|
| 116 |
-
class Position2MoveModel(nn.Module):
|
| 117 |
-
def __init__(
|
| 118 |
-
self,
|
| 119 |
-
vocab_size: int = 13,
|
| 120 |
-
move_vocab_size: int = 1856,
|
| 121 |
-
embed_dim: int = 512,
|
| 122 |
-
nb_transformer_layers: int = 8,
|
| 123 |
-
num_heads: int = 8,
|
| 124 |
-
num_kv_groups: int = 4,
|
| 125 |
-
dropout: float = 0.1,
|
| 126 |
-
kvq_bias: bool = False,
|
| 127 |
-
mask_future: bool = False,
|
| 128 |
-
rope: bool = True,
|
| 129 |
-
use_swiglu: bool = True,
|
| 130 |
-
use_col_row_emb: bool = False,
|
| 131 |
-
):
|
| 132 |
-
super().__init__()
|
| 133 |
-
# Learnable CLS token for classification (instead of mean pooling)
|
| 134 |
-
self.cls_token = nn.Parameter(torch.randn(1, 1, embed_dim) * 0.02)
|
| 135 |
-
|
| 136 |
-
# Board piece embeddings
|
| 137 |
-
self.token_embedding = nn.Embedding(vocab_size, embed_dim)
|
| 138 |
-
if not rope:
|
| 139 |
-
self.position_embedding = nn.Embedding(64, embed_dim) # 64 squares
|
| 140 |
-
self.player_embedding = nn.Embedding(2, embed_dim) # 2 players: white and black
|
| 141 |
-
|
| 142 |
-
self.use_col_row_emb = use_col_row_emb
|
| 143 |
-
|
| 144 |
-
if use_col_row_emb:
|
| 145 |
-
# Optional column and row embeddings for board squares
|
| 146 |
-
self.col_embedding = nn.Embedding(8, embed_dim)
|
| 147 |
-
self.row_embedding = nn.Embedding(8, embed_dim)
|
| 148 |
-
|
| 149 |
-
# Additional game state embeddings
|
| 150 |
-
# Castling rights: 4 binary flags (white_king, white_queen, black_king, black_queen)
|
| 151 |
-
self.castling_embedding = nn.Embedding(16, embed_dim) # 2^4 = 16 combinations
|
| 152 |
-
|
| 153 |
-
# En passant: 9 values (files a-h = 0-7, none = 8)
|
| 154 |
-
self.en_passant_embedding = nn.Embedding(9, embed_dim)
|
| 155 |
-
|
| 156 |
-
# Halfmove clock: bucketed into ranges for 50-move rule awareness
|
| 157 |
-
# Buckets: 0, 1-5, 6-10, 11-20, 21-30, 31-40, 41-50, 50+
|
| 158 |
-
self.halfmove_embedding = nn.Embedding(8, embed_dim)
|
| 159 |
-
|
| 160 |
-
self.dropout = nn.Dropout(dropout)
|
| 161 |
-
self.transformer_blocks = nn.Sequential(
|
| 162 |
-
*[
|
| 163 |
-
TransformerBlock(
|
| 164 |
-
embed_dim=embed_dim,
|
| 165 |
-
num_heads=num_heads,
|
| 166 |
-
num_kv_groups=num_kv_groups,
|
| 167 |
-
context_length=64 + 3 + 1, # 64 squares + 3 game state tokens + 1 CLS
|
| 168 |
-
dropout=dropout,
|
| 169 |
-
qkv_bias=kvq_bias,
|
| 170 |
-
mask_future=mask_future,
|
| 171 |
-
apply_rope=rope,
|
| 172 |
-
use_swiglu=use_swiglu,
|
| 173 |
-
)
|
| 174 |
-
for _ in range(nb_transformer_layers)
|
| 175 |
-
]
|
| 176 |
-
)
|
| 177 |
-
# Final RMSNorm
|
| 178 |
-
self.norm = RMSNorm(embed_dim)
|
| 179 |
-
|
| 180 |
-
self.lm_head = nn.Linear(embed_dim, move_vocab_size, bias=False)
|
| 181 |
-
|
| 182 |
-
self._init_weights()
|
| 183 |
-
|
| 184 |
-
def _init_weights(self):
|
| 185 |
-
torch.nn.init.normal_(self.token_embedding.weight, mean=0.0, std=0.02)
|
| 186 |
-
if hasattr(self, "position_embedding"):
|
| 187 |
-
torch.nn.init.normal_(self.position_embedding.weight, mean=0.0, std=0.02)
|
| 188 |
-
if self.use_col_row_emb:
|
| 189 |
-
torch.nn.init.normal_(self.col_embedding.weight, mean=0.0, std=0.02)
|
| 190 |
-
torch.nn.init.normal_(self.row_embedding.weight, mean=0.0, std=0.02)
|
| 191 |
-
|
| 192 |
-
torch.nn.init.normal_(self.player_embedding.weight, mean=0.0, std=0.02)
|
| 193 |
-
torch.nn.init.normal_(self.castling_embedding.weight, mean=0.0, std=0.02)
|
| 194 |
-
torch.nn.init.normal_(self.en_passant_embedding.weight, mean=0.0, std=0.02)
|
| 195 |
-
torch.nn.init.normal_(self.halfmove_embedding.weight, mean=0.0, std=0.02)
|
| 196 |
-
|
| 197 |
-
# init the q, k, v, and output projection layers using xavier initialization
|
| 198 |
-
for layer in self.transformer_blocks:
|
| 199 |
-
if isinstance(layer.att, GroupedQueryAttention):
|
| 200 |
-
torch.nn.init.xavier_uniform_(layer.att.q_proj.weight)
|
| 201 |
-
if layer.att.q_proj.bias is not None:
|
| 202 |
-
torch.nn.init.zeros_(layer.att.q_proj.bias)
|
| 203 |
-
torch.nn.init.xavier_uniform_(layer.att.k_proj.weight)
|
| 204 |
-
if layer.att.k_proj.bias is not None:
|
| 205 |
-
torch.nn.init.zeros_(layer.att.k_proj.bias)
|
| 206 |
-
torch.nn.init.xavier_uniform_(layer.att.v_proj.weight)
|
| 207 |
-
if layer.att.v_proj.bias is not None:
|
| 208 |
-
torch.nn.init.zeros_(layer.att.v_proj.bias)
|
| 209 |
-
torch.nn.init.xavier_uniform_(layer.att.proj.weight)
|
| 210 |
-
|
| 211 |
-
# Initialize SwiGLU layers
|
| 212 |
-
if isinstance(layer.feed_forward, SwiGLUFeedForward):
|
| 213 |
-
torch.nn.init.xavier_uniform_(layer.feed_forward.w1.weight)
|
| 214 |
-
torch.nn.init.xavier_uniform_(layer.feed_forward.w2.weight)
|
| 215 |
-
torch.nn.init.xavier_uniform_(layer.feed_forward.w3.weight)
|
| 216 |
-
|
| 217 |
-
torch.nn.init.xavier_uniform_(self.lm_head.weight)
|
| 218 |
-
|
| 219 |
-
def _bucket_halfmove(self, halfmove: torch.Tensor) -> torch.Tensor:
|
| 220 |
-
"""Convert halfmove clock values to bucket indices.
|
| 221 |
-
|
| 222 |
-
Buckets: 0, 1-5, 6-10, 11-20, 21-30, 31-40, 41-50, 50+
|
| 223 |
-
Uses searchsorted to avoid torch.where (better for MPS compilation).
|
| 224 |
-
"""
|
| 225 |
-
# Define bucket boundaries
|
| 226 |
-
boundaries = torch.tensor([1, 6, 11, 21, 31, 41, 51], device=halfmove.device)
|
| 227 |
-
# searchsorted gives us the bucket index directly
|
| 228 |
-
# Values less than 1 -> bucket 0
|
| 229 |
-
# Values in [1, 6) -> bucket 1
|
| 230 |
-
# Values in [6, 11) -> bucket 2, etc.
|
| 231 |
-
buckets = torch.searchsorted(boundaries, halfmove.float(), right=False)
|
| 232 |
-
buckets = torch.clamp(buckets, 0, 7)
|
| 233 |
-
return buckets.long()
|
| 234 |
-
|
| 235 |
-
def forward(self, x, is_white, castling_rights=None, en_passant_file=None, halfmove_clock=None):
|
| 236 |
-
"""
|
| 237 |
-
Args:
|
| 238 |
-
x: Tensor of shape (batch_size, 64) containing token IDs for each square.
|
| 239 |
-
is_white: Tensor of shape (batch_size,) indicating if the player to move is white (True) or black (False).
|
| 240 |
-
castling_rights: Tensor of shape (batch_size,) with values 0-15 encoding 4 binary flags.
|
| 241 |
-
Bit 0: white kingside, Bit 1: white queenside,
|
| 242 |
-
Bit 2: black kingside, Bit 3: black queenside.
|
| 243 |
-
If None, defaults to all castling allowed (15).
|
| 244 |
-
en_passant_file: Tensor of shape (batch_size,) with values 0-7 for files a-h, or 8 for none.
|
| 245 |
-
If None, defaults to no en passant (8).
|
| 246 |
-
halfmove_clock: Tensor of shape (batch_size,) with halfmove clock values (0-100+).
|
| 247 |
-
If None, defaults to 0.
|
| 248 |
-
Returns:
|
| 249 |
-
logits: Tensor of shape (batch_size, move_vocab_size) containing
|
| 250 |
-
the logits for the next move prediction at each position.
|
| 251 |
-
"""
|
| 252 |
-
batch_size, seq_len = x.size()
|
| 253 |
-
assert seq_len == 64, "Input sequence length must be 64 (8x8 chess board)."
|
| 254 |
-
|
| 255 |
-
device = x.device
|
| 256 |
-
|
| 257 |
-
# Default values if not provided
|
| 258 |
-
if castling_rights is None:
|
| 259 |
-
castling_rights = torch.full((batch_size,), 15, dtype=torch.long, device=device)
|
| 260 |
-
if en_passant_file is None:
|
| 261 |
-
en_passant_file = torch.full((batch_size,), 8, dtype=torch.long, device=device)
|
| 262 |
-
if halfmove_clock is None:
|
| 263 |
-
halfmove_clock = torch.zeros(batch_size, dtype=torch.long, device=device)
|
| 264 |
-
|
| 265 |
-
positions = torch.arange(seq_len, device=device).unsqueeze(0).expand(batch_size, -1)
|
| 266 |
-
token_emb = self.token_embedding(x) # (batch_size, 64, embed_dim)
|
| 267 |
-
if hasattr(self, "position_embedding"):
|
| 268 |
-
pos_emb = self.position_embedding(positions) # (batch_size, 64, embed_dim)
|
| 269 |
-
else:
|
| 270 |
-
pos_emb = torch.zeros_like(token_emb) # Fallback if RoPE is used
|
| 271 |
-
|
| 272 |
-
player_ids = is_white.long().unsqueeze(1).expand(-1, seq_len) # (batch_size, 64)
|
| 273 |
-
|
| 274 |
-
player_emb = self.player_embedding(player_ids) # (batch_size, 64, embed_dim)
|
| 275 |
-
|
| 276 |
-
board_emb = token_emb + pos_emb + player_emb # (batch_size, 64, embed_dim)
|
| 277 |
-
|
| 278 |
-
if self.use_col_row_emb:
|
| 279 |
-
cols = (positions % 8).long() # (batch_size, 64)
|
| 280 |
-
rows = (positions // 8).long() # (batch_size, 64)
|
| 281 |
-
col_emb = self.col_embedding(cols) # (batch_size, 64, embed_dim)
|
| 282 |
-
row_emb = self.row_embedding(rows) # (batch_size, 64, embed_dim)
|
| 283 |
-
board_emb = board_emb + col_emb + row_emb # (batch_size, 64, embed_dim)
|
| 284 |
-
|
| 285 |
-
# Create game state tokens
|
| 286 |
-
castling_emb = self.castling_embedding(castling_rights.long()) # (batch_size, embed_dim)
|
| 287 |
-
en_passant_emb = self.en_passant_embedding(en_passant_file.long()) # (batch_size, embed_dim)
|
| 288 |
-
halfmove_bucket = self._bucket_halfmove(halfmove_clock.long())
|
| 289 |
-
halfmove_emb = self.halfmove_embedding(halfmove_bucket) # (batch_size, embed_dim)
|
| 290 |
-
|
| 291 |
-
# Stack game state tokens: (batch_size, 3, embed_dim)
|
| 292 |
-
game_state_tokens = torch.stack([castling_emb, en_passant_emb, halfmove_emb], dim=1)
|
| 293 |
-
|
| 294 |
-
# Expand CLS token for batch: (batch_size, 1, embed_dim)
|
| 295 |
-
cls_tokens = self.cls_token.expand(batch_size, -1, -1)
|
| 296 |
-
|
| 297 |
-
# Concatenate: [CLS] + board + game_state: (batch_size, 68, embed_dim)
|
| 298 |
-
x = torch.cat([cls_tokens, board_emb, game_state_tokens], dim=1)
|
| 299 |
-
x = self.dropout(x)
|
| 300 |
-
|
| 301 |
-
x = self.transformer_blocks(x) # (batch_size, 68, embed_dim)
|
| 302 |
-
x = self.norm(x) # (batch_size, 68, embed_dim)
|
| 303 |
-
|
| 304 |
-
# Use CLS token output for classification (instead of mean pooling)
|
| 305 |
-
x = x[:, 0, :] # (batch_size, embed_dim)
|
| 306 |
-
|
| 307 |
-
logits = self.lm_head(x) # (batch_size, move_vocab_size)
|
| 308 |
-
return logits
|
| 309 |
-
|
| 310 |
-
|
| 311 |
-
if __name__ == "__main__":
|
| 312 |
-
# Example usage
|
| 313 |
-
model = Position2MoveModel()
|
| 314 |
-
batch_size = 2
|
| 315 |
-
dummy_input = torch.randint(0, 13, (batch_size, 64)) # batch_size=2, seq_len=64
|
| 316 |
-
is_white = torch.tensor([1, 0]) # First sample is white to move, second is black
|
| 317 |
-
|
| 318 |
-
# With game state info
|
| 319 |
-
castling_rights = torch.tensor([15, 3]) # First: all castling, Second: only white can castle
|
| 320 |
-
en_passant_file = torch.tensor([8, 4]) # First: no EP, Second: EP on file e (4)
|
| 321 |
-
halfmove_clock = torch.tensor([0, 25]) # Halfmove counters
|
| 322 |
-
|
| 323 |
-
# Forward pass with all info
|
| 324 |
-
logits = model(dummy_input, is_white, castling_rights, en_passant_file, halfmove_clock)
|
| 325 |
-
print("Output shape:", logits.shape) # Should be (2, move_vocab_size)
|
| 326 |
-
|
| 327 |
-
# Also works without game state (backward compatible)
|
| 328 |
-
logits_simple = model(dummy_input, is_white)
|
| 329 |
-
print("Output shape (simple):", logits_simple.shape)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
chesstransformer/models/transformer/rope.py
DELETED
|
@@ -1,104 +0,0 @@
|
|
| 1 |
-
"""This module implements Rotary Positional Embeddings (RoPE) as described in
|
| 2 |
-
the paper "RoFormer: Enhanced Transformer with Rotary Position Embedding" (Su et al., 2021).
|
| 3 |
-
"""
|
| 4 |
-
|
| 5 |
-
import torch
|
| 6 |
-
import torch.nn as nn
|
| 7 |
-
|
| 8 |
-
|
| 9 |
-
class RoPEEmbedding(nn.Module):
|
| 10 |
-
"""Rotary Positional Embedding module.
|
| 11 |
-
|
| 12 |
-
This module applies rotary positional embeddings to the input tensor.
|
| 13 |
-
Works with input shape (batch, num_heads, seq_len, head_dim).
|
| 14 |
-
"""
|
| 15 |
-
|
| 16 |
-
def __init__(self, dim: int, base: int = 10_000, max_position_embeddings: int = 2048):
|
| 17 |
-
"""Initialize the RoPE embedding module.
|
| 18 |
-
|
| 19 |
-
Args:
|
| 20 |
-
dim (int): Dimension of each attention head (head_dim).
|
| 21 |
-
base (int): Base for the frequency computation. Lower values = faster rotation.
|
| 22 |
-
Default 10,000 works well for long sequences. For short sequences
|
| 23 |
-
like chess (67 tokens), consider lower values like 1,000.
|
| 24 |
-
max_position_embeddings (int): Maximum number of position embeddings.
|
| 25 |
-
"""
|
| 26 |
-
super(RoPEEmbedding, self).__init__()
|
| 27 |
-
self.dim = dim
|
| 28 |
-
self.base = base
|
| 29 |
-
|
| 30 |
-
# Compute inverse frequencies
|
| 31 |
-
inv_freq = 1.0 / (base ** (torch.arange(0, dim, 2).float() / dim))
|
| 32 |
-
|
| 33 |
-
# Compute position encodings for all positions
|
| 34 |
-
position = torch.arange(max_position_embeddings, dtype=torch.float)
|
| 35 |
-
# (max_pos, dim/2)
|
| 36 |
-
idx_theta = torch.einsum("i, j -> i j", position, inv_freq)
|
| 37 |
-
|
| 38 |
-
# Duplicate for both halves: (max_pos, dim)
|
| 39 |
-
idx_theta2 = torch.cat([idx_theta, idx_theta], dim=-1)
|
| 40 |
-
|
| 41 |
-
# Shape: (1, 1, max_pos, dim) for broadcasting with (batch, heads, seq, dim)
|
| 42 |
-
self.register_buffer("cos_cached", idx_theta2.cos()[None, None, :, :], persistent=False)
|
| 43 |
-
self.register_buffer("sin_cached", idx_theta2.sin()[None, None, :, :], persistent=False)
|
| 44 |
-
|
| 45 |
-
def _neg_half(self, x: torch.Tensor):
|
| 46 |
-
"""Rotate tensor by swapping and negating halves.
|
| 47 |
-
|
| 48 |
-
For RoPE, we need to compute: [-x2, x1] where x = [x1, x2]
|
| 49 |
-
This implements the rotation matrix multiplication efficiently.
|
| 50 |
-
|
| 51 |
-
Args:
|
| 52 |
-
x (torch.Tensor): Input tensor of shape (..., dim).
|
| 53 |
-
|
| 54 |
-
Returns:
|
| 55 |
-
torch.Tensor: Rotated tensor.
|
| 56 |
-
"""
|
| 57 |
-
d_2 = self.dim // 2
|
| 58 |
-
x1 = x[..., :d_2]
|
| 59 |
-
x2 = x[..., d_2:]
|
| 60 |
-
return torch.cat([-x2, x1], dim=-1)
|
| 61 |
-
|
| 62 |
-
def forward(self, x: torch.Tensor):
|
| 63 |
-
"""Apply RoPE to the input tensor.
|
| 64 |
-
|
| 65 |
-
Args:
|
| 66 |
-
x (torch.Tensor): Input tensor of shape (batch, num_heads, seq_len, head_dim).
|
| 67 |
-
|
| 68 |
-
Returns:
|
| 69 |
-
torch.Tensor: Tensor with RoPE applied, same shape as input.
|
| 70 |
-
"""
|
| 71 |
-
# x shape: (batch, num_heads, seq_len, head_dim)
|
| 72 |
-
seq_len = x.shape[2] # Get seq_len from the correct dimension
|
| 73 |
-
|
| 74 |
-
# Slice cached values to sequence length: (1, 1, seq_len, dim)
|
| 75 |
-
cos = self.cos_cached[:, :, :seq_len, :].to(x.device)
|
| 76 |
-
sin = self.sin_cached[:, :, :seq_len, :].to(x.device)
|
| 77 |
-
|
| 78 |
-
# Apply rotary embedding: x * cos + rotate(x) * sin
|
| 79 |
-
x_rotated = self._neg_half(x)
|
| 80 |
-
return x * cos + x_rotated * sin
|
| 81 |
-
|
| 82 |
-
|
| 83 |
-
if __name__ == "__main__":
|
| 84 |
-
# Test with chess-like dimensions
|
| 85 |
-
batch_size = 2
|
| 86 |
-
num_heads = 8
|
| 87 |
-
seq_len = 67 # 64 squares + 3 game state tokens
|
| 88 |
-
head_dim = 64
|
| 89 |
-
|
| 90 |
-
rope = RoPEEmbedding(dim=head_dim, base=10_000, max_position_embeddings=128)
|
| 91 |
-
x = torch.randn(batch_size, num_heads, seq_len, head_dim)
|
| 92 |
-
|
| 93 |
-
output = rope(x)
|
| 94 |
-
print(f"Input shape: {x.shape}")
|
| 95 |
-
print(f"Output shape: {output.shape}")
|
| 96 |
-
|
| 97 |
-
# Verify positions are encoded differently
|
| 98 |
-
# Two different positions should produce different outputs
|
| 99 |
-
x_same = torch.ones(1, 1, 10, head_dim)
|
| 100 |
-
out = rope(x_same)
|
| 101 |
-
print(f"\nPosition encoding test (should be different):")
|
| 102 |
-
print(f" Position 0 norm: {out[0, 0, 0].norm():.4f}")
|
| 103 |
-
print(f" Position 5 norm: {out[0, 0, 5].norm():.4f}")
|
| 104 |
-
print(f" Position 0 vs 5 cosine sim: {torch.cosine_similarity(out[0, 0, 0], out[0, 0, 5], dim=0):.4f}")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
chesstransformer/models/transformer/utils.py
DELETED
|
@@ -1,46 +0,0 @@
|
|
| 1 |
-
"""Utility functions for models."""
|
| 2 |
-
|
| 3 |
-
import torch
|
| 4 |
-
import torch.nn as nn
|
| 5 |
-
|
| 6 |
-
|
| 7 |
-
class LayerNorm(nn.Module):
|
| 8 |
-
"""Implements Layer Normalization"""
|
| 9 |
-
|
| 10 |
-
def __init__(self, dim, eps=1e-5):
|
| 11 |
-
super().__init__()
|
| 12 |
-
self.eps = eps
|
| 13 |
-
self.gamma = nn.Parameter(torch.ones(dim))
|
| 14 |
-
self.beta = nn.Parameter(torch.zeros(dim))
|
| 15 |
-
|
| 16 |
-
def forward(self, x):
|
| 17 |
-
"""
|
| 18 |
-
Forward pass of layer normalization
|
| 19 |
-
Args:
|
| 20 |
-
x: Input tensor of shape (batch_size, seq_length, dim)
|
| 21 |
-
Returns:
|
| 22 |
-
out: Layer-normalized tensor of the same shape as input
|
| 23 |
-
"""
|
| 24 |
-
|
| 25 |
-
mean = x.mean(-1, keepdim=True)
|
| 26 |
-
var = x.var(-1, keepdim=True, unbiased=False)
|
| 27 |
-
x_hat = (x - mean) / torch.sqrt(var + self.eps)
|
| 28 |
-
out = self.gamma * x_hat + self.beta
|
| 29 |
-
return out
|
| 30 |
-
|
| 31 |
-
|
| 32 |
-
class GELU(nn.Module):
|
| 33 |
-
"""Implements the GELU activation function"""
|
| 34 |
-
|
| 35 |
-
def __init__(self):
|
| 36 |
-
super().__init__()
|
| 37 |
-
|
| 38 |
-
def forward(self, x):
|
| 39 |
-
"""
|
| 40 |
-
Forward pass of GELU activation
|
| 41 |
-
Args:
|
| 42 |
-
x: Input tensor
|
| 43 |
-
Returns:
|
| 44 |
-
out: Tensor after applying GELU activation
|
| 45 |
-
"""
|
| 46 |
-
return 0.5 * x * (1 + torch.tanh(torch.sqrt(torch.tensor(2.0 / torch.pi)) * (x + 0.044715 * torch.pow(x, 3))))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
chesstransformer/optimizer.py
DELETED
|
@@ -1,206 +0,0 @@
|
|
| 1 |
-
"""Combined AdamW + Muon optimizer.
|
| 2 |
-
|
| 3 |
-
The two underlying optimizers are kept as separate ``torch.optim`` instances
|
| 4 |
-
internally; this class is a thin wrapper that exposes the standard
|
| 5 |
-
``torch.optim.Optimizer`` interface (``step``, ``zero_grad``, ``param_groups``,
|
| 6 |
-
``state_dict`` / ``load_state_dict``) so that LR schedulers, gradient
|
| 7 |
-
accumulation, and HuggingFace ``Accelerator`` treat it as a single optimizer.
|
| 8 |
-
|
| 9 |
-
Routing rule (when constructed from an ``nn.Module``):
|
| 10 |
-
|
| 11 |
-
* parameters whose name contains an ``embedding_keyword`` → AdamW (low LR)
|
| 12 |
-
* parameters whose name contains a ``head_keyword`` → AdamW (no decay)
|
| 13 |
-
* remaining parameters with ``ndim >= 2`` → Muon
|
| 14 |
-
* remaining parameters with ``ndim < 2`` (biases/norms) → AdamW (no decay)
|
| 15 |
-
"""
|
| 16 |
-
|
| 17 |
-
from __future__ import annotations
|
| 18 |
-
|
| 19 |
-
from typing import Iterable
|
| 20 |
-
|
| 21 |
-
import torch
|
| 22 |
-
from torch import nn
|
| 23 |
-
from torch.optim import AdamW, Muon, Optimizer
|
| 24 |
-
|
| 25 |
-
|
| 26 |
-
class AdamWMuon(Optimizer):
|
| 27 |
-
"""Wrapper that drives an inner ``AdamW`` and ``Muon`` in lock-step.
|
| 28 |
-
|
| 29 |
-
Parameters
|
| 30 |
-
----------
|
| 31 |
-
model:
|
| 32 |
-
``nn.Module`` whose parameters will be auto-routed by name.
|
| 33 |
-
lr:
|
| 34 |
-
Base AdamW learning rate (used for the ``other`` 1D parameter group).
|
| 35 |
-
lr_embedding, lr_head:
|
| 36 |
-
Per-group AdamW LR overrides. Default to ``lr``.
|
| 37 |
-
lr_muon:
|
| 38 |
-
Muon LR for ≥2D body weights. Default ``= 3 * lr`` (Muon tolerates a
|
| 39 |
-
higher LR than AdamW because updates are orthogonalised).
|
| 40 |
-
weight_decay:
|
| 41 |
-
AdamW weight decay applied to the embedding group. Heads and 1D
|
| 42 |
-
params get ``0.0`` by convention.
|
| 43 |
-
weight_decay_muon:
|
| 44 |
-
Muon weight decay. Default = ``weight_decay``.
|
| 45 |
-
embedding_keywords / head_keywords:
|
| 46 |
-
Substrings used to classify parameters by ``named_parameters`` key.
|
| 47 |
-
"""
|
| 48 |
-
|
| 49 |
-
def __init__(
|
| 50 |
-
self,
|
| 51 |
-
model: nn.Module,
|
| 52 |
-
*,
|
| 53 |
-
lr: float = 3e-4,
|
| 54 |
-
lr_embedding: float | None = None,
|
| 55 |
-
lr_head: float | None = None,
|
| 56 |
-
lr_muon: float | None = None,
|
| 57 |
-
weight_decay: float = 0.1,
|
| 58 |
-
weight_decay_muon: float | None = None,
|
| 59 |
-
betas: tuple[float, float] = (0.9, 0.95),
|
| 60 |
-
eps: float = 1e-8,
|
| 61 |
-
muon_momentum: float = 0.95,
|
| 62 |
-
muon_nesterov: bool = True,
|
| 63 |
-
muon_ns_steps: int = 5,
|
| 64 |
-
embedding_keywords: Iterable[str] = ("embedding", "cls_token"),
|
| 65 |
-
head_keywords: Iterable[str] = ("lm_head", "value_head"),
|
| 66 |
-
) -> None:
|
| 67 |
-
if not isinstance(model, nn.Module):
|
| 68 |
-
raise TypeError(
|
| 69 |
-
f"AdamWMuon expects an nn.Module to auto-route parameters by name, "
|
| 70 |
-
f"got {type(model).__name__}."
|
| 71 |
-
)
|
| 72 |
-
|
| 73 |
-
lr_embedding = lr if lr_embedding is None else lr_embedding
|
| 74 |
-
lr_head = lr if lr_head is None else lr_head
|
| 75 |
-
lr_muon = 3.0 * lr if lr_muon is None else lr_muon
|
| 76 |
-
weight_decay_muon = weight_decay if weight_decay_muon is None else weight_decay_muon
|
| 77 |
-
|
| 78 |
-
embedding_keywords = tuple(embedding_keywords)
|
| 79 |
-
head_keywords = tuple(head_keywords)
|
| 80 |
-
|
| 81 |
-
emb_params: list[torch.Tensor] = []
|
| 82 |
-
head_params: list[torch.Tensor] = []
|
| 83 |
-
other_params: list[torch.Tensor] = []
|
| 84 |
-
muon_params: list[torch.Tensor] = []
|
| 85 |
-
|
| 86 |
-
for name, param in model.named_parameters():
|
| 87 |
-
if not param.requires_grad:
|
| 88 |
-
continue
|
| 89 |
-
if any(kw in name for kw in embedding_keywords):
|
| 90 |
-
emb_params.append(param)
|
| 91 |
-
elif any(kw in name for kw in head_keywords):
|
| 92 |
-
head_params.append(param)
|
| 93 |
-
elif param.ndim >= 2:
|
| 94 |
-
muon_params.append(param)
|
| 95 |
-
else:
|
| 96 |
-
other_params.append(param)
|
| 97 |
-
|
| 98 |
-
adamw_groups: list[dict] = []
|
| 99 |
-
if emb_params:
|
| 100 |
-
adamw_groups.append({
|
| 101 |
-
"params": emb_params, "lr": lr_embedding,
|
| 102 |
-
"weight_decay": weight_decay, "name": "adamw_embedding",
|
| 103 |
-
})
|
| 104 |
-
if other_params:
|
| 105 |
-
adamw_groups.append({
|
| 106 |
-
"params": other_params, "lr": lr,
|
| 107 |
-
"weight_decay": 0.0, "name": "adamw_other",
|
| 108 |
-
})
|
| 109 |
-
if head_params:
|
| 110 |
-
adamw_groups.append({
|
| 111 |
-
"params": head_params, "lr": lr_head,
|
| 112 |
-
"weight_decay": 0.0, "name": "adamw_head",
|
| 113 |
-
})
|
| 114 |
-
|
| 115 |
-
if not adamw_groups:
|
| 116 |
-
raise ValueError("No AdamW-eligible parameters found in model.")
|
| 117 |
-
|
| 118 |
-
self.adamw = AdamW(adamw_groups, lr=lr, betas=betas, eps=eps,
|
| 119 |
-
weight_decay=weight_decay)
|
| 120 |
-
|
| 121 |
-
if muon_params:
|
| 122 |
-
self.muon: Muon | None = Muon(
|
| 123 |
-
[{"params": muon_params, "lr": lr_muon,
|
| 124 |
-
"weight_decay": weight_decay_muon, "name": "muon_body"}],
|
| 125 |
-
lr=lr_muon, weight_decay=weight_decay_muon,
|
| 126 |
-
momentum=muon_momentum, nesterov=muon_nesterov,
|
| 127 |
-
ns_steps=muon_ns_steps,
|
| 128 |
-
)
|
| 129 |
-
else:
|
| 130 |
-
self.muon = None
|
| 131 |
-
|
| 132 |
-
# Initialise the ``Optimizer`` base with a placeholder group so
|
| 133 |
-
# ``isinstance(opt, Optimizer)`` checks (Accelerate, schedulers) pass
|
| 134 |
-
# and ``self.state`` / ``self.defaults`` exist. We then expose the
|
| 135 |
-
# *inner* groups via ``param_groups`` so schedulers mutate them in
|
| 136 |
-
# place and both sub-optimizers see the new LR.
|
| 137 |
-
all_params = emb_params + other_params + head_params + muon_params
|
| 138 |
-
self._init_done = False
|
| 139 |
-
super().__init__([{"params": all_params}], defaults={"lr": lr})
|
| 140 |
-
self.param_groups = self._combined_param_groups()
|
| 141 |
-
self._init_done = True
|
| 142 |
-
|
| 143 |
-
# ── helpers ─────────────────────────────────────────────────────────
|
| 144 |
-
def _combined_param_groups(self) -> list[dict]:
|
| 145 |
-
groups = list(self.adamw.param_groups)
|
| 146 |
-
if self.muon is not None:
|
| 147 |
-
groups += list(self.muon.param_groups)
|
| 148 |
-
return groups
|
| 149 |
-
|
| 150 |
-
# ── Optimizer API ───────────────────────────────────────────────────
|
| 151 |
-
@torch.no_grad()
|
| 152 |
-
def step(self, closure=None):
|
| 153 |
-
loss = None
|
| 154 |
-
if closure is not None:
|
| 155 |
-
with torch.enable_grad():
|
| 156 |
-
loss = closure()
|
| 157 |
-
self.adamw.step()
|
| 158 |
-
if self.muon is not None:
|
| 159 |
-
self.muon.step()
|
| 160 |
-
return loss
|
| 161 |
-
|
| 162 |
-
def zero_grad(self, set_to_none: bool = True) -> None:
|
| 163 |
-
self.adamw.zero_grad(set_to_none=set_to_none)
|
| 164 |
-
if self.muon is not None:
|
| 165 |
-
self.muon.zero_grad(set_to_none=set_to_none)
|
| 166 |
-
|
| 167 |
-
def state_dict(self) -> dict:
|
| 168 |
-
return {
|
| 169 |
-
"adamw": self.adamw.state_dict(),
|
| 170 |
-
"muon": self.muon.state_dict() if self.muon is not None else None,
|
| 171 |
-
}
|
| 172 |
-
|
| 173 |
-
def load_state_dict(self, state_dict: dict) -> None:
|
| 174 |
-
self.adamw.load_state_dict(state_dict["adamw"])
|
| 175 |
-
if self.muon is not None and state_dict.get("muon") is not None:
|
| 176 |
-
self.muon.load_state_dict(state_dict["muon"])
|
| 177 |
-
# Re-link in case sub-optimizers rebuilt their group dicts.
|
| 178 |
-
self.param_groups = self._combined_param_groups()
|
| 179 |
-
|
| 180 |
-
def add_param_group(self, param_group) -> None:
|
| 181 |
-
if not getattr(self, "_init_done", False):
|
| 182 |
-
# Called from ``Optimizer.__init__`` with the placeholder group.
|
| 183 |
-
super().add_param_group(param_group)
|
| 184 |
-
return
|
| 185 |
-
raise NotImplementedError(
|
| 186 |
-
"add_param_group is not supported on AdamWMuon after construction; "
|
| 187 |
-
"build a new optimizer with the full parameter set instead."
|
| 188 |
-
)
|
| 189 |
-
|
| 190 |
-
def __repr__(self) -> str:
|
| 191 |
-
n_emb = sum(p.numel() for g in self.adamw.param_groups
|
| 192 |
-
if g.get("name") == "adamw_embedding" for p in g["params"])
|
| 193 |
-
n_head = sum(p.numel() for g in self.adamw.param_groups
|
| 194 |
-
if g.get("name") == "adamw_head" for p in g["params"])
|
| 195 |
-
n_other = sum(p.numel() for g in self.adamw.param_groups
|
| 196 |
-
if g.get("name") == "adamw_other" for p in g["params"])
|
| 197 |
-
n_muon = (sum(p.numel() for g in self.muon.param_groups for p in g["params"])
|
| 198 |
-
if self.muon is not None else 0)
|
| 199 |
-
return (
|
| 200 |
-
f"AdamWMuon(emb={n_emb:,}, other_1d={n_other:,}, "
|
| 201 |
-
f"head={n_head:,}, muon_2d={n_muon:,})"
|
| 202 |
-
)
|
| 203 |
-
|
| 204 |
-
|
| 205 |
-
# Backwards-compat alias for the previous skeleton class.
|
| 206 |
-
AdamWMuonOptim = AdamWMuon
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
chesstransformer/trainers/__init__.py
DELETED
|
@@ -1,11 +0,0 @@
|
|
| 1 |
-
|
| 2 |
-
|
| 3 |
-
__all__ = [
|
| 4 |
-
"SelfPlayTrainer",
|
| 5 |
-
"SelfPlayEngine",
|
| 6 |
-
"GameRecord",
|
| 7 |
-
"MoveRecord",
|
| 8 |
-
"OpeningPositionSampler",
|
| 9 |
-
"compute_rewards",
|
| 10 |
-
"COMMON_OPENINGS",
|
| 11 |
-
]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
chesstransformer/trainers/moveseq2moveseq_trainer.py
DELETED
|
@@ -1,342 +0,0 @@
|
|
| 1 |
-
from pathlib import Path
|
| 2 |
-
from datetime import datetime
|
| 3 |
-
|
| 4 |
-
from tqdm.auto import tqdm
|
| 5 |
-
import torch
|
| 6 |
-
import accelerate
|
| 7 |
-
from torch.utils.data import DataLoader
|
| 8 |
-
from torch.utils.data import random_split
|
| 9 |
-
from chesstransformer.datasets import LichessSimpleUciDataset
|
| 10 |
-
from chesstransformer.models.transformer.gpt import GPT2Model
|
| 11 |
-
|
| 12 |
-
|
| 13 |
-
ROOT_PATH = Path(__file__).parents[1]
|
| 14 |
-
|
| 15 |
-
|
| 16 |
-
def get_next_run_number(log_dir="logs"):
|
| 17 |
-
"""Get the next run number by checking existing run directories."""
|
| 18 |
-
log_path = Path(log_dir)
|
| 19 |
-
if not log_path.exists():
|
| 20 |
-
return 1
|
| 21 |
-
|
| 22 |
-
# Find all directories that match the pattern "run_XXX"
|
| 23 |
-
existing_runs = [d.name for d in log_path.iterdir() if d.is_dir() and d.name.startswith("run_")]
|
| 24 |
-
|
| 25 |
-
if not existing_runs:
|
| 26 |
-
return 1
|
| 27 |
-
|
| 28 |
-
# Extract run numbers and find the maximum
|
| 29 |
-
run_numbers = []
|
| 30 |
-
for run_name in existing_runs:
|
| 31 |
-
try:
|
| 32 |
-
num = int(run_name.split("_")[1])
|
| 33 |
-
run_numbers.append(num)
|
| 34 |
-
except (IndexError, ValueError):
|
| 35 |
-
continue
|
| 36 |
-
|
| 37 |
-
return max(run_numbers) + 1 if run_numbers else 1
|
| 38 |
-
|
| 39 |
-
|
| 40 |
-
def main():
|
| 41 |
-
# Load the full dataset with LichessSimpleUciDataset
|
| 42 |
-
print("Loading dataset...")
|
| 43 |
-
full_dataset = LichessSimpleUciDataset(
|
| 44 |
-
dataset_path=ROOT_PATH.parents[1] / "data/Lichess Rated Games 2017.pgn.zst",
|
| 45 |
-
vocab_path=ROOT_PATH / "data/tokenizer_models/uci_vocab.json",
|
| 46 |
-
num_samples=1_000_000, # Load all available games
|
| 47 |
-
)
|
| 48 |
-
|
| 49 |
-
vocab_size = len(full_dataset.vocab)
|
| 50 |
-
print(f"Vocabulary size: {vocab_size}")
|
| 51 |
-
|
| 52 |
-
# Split the dataset into training, validation, and test sets
|
| 53 |
-
val = 20_000
|
| 54 |
-
test = 20_000
|
| 55 |
-
train_size = len(full_dataset) - val - test
|
| 56 |
-
train_dataset, val_dataset, test_dataset = random_split(
|
| 57 |
-
full_dataset,
|
| 58 |
-
[train_size, val, test],
|
| 59 |
-
generator=torch.Generator().manual_seed(42),
|
| 60 |
-
)
|
| 61 |
-
|
| 62 |
-
print(f"Train size: {len(train_dataset)}")
|
| 63 |
-
print(f"Validation size: {len(val_dataset)}")
|
| 64 |
-
print(f"Test size: {len(test_dataset)}")
|
| 65 |
-
|
| 66 |
-
def collate_fn(batch):
|
| 67 |
-
max_length = max(item["input_ids"].shape[0] for item in batch)
|
| 68 |
-
input_ids = torch.full((len(batch), max_length), fill_value=-100, dtype=torch.long)
|
| 69 |
-
target_ids = torch.full((len(batch), max_length), fill_value=-100, dtype=torch.long)
|
| 70 |
-
legal_moves_mask = torch.zeros((len(batch), max_length, vocab_size), dtype=torch.float32)
|
| 71 |
-
|
| 72 |
-
for i, item in enumerate(batch):
|
| 73 |
-
length = item["input_ids"].shape[0]
|
| 74 |
-
input_ids[i, :length] = item["input_ids"]
|
| 75 |
-
target_ids[i, :length] = item["target_ids"]
|
| 76 |
-
legal_moves_mask[i, :length, :] = item["legal_moves_mask"]
|
| 77 |
-
|
| 78 |
-
if max_length > config["context_size"]:
|
| 79 |
-
input_ids = input_ids[:, : config["context_size"]]
|
| 80 |
-
target_ids = target_ids[:, : config["context_size"]]
|
| 81 |
-
legal_moves_mask = legal_moves_mask[:, : config["context_size"], :]
|
| 82 |
-
|
| 83 |
-
return {
|
| 84 |
-
"input_ids": input_ids,
|
| 85 |
-
"target_ids": target_ids,
|
| 86 |
-
"legal_moves_mask": legal_moves_mask,
|
| 87 |
-
}
|
| 88 |
-
|
| 89 |
-
train_dl = DataLoader(train_dataset, batch_size=8, shuffle=True, collate_fn=collate_fn, drop_last=True)
|
| 90 |
-
val_dl = DataLoader(val_dataset, batch_size=8, shuffle=False, collate_fn=collate_fn)
|
| 91 |
-
test_dl = DataLoader(test_dataset, batch_size=8, shuffle=False, collate_fn=collate_fn)
|
| 92 |
-
|
| 93 |
-
config = {
|
| 94 |
-
"vocab_size": vocab_size,
|
| 95 |
-
"context_size": 256,
|
| 96 |
-
"embed_dim": 512,
|
| 97 |
-
"num_heads": 8,
|
| 98 |
-
"num_layers": 8,
|
| 99 |
-
"dropout": 0.1,
|
| 100 |
-
"qkv_bias": False,
|
| 101 |
-
}
|
| 102 |
-
|
| 103 |
-
model = GPT2Model(config)
|
| 104 |
-
print(model)
|
| 105 |
-
|
| 106 |
-
# Generate unique run identifier
|
| 107 |
-
run_number = get_next_run_number("logs")
|
| 108 |
-
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
|
| 109 |
-
run_name = f"run_{run_number:03d}_{timestamp}"
|
| 110 |
-
|
| 111 |
-
print(f"Starting training run: {run_name}")
|
| 112 |
-
|
| 113 |
-
# Gradient accumulation settings
|
| 114 |
-
gradient_accumulation_steps = 8 # Effective batch size = 16 * 4 = 64
|
| 115 |
-
|
| 116 |
-
# Initialize accelerator with logging and gradient accumulation
|
| 117 |
-
accelerator = accelerate.Accelerator(
|
| 118 |
-
log_with="tensorboard",
|
| 119 |
-
project_dir=f"logs/{run_name}",
|
| 120 |
-
mixed_precision="fp16" if torch.cuda.is_available() else "no",
|
| 121 |
-
gradient_accumulation_steps=gradient_accumulation_steps,
|
| 122 |
-
)
|
| 123 |
-
device = accelerator.device
|
| 124 |
-
print(f"Using device: {device}")
|
| 125 |
-
print(f"Gradient accumulation steps: {gradient_accumulation_steps}")
|
| 126 |
-
print(f"Effective batch size: {16 * gradient_accumulation_steps}")
|
| 127 |
-
model.to(device)
|
| 128 |
-
|
| 129 |
-
# Initialize TensorBoard tracker
|
| 130 |
-
accelerator.init_trackers(
|
| 131 |
-
project_name="chess_transformer",
|
| 132 |
-
config={
|
| 133 |
-
**config,
|
| 134 |
-
"run_name": run_name,
|
| 135 |
-
"run_number": run_number,
|
| 136 |
-
"timestamp": timestamp,
|
| 137 |
-
"learning_rate": 1e-3,
|
| 138 |
-
"weight_decay": 1e-1,
|
| 139 |
-
"num_epochs": 10,
|
| 140 |
-
"batch_size": 16,
|
| 141 |
-
"gradient_accumulation_steps": 8,
|
| 142 |
-
"effective_batch_size": 128,
|
| 143 |
-
"warmup_steps": 1000,
|
| 144 |
-
},
|
| 145 |
-
)
|
| 146 |
-
|
| 147 |
-
optimizer = torch.optim.AdamW(model.parameters(), lr=5e-4, weight_decay=1e-1)
|
| 148 |
-
|
| 149 |
-
# Calculate total training steps (accounting for gradient accumulation)
|
| 150 |
-
num_epochs = 10
|
| 151 |
-
total_steps = (len(train_dl) // gradient_accumulation_steps) * num_epochs
|
| 152 |
-
warmup_steps = 1000
|
| 153 |
-
|
| 154 |
-
print(f"Total training steps: {total_steps}")
|
| 155 |
-
print(f"Warmup steps: {warmup_steps}")
|
| 156 |
-
|
| 157 |
-
# Create linear scheduler with warmup
|
| 158 |
-
def get_linear_schedule_with_warmup(optimizer, num_warmup_steps, num_training_steps):
|
| 159 |
-
def lr_lambda(current_step):
|
| 160 |
-
if current_step < num_warmup_steps:
|
| 161 |
-
# Linear warmup
|
| 162 |
-
return float(current_step) / float(max(1, num_warmup_steps))
|
| 163 |
-
# Linear decay
|
| 164 |
-
return max(
|
| 165 |
-
0.0, float(num_training_steps - current_step) / float(max(1, num_training_steps - num_warmup_steps))
|
| 166 |
-
)
|
| 167 |
-
|
| 168 |
-
return torch.optim.lr_scheduler.LambdaLR(optimizer, lr_lambda)
|
| 169 |
-
|
| 170 |
-
scheduler = get_linear_schedule_with_warmup(optimizer, warmup_steps, total_steps)
|
| 171 |
-
|
| 172 |
-
model, optimizer, train_dl, val_dl, scheduler = accelerator.prepare(model, optimizer, train_dl, val_dl, scheduler)
|
| 173 |
-
criterion = torch.nn.CrossEntropyLoss(ignore_index=-100)
|
| 174 |
-
|
| 175 |
-
best_val_loss = float("inf")
|
| 176 |
-
global_step = 0
|
| 177 |
-
|
| 178 |
-
for epoch in range(num_epochs):
|
| 179 |
-
model.train()
|
| 180 |
-
total_loss = 0.0
|
| 181 |
-
total_llm_loss = 0.0
|
| 182 |
-
total_l2_loss = 0.0
|
| 183 |
-
|
| 184 |
-
loader = tqdm(train_dl, desc=f"Epoch {epoch + 1}/{num_epochs}")
|
| 185 |
-
for batch_idx, batch in enumerate(loader):
|
| 186 |
-
# Accelerate automatically handles gradient accumulation context
|
| 187 |
-
with accelerator.accumulate(model):
|
| 188 |
-
inputs = batch["input_ids"].to(device)
|
| 189 |
-
targets = batch["target_ids"].to(device)
|
| 190 |
-
legal_moves_mask = batch["legal_moves_mask"].to(device)
|
| 191 |
-
|
| 192 |
-
logits = model(inputs)
|
| 193 |
-
|
| 194 |
-
llm_loss = criterion(logits.flatten(0, 1), targets.flatten())
|
| 195 |
-
|
| 196 |
-
probas = torch.nn.functional.softmax(logits, dim=-1)
|
| 197 |
-
|
| 198 |
-
ilegal_moves_probas = probas * (1 - legal_moves_mask)
|
| 199 |
-
l2_loss = torch.sum(ilegal_moves_probas**2, dim=-1).mean()
|
| 200 |
-
loss = llm_loss + 10 * l2_loss
|
| 201 |
-
|
| 202 |
-
loader.set_postfix({"llm_loss": llm_loss.item(), "l2_loss": l2_loss.item(), "total_loss": loss.item()})
|
| 203 |
-
|
| 204 |
-
accelerator.backward(loss)
|
| 205 |
-
optimizer.step()
|
| 206 |
-
scheduler.step() # Update learning rate
|
| 207 |
-
optimizer.zero_grad()
|
| 208 |
-
|
| 209 |
-
total_loss += loss.item()
|
| 210 |
-
total_llm_loss += llm_loss.item()
|
| 211 |
-
total_l2_loss += l2_loss.item()
|
| 212 |
-
|
| 213 |
-
# Log step metrics only on sync steps (when gradients are actually applied)
|
| 214 |
-
if accelerator.sync_gradients:
|
| 215 |
-
# Get current learning rate
|
| 216 |
-
current_lr = scheduler.get_last_lr()[0]
|
| 217 |
-
|
| 218 |
-
accelerator.log(
|
| 219 |
-
{
|
| 220 |
-
"train/step_loss": loss.item(),
|
| 221 |
-
"train/step_llm_loss": llm_loss.item(),
|
| 222 |
-
"train/step_l2_loss": l2_loss.item(),
|
| 223 |
-
"train/learning_rate": current_lr,
|
| 224 |
-
},
|
| 225 |
-
step=global_step,
|
| 226 |
-
)
|
| 227 |
-
|
| 228 |
-
global_step += 1
|
| 229 |
-
|
| 230 |
-
avg_train_loss = total_loss / len(train_dl)
|
| 231 |
-
avg_train_llm_loss = total_llm_loss / len(train_dl)
|
| 232 |
-
avg_train_l2_loss = total_l2_loss / len(train_dl)
|
| 233 |
-
print(f"Epoch {epoch + 1}/{num_epochs}, Training Loss: {avg_train_loss:.4f}")
|
| 234 |
-
|
| 235 |
-
# Validation
|
| 236 |
-
model.eval()
|
| 237 |
-
total_val_loss = 0.0
|
| 238 |
-
total_val_llm_loss = 0.0
|
| 239 |
-
total_val_l2_loss = 0.0
|
| 240 |
-
with torch.no_grad():
|
| 241 |
-
for batch in val_dl:
|
| 242 |
-
inputs = batch["input_ids"].to(device)
|
| 243 |
-
targets = batch["target_ids"].to(device)
|
| 244 |
-
legal_moves_mask = batch["legal_moves_mask"].to(device)
|
| 245 |
-
|
| 246 |
-
logits = model(inputs)
|
| 247 |
-
|
| 248 |
-
llm_loss = criterion(logits.flatten(0, 1), targets.flatten())
|
| 249 |
-
|
| 250 |
-
probas = torch.nn.functional.softmax(logits, dim=-1)
|
| 251 |
-
ilegal_moves_probas = probas * (1 - legal_moves_mask)
|
| 252 |
-
l2_loss = torch.sum(ilegal_moves_probas**2, dim=-1).mean()
|
| 253 |
-
loss = llm_loss + l2_loss
|
| 254 |
-
|
| 255 |
-
total_val_loss += loss.item()
|
| 256 |
-
total_val_llm_loss += llm_loss.item()
|
| 257 |
-
total_val_l2_loss += l2_loss.item()
|
| 258 |
-
|
| 259 |
-
avg_val_loss = total_val_loss / len(val_dl)
|
| 260 |
-
avg_val_llm_loss = total_val_llm_loss / len(val_dl)
|
| 261 |
-
avg_val_l2_loss = total_val_l2_loss / len(val_dl)
|
| 262 |
-
print(f"Epoch {epoch + 1}/{num_epochs}, Validation Loss: {avg_val_loss:.4f}")
|
| 263 |
-
|
| 264 |
-
# Log epoch metrics
|
| 265 |
-
accelerator.log(
|
| 266 |
-
{
|
| 267 |
-
"train/epoch_loss": avg_train_loss,
|
| 268 |
-
"train/epoch_llm_loss": avg_train_llm_loss,
|
| 269 |
-
"train/epoch_l2_loss": avg_train_l2_loss,
|
| 270 |
-
"val/epoch_loss": avg_val_loss,
|
| 271 |
-
"val/epoch_llm_loss": avg_val_llm_loss,
|
| 272 |
-
"val/epoch_l2_loss": avg_val_l2_loss,
|
| 273 |
-
"epoch": epoch + 1,
|
| 274 |
-
},
|
| 275 |
-
step=global_step,
|
| 276 |
-
)
|
| 277 |
-
|
| 278 |
-
# Save the best model
|
| 279 |
-
if avg_val_loss < best_val_loss:
|
| 280 |
-
best_val_loss = avg_val_loss
|
| 281 |
-
unwrapped_model = accelerator.unwrap_model(model)
|
| 282 |
-
torch.save(unwrapped_model.state_dict(), ROOT_PATH / "models/best_model.pth")
|
| 283 |
-
print(f"Best model saved with validation loss: {best_val_loss:.4f}")
|
| 284 |
-
|
| 285 |
-
# Log best metrics
|
| 286 |
-
accelerator.log(
|
| 287 |
-
{
|
| 288 |
-
"best/val_loss": best_val_loss,
|
| 289 |
-
"best/epoch": epoch + 1,
|
| 290 |
-
},
|
| 291 |
-
step=global_step,
|
| 292 |
-
)
|
| 293 |
-
|
| 294 |
-
# Testing
|
| 295 |
-
model.eval()
|
| 296 |
-
total_test_loss = 0.0
|
| 297 |
-
total_test_llm_loss = 0.0
|
| 298 |
-
total_test_l2_loss = 0.0
|
| 299 |
-
with torch.no_grad():
|
| 300 |
-
for batch in test_dl:
|
| 301 |
-
inputs = batch["input_ids"].to(device)
|
| 302 |
-
targets = batch["target_ids"].to(device)
|
| 303 |
-
legal_moves_mask = batch["legal_moves_mask"].to(device)
|
| 304 |
-
|
| 305 |
-
logits = model(inputs)
|
| 306 |
-
|
| 307 |
-
llm_loss = criterion(logits.flatten(0, 1), targets.flatten())
|
| 308 |
-
|
| 309 |
-
probas = torch.nn.functional.softmax(logits, dim=-1)
|
| 310 |
-
ilegal_moves_probas = probas * (1 - legal_moves_mask)
|
| 311 |
-
l2_loss = torch.sum(ilegal_moves_probas**2, dim=-1).mean()
|
| 312 |
-
loss = llm_loss + l2_loss
|
| 313 |
-
|
| 314 |
-
total_test_loss += loss.item()
|
| 315 |
-
total_test_llm_loss += llm_loss.item()
|
| 316 |
-
total_test_l2_loss += l2_loss.item()
|
| 317 |
-
|
| 318 |
-
avg_test_loss = total_test_loss / len(test_dl)
|
| 319 |
-
avg_test_llm_loss = total_test_llm_loss / len(test_dl)
|
| 320 |
-
avg_test_l2_loss = total_test_l2_loss / len(test_dl)
|
| 321 |
-
print(f"Test Loss: {avg_test_loss:.4f}")
|
| 322 |
-
|
| 323 |
-
# Log test metrics
|
| 324 |
-
accelerator.log(
|
| 325 |
-
{
|
| 326 |
-
"test/loss": avg_test_loss,
|
| 327 |
-
"test/llm_loss": avg_test_llm_loss,
|
| 328 |
-
"test/l2_loss": avg_test_l2_loss,
|
| 329 |
-
},
|
| 330 |
-
step=global_step,
|
| 331 |
-
)
|
| 332 |
-
|
| 333 |
-
# End tracking
|
| 334 |
-
accelerator.end_training()
|
| 335 |
-
print(f"\nTraining complete! Run: {run_name}")
|
| 336 |
-
print(f"TensorBoard logs saved to 'logs/{run_name}/'")
|
| 337 |
-
print("To view logs, run: tensorboard --logdir=logs")
|
| 338 |
-
print(f"To view this run only: tensorboard --logdir=logs/{run_name}")
|
| 339 |
-
|
| 340 |
-
|
| 341 |
-
if __name__ == "__main__":
|
| 342 |
-
main()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
chesstransformer/trainers/pos2move_v2_trainer.py
DELETED
|
@@ -1,1128 +0,0 @@
|
|
| 1 |
-
#!/usr/bin/env python3
|
| 2 |
-
"""Training script for Pos2MoveV2.
|
| 3 |
-
|
| 4 |
-
Trains the v2 transformer with GQA + QK-norm + chess-geometry attention bias,
|
| 5 |
-
AlphaZero-style 64×73 action output and value head on HDF5 chess datasets.
|
| 6 |
-
|
| 7 |
-
Usage:
|
| 8 |
-
python src/chesstransformer/trainers/pos2move_v2_trainer.py --data data/elite_db.h5
|
| 9 |
-
"""
|
| 10 |
-
|
| 11 |
-
from pathlib import Path
|
| 12 |
-
from datetime import datetime
|
| 13 |
-
import argparse
|
| 14 |
-
import json
|
| 15 |
-
import os
|
| 16 |
-
import random
|
| 17 |
-
import shutil
|
| 18 |
-
import sys
|
| 19 |
-
|
| 20 |
-
# Make the checkout runnable without installing the package. `uv run` installs it into the
|
| 21 |
-
# venv, so this never fires locally -- but a bare `git clone` on Kaggle or a rented box has
|
| 22 |
-
# no chesstransformer on sys.path and every import below fails. scripts/build_shards.py,
|
| 23 |
-
# scripts/verify_shards.py and tests/conftest.py already do this; the trainer did not, which
|
| 24 |
-
# is exactly where it was first hit.
|
| 25 |
-
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
|
| 26 |
-
|
| 27 |
-
import numpy as np
|
| 28 |
-
import torch
|
| 29 |
-
from torch.nn import functional as F
|
| 30 |
-
from torch.utils.data import DataLoader, Subset, random_split
|
| 31 |
-
from torch.optim.lr_scheduler import LambdaLR
|
| 32 |
-
from tqdm.auto import tqdm
|
| 33 |
-
import accelerate
|
| 34 |
-
from accelerate.utils import broadcast_object_list, set_seed
|
| 35 |
-
|
| 36 |
-
from chesstransformer.datasets.flat_shard_dataset import (
|
| 37 |
-
FlatShardDataset,
|
| 38 |
-
make_dataloader as make_shard_dataloader,
|
| 39 |
-
scatter_legal_planes,
|
| 40 |
-
worker_init_fn as shard_worker_init_fn,
|
| 41 |
-
)
|
| 42 |
-
from chesstransformer.datasets.h5_lichess_dataset import HDF5ChessDataset
|
| 43 |
-
from chesstransformer.distributed_muon import (
|
| 44 |
-
DistributedMuon,
|
| 45 |
-
muon_momentum,
|
| 46 |
-
muon_weight_decay,
|
| 47 |
-
)
|
| 48 |
-
from chesstransformer.models.tokenizer.position_tokenizer import PostionTokenizer
|
| 49 |
-
from chesstransformer.models.transformer.pos2move_v2 import Pos2MoveV2, NUM_ACTION_PLANES
|
| 50 |
-
|
| 51 |
-
|
| 52 |
-
#: Model sizes. DDP replicates weights *and* optimizer state on every rank.
|
| 53 |
-
#:
|
| 54 |
-
#: The static column is **measured**, not derived: one optimizer step then
|
| 55 |
-
#: torch.cuda.memory_allocated(), on a 16 GB card. It comes out at ~8.4 bytes/param, not
|
| 56 |
-
#: the ~18 a generic "fp32 master + two Adam moments + grads" estimate gives, because the
|
| 57 |
-
#: optimizer is split -- 2D params go to Muon, which keeps a single momentum buffer, and
|
| 58 |
-
#: the AdamW params (norms, biases, embeddings' non-2D parts) are a small remainder.
|
| 59 |
-
#: An earlier version of this table used the 18 B/param estimate and overstated xl by 2.3x.
|
| 60 |
-
#:
|
| 61 |
-
#: Activations dominate and scale with micro-batch: measured ~76 MB/sample for `large`
|
| 62 |
-
#: under torch.compile (~90 MB uncompiled). That is what actually decides whether a config
|
| 63 |
-
#: fits, so the static column alone is not a fit test.
|
| 64 |
-
#:
|
| 65 |
-
#: ===== ============ ======== ============= =============================================
|
| 66 |
-
#: name dim x layers params static/rank notes
|
| 67 |
-
#: ===== ============ ======== ============= =============================================
|
| 68 |
-
#: base 256 x 16 11.7 M 0.10 GB anything
|
| 69 |
-
#: 46m 512 x 16 46.5 M 0.35 GB anything
|
| 70 |
-
#: large 768 x 24 156.4 M 1.23 GB measured 8.5 GB/rank total at micro-batch 96
|
| 71 |
-
#: on a T4; micro-batch 128 fits, 160 OOMs on 16 GB
|
| 72 |
-
#: xl 1536 x 32 832.8 M ~6.4 GB extrapolated. Static alone would fit a 40 GB
|
| 73 |
-
#: card under DDP; whether activations do is
|
| 74 |
-
#: unmeasured -- measure before assuming FSDP.
|
| 75 |
-
#: ===== ============ ======== ============= =============================================
|
| 76 |
-
#:
|
| 77 |
-
#: `xl` is deliberately oversized as a sharding/scaling testbed. It is not a bid for
|
| 78 |
-
#: playing strength -- 46M already lost its head-to-head against v2.1 at 44.8%.
|
| 79 |
-
PRESETS = {
|
| 80 |
-
"base": {"embed_dim": 256, "num_layers": 16, "num_heads": 8},
|
| 81 |
-
"46m": {"embed_dim": 512, "num_layers": 16, "num_heads": 8},
|
| 82 |
-
"large": {"embed_dim": 768, "num_layers": 24, "num_heads": 12},
|
| 83 |
-
"xl": {"embed_dim": 1536, "num_layers": 32, "num_heads": 16},
|
| 84 |
-
}
|
| 85 |
-
|
| 86 |
-
torch.set_float32_matmul_precision('high')
|
| 87 |
-
|
| 88 |
-
|
| 89 |
-
def rank0_print(*a, **kw):
|
| 90 |
-
"""print() on the main process only.
|
| 91 |
-
|
| 92 |
-
Keyed off the RANK env var rather than accelerator.is_main_process so it also works
|
| 93 |
-
before the Accelerator is constructed -- dataset loading logs before that point. N
|
| 94 |
-
ranks echoing the same line is noise; N ranks echoing *different* lines is a
|
| 95 |
-
debugging trap, and the dataset banners are exactly where they would differ.
|
| 96 |
-
"""
|
| 97 |
-
if int(os.environ.get("RANK", 0)) == 0:
|
| 98 |
-
print(*a, **kw)
|
| 99 |
-
|
| 100 |
-
|
| 101 |
-
def resolve_precision(requested: str) -> str:
|
| 102 |
-
"""Pick a mixed-precision mode this GPU can actually run.
|
| 103 |
-
|
| 104 |
-
bf16 needs compute capability 8.0 (Ampere). Kaggle's T4 is Turing, 7.5 -- so the
|
| 105 |
-
trainer's bf16 default silently does not apply there, and a run that looks fine is
|
| 106 |
-
quietly not using tensor cores as intended. Fall back to fp16 (which Turing does have
|
| 107 |
-
tensor cores for) and say so, rather than let the first multi-GPU session discover it.
|
| 108 |
-
"""
|
| 109 |
-
# Accelerate spells "no mixed precision" as "no"; "fp32" is not a value it accepts.
|
| 110 |
-
# Passing it through raises ValueError, which means --precision fp32 has never worked
|
| 111 |
-
# and a CPU-only box (where the old code forced "fp32") could never start at all.
|
| 112 |
-
if not torch.cuda.is_available() or os.environ.get("ACCELERATE_USE_CPU"):
|
| 113 |
-
return "no"
|
| 114 |
-
if requested == "fp32":
|
| 115 |
-
return "no"
|
| 116 |
-
if requested == "bf16" and not torch.cuda.is_bf16_supported():
|
| 117 |
-
cap = torch.cuda.get_device_capability(0)
|
| 118 |
-
rank0_print(f"WARNING: {torch.cuda.get_device_name(0)} is compute capability "
|
| 119 |
-
f"{cap[0]}.{cap[1]}; bf16 needs 8.0+. Falling back to fp16 "
|
| 120 |
-
f"(Accelerate adds a GradScaler). Pass --precision fp32 to opt out.")
|
| 121 |
-
return "fp16"
|
| 122 |
-
return requested
|
| 123 |
-
|
| 124 |
-
|
| 125 |
-
def unpack_batch(batch, device):
|
| 126 |
-
"""Normalise a batch from either data path into the tensors the loss expects.
|
| 127 |
-
|
| 128 |
-
The shard loader ships the legal mask as a ``(B, 64)`` uint16 index list — 64 KB per
|
| 129 |
-
batch of 512 against 2.4 MB for the dense ``(64, 73)`` bool mask — so the mask is
|
| 130 |
-
rebuilt here, on device, after the H2D copy. Doing it in the dataset would put the
|
| 131 |
-
2.4 MB straight back on the wire and undo the entire point of the shard format.
|
| 132 |
-
"""
|
| 133 |
-
if "legal_idx" in batch:
|
| 134 |
-
legal_planes = scatter_legal_planes(batch["legal_idx"].to(device, non_blocking=True))
|
| 135 |
-
else:
|
| 136 |
-
legal_planes = batch["legal_moves_planes"].to(device, non_blocking=True)
|
| 137 |
-
return {
|
| 138 |
-
"board": batch["position"].to(device, non_blocking=True).long(),
|
| 139 |
-
"player": batch["is_white"].to(device, non_blocking=True).long(),
|
| 140 |
-
"castling": batch["castling_rights"].to(device, non_blocking=True).long(),
|
| 141 |
-
"en_passant": batch["en_passant_file"].to(device, non_blocking=True).long(),
|
| 142 |
-
"from_sq": batch["from_square"].to(device, non_blocking=True).long(),
|
| 143 |
-
"action_plane": batch["action_plane"].to(device, non_blocking=True).long(),
|
| 144 |
-
"legal_planes": legal_planes,
|
| 145 |
-
# Narrow on the wire (int8/int16 out of the shard), widened here so both data
|
| 146 |
-
# paths hand compute_loss exactly the same dtypes.
|
| 147 |
-
"result": batch["result"].to(device, non_blocking=True).long(),
|
| 148 |
-
# Must stay bool: compute_loss uses it as a *mask* (`target_value[white_win &
|
| 149 |
-
# is_white]`). An int tensor there would silently become fancy indexing and
|
| 150 |
-
# scatter the value targets onto the wrong rows.
|
| 151 |
-
"is_white": batch["is_white"].to(device, non_blocking=True).bool(),
|
| 152 |
-
"move_number": batch["move_number"].to(device, non_blocking=True).long(),
|
| 153 |
-
}
|
| 154 |
-
|
| 155 |
-
|
| 156 |
-
def get_next_run_number(log_dir: str) -> int:
|
| 157 |
-
log_path = Path(log_dir)
|
| 158 |
-
if not log_path.exists():
|
| 159 |
-
return 1
|
| 160 |
-
run_numbers = []
|
| 161 |
-
for d in log_path.iterdir():
|
| 162 |
-
if d.is_dir() and d.name.startswith("run_"):
|
| 163 |
-
try:
|
| 164 |
-
run_numbers.append(int(d.name.split("_")[1]))
|
| 165 |
-
except (IndexError, ValueError):
|
| 166 |
-
continue
|
| 167 |
-
return max(run_numbers, default=0) + 1
|
| 168 |
-
|
| 169 |
-
|
| 170 |
-
def point_project_config_at(project_config, run_dir) -> None:
|
| 171 |
-
"""Move both project_dir and logging_dir onto the resolved run directory.
|
| 172 |
-
|
| 173 |
-
Not ``set_directories(run_dir)``: that fills logging_dir only when it is None, and it is
|
| 174 |
-
not. ``ProjectConfiguration.__post_init__`` already defaulted it to whatever project_dir
|
| 175 |
-
the Accelerator was built with -- here the *parent*, logs/pos2move_v2, because the run
|
| 176 |
-
name is not known until after the process group exists (see main()). Moving project_dir
|
| 177 |
-
alone leaves the TensorBoard writer on the parent, where it appends the project name, so
|
| 178 |
-
every run's events pile into one shared logs/pos2move_v2/pos2move_v2 instead of landing
|
| 179 |
-
in the run directory beside that run's checkpoints.
|
| 180 |
-
"""
|
| 181 |
-
project_config.project_dir = str(run_dir)
|
| 182 |
-
project_config.logging_dir = str(run_dir)
|
| 183 |
-
|
| 184 |
-
|
| 185 |
-
def compute_loss(
|
| 186 |
-
move_logits,
|
| 187 |
-
value,
|
| 188 |
-
from_sq,
|
| 189 |
-
action_plane,
|
| 190 |
-
legal_moves_planes,
|
| 191 |
-
result,
|
| 192 |
-
is_white,
|
| 193 |
-
move_number,
|
| 194 |
-
value_loss_weight,
|
| 195 |
-
value_rampup_halfmoves=40,
|
| 196 |
-
label_smoothing: float = 0.0,
|
| 197 |
-
):
|
| 198 |
-
"""Total loss = (label-smoothed) policy CE + value MSE.
|
| 199 |
-
|
| 200 |
-
Returns
|
| 201 |
-
-------
|
| 202 |
-
total_loss : scalar tensor (the only thing that gets backprop'd)
|
| 203 |
-
metrics : dict of detached scalar tensors for logging
|
| 204 |
-
- ce : standard cross-entropy on the full action space
|
| 205 |
-
- legal_ce : cross-entropy after masking to legal moves only
|
| 206 |
-
- value_loss : weighted MSE on the value head
|
| 207 |
-
- acc : top-1 accuracy on the full action space
|
| 208 |
-
- legal_acc : top-1 accuracy after masking to legal moves only
|
| 209 |
-
"""
|
| 210 |
-
B = move_logits.size(0)
|
| 211 |
-
|
| 212 |
-
# Flatten (B, 64, 73) -> (B, 64*73). Upcast to fp32 for loss stability under bf16.
|
| 213 |
-
flat_logits = move_logits.reshape(B, -1).float()
|
| 214 |
-
flat_target = from_sq * NUM_ACTION_PLANES + action_plane
|
| 215 |
-
flat_mask = legal_moves_planes.reshape(B, -1).float()
|
| 216 |
-
|
| 217 |
-
# ── Policy loss: label-smoothed CE over LEGAL moves only ────────────
|
| 218 |
-
if label_smoothing > 0:
|
| 219 |
-
eps = label_smoothing
|
| 220 |
-
legal_count = flat_mask.sum(dim=-1, keepdim=True).clamp(min=1.0)
|
| 221 |
-
smooth = eps / legal_count
|
| 222 |
-
target_dist = flat_mask * smooth
|
| 223 |
-
idx = flat_target.unsqueeze(1)
|
| 224 |
-
target_dist.scatter_add_(1, idx, torch.full_like(idx, 1.0 - eps, dtype=target_dist.dtype))
|
| 225 |
-
log_pred = F.log_softmax(flat_logits, dim=-1)
|
| 226 |
-
policy_loss = F.kl_div(log_pred, target_dist, reduction="batchmean")
|
| 227 |
-
else:
|
| 228 |
-
policy_loss = F.cross_entropy(flat_logits, flat_target)
|
| 229 |
-
|
| 230 |
-
# ── Value loss: per-sample MSE, ramped up by move number ────────────
|
| 231 |
-
target_value = torch.zeros_like(value)
|
| 232 |
-
white_win = result == 1
|
| 233 |
-
black_win = result == 2
|
| 234 |
-
target_value[white_win & is_white] = 1.0
|
| 235 |
-
target_value[white_win & ~is_white] = -1.0
|
| 236 |
-
target_value[black_win & is_white] = -1.0
|
| 237 |
-
target_value[black_win & ~is_white] = 1.0
|
| 238 |
-
|
| 239 |
-
progress = (move_number.float() / value_rampup_halfmoves).clamp(max=1.0)
|
| 240 |
-
per_sample_value_loss = (value.float() - target_value) ** 2
|
| 241 |
-
value_loss = (progress * per_sample_value_loss).mean()
|
| 242 |
-
|
| 243 |
-
total_loss = policy_loss + value_loss_weight * value_loss
|
| 244 |
-
|
| 245 |
-
# ── Metrics (no grad) ───────────────────────────────────────────────
|
| 246 |
-
with torch.no_grad():
|
| 247 |
-
ce_metric = F.cross_entropy(flat_logits, flat_target)
|
| 248 |
-
|
| 249 |
-
masked_logits = flat_logits.masked_fill(flat_mask == 0, float("-inf"))
|
| 250 |
-
legal_ce_metric = F.cross_entropy(masked_logits, flat_target)
|
| 251 |
-
|
| 252 |
-
preds = flat_logits.argmax(dim=-1)
|
| 253 |
-
acc = (preds == flat_target).float().mean()
|
| 254 |
-
|
| 255 |
-
legal_preds = masked_logits.argmax(dim=-1)
|
| 256 |
-
legal_acc = (legal_preds == flat_target).float().mean()
|
| 257 |
-
|
| 258 |
-
metrics = {
|
| 259 |
-
"ce": ce_metric,
|
| 260 |
-
"legal_ce": legal_ce_metric,
|
| 261 |
-
"value_loss": value_loss.detach(),
|
| 262 |
-
"acc": acc,
|
| 263 |
-
"legal_acc": legal_acc,
|
| 264 |
-
}
|
| 265 |
-
return total_loss, metrics
|
| 266 |
-
|
| 267 |
-
|
| 268 |
-
def create_lr_scheduler(optimizer, warmup_steps, total_steps, final_lr_ratio,
|
| 269 |
-
schedule="linear", warmdown_ratio=0.2):
|
| 270 |
-
"""Build the LR multiplier schedule.
|
| 271 |
-
|
| 272 |
-
``wsd`` -- warmup, then **stable at peak**, then a linear warmdown over the final
|
| 273 |
-
``warmdown_ratio`` of the run (nanochat's ``get_lr_multiplier``). ``linear`` -- warmup
|
| 274 |
-
then decay across the entire remainder, which is what this trainer used to do.
|
| 275 |
-
|
| 276 |
-
WSD is the default for two reasons. It decouples the horizon from the schedule: peak LR
|
| 277 |
-
is held flat, so a run can be stopped or extended at any point in the stable phase and
|
| 278 |
-
only the warmdown re-run, where a decay-from-peak schedule bakes ``total_steps`` into
|
| 279 |
-
every step it takes. And it is the phase structure Muon's momentum schedule was designed
|
| 280 |
-
against -- momentum warms down *during the LR warmdown*, which only exists here.
|
| 281 |
-
|
| 282 |
-
Note the asymmetry in defaults, which is deliberate: the CLI defaults to ``wsd`` for new
|
| 283 |
-
runs, but this function defaults to ``linear``. scheduler_config dicts persisted in
|
| 284 |
-
checkpoints from before this change carry no ``schedule`` key, and are replayed through
|
| 285 |
-
here on resume -- so the function default is what those runs get. Defaulting it to
|
| 286 |
-
``wsd`` would silently change the LR shape of an in-flight run at the point it resumed.
|
| 287 |
-
"""
|
| 288 |
-
warmup_steps = max(1, warmup_steps)
|
| 289 |
-
total_steps = max(warmup_steps + 1, total_steps)
|
| 290 |
-
warmdown = max(1, round(warmdown_ratio * total_steps))
|
| 291 |
-
|
| 292 |
-
def lr_lambda(step):
|
| 293 |
-
if step < warmup_steps:
|
| 294 |
-
return float(step + 1) / float(warmup_steps)
|
| 295 |
-
if schedule == "wsd":
|
| 296 |
-
if step <= total_steps - warmdown:
|
| 297 |
-
return 1.0
|
| 298 |
-
progress = max(0.0, min(1.0, (total_steps - step) / warmdown))
|
| 299 |
-
return progress * 1.0 + (1 - progress) * final_lr_ratio
|
| 300 |
-
progress = (step - warmup_steps + 1) / float(max(1, total_steps - warmup_steps))
|
| 301 |
-
return 1.0 - (1.0 - final_lr_ratio) * min(1.0, progress)
|
| 302 |
-
|
| 303 |
-
return LambdaLR(optimizer, lr_lambda)
|
| 304 |
-
|
| 305 |
-
|
| 306 |
-
def save_trainer_state(checkpoint_dir, data_pass, global_step, best_val_loss, scheduler_config=None):
|
| 307 |
-
# "epoch" is written alongside "data_pass" only so a checkpoint from this trainer still
|
| 308 |
-
# loads in anything that predates the step paradigm. Nothing here reads it back.
|
| 309 |
-
state = {"data_pass": data_pass, "epoch": data_pass,
|
| 310 |
-
"global_step": global_step, "best_val_loss": best_val_loss}
|
| 311 |
-
if scheduler_config:
|
| 312 |
-
state["scheduler_config"] = scheduler_config
|
| 313 |
-
with (Path(checkpoint_dir) / "trainer_state.json").open("w") as f:
|
| 314 |
-
json.dump(state, f)
|
| 315 |
-
|
| 316 |
-
|
| 317 |
-
def load_trainer_state(checkpoint_dir):
|
| 318 |
-
path = Path(checkpoint_dir) / "trainer_state.json"
|
| 319 |
-
if not path.exists():
|
| 320 |
-
return None
|
| 321 |
-
with path.open() as f:
|
| 322 |
-
return json.load(f)
|
| 323 |
-
|
| 324 |
-
|
| 325 |
-
# ── EMA helpers ──────────────────��───────────────────────────────────────
|
| 326 |
-
# EMA state is kept in fp32 to avoid bf16 precision loss on the
|
| 327 |
-
# (1 - decay) * param update term (which is ~0.001 * param at decay=0.999
|
| 328 |
-
# and would round to zero for many small values in bf16).
|
| 329 |
-
|
| 330 |
-
def create_ema_state(model):
|
| 331 |
-
return {name: p.data.detach().float().clone() for name, p in model.named_parameters()}
|
| 332 |
-
|
| 333 |
-
|
| 334 |
-
@torch.no_grad()
|
| 335 |
-
def update_ema(model, ema_state, decay):
|
| 336 |
-
for name, param in model.named_parameters():
|
| 337 |
-
# Cast live param to fp32 for the lerp; ema_state stays fp32.
|
| 338 |
-
ema_state[name].lerp_(param.data.float(), 1.0 - decay)
|
| 339 |
-
|
| 340 |
-
|
| 341 |
-
def swap_ema_weights(model, ema_state):
|
| 342 |
-
# Backup current (compute-dtype) weights into ema_state, load EMA weights
|
| 343 |
-
# into the model — cast to the model's dtype on the way in.
|
| 344 |
-
for name, param in model.named_parameters():
|
| 345 |
-
tmp = param.data.detach().float().clone()
|
| 346 |
-
param.data.copy_(ema_state[name].to(param.dtype))
|
| 347 |
-
ema_state[name].copy_(tmp)
|
| 348 |
-
|
| 349 |
-
|
| 350 |
-
def save_ema_state(ema_state, path):
|
| 351 |
-
torch.save(ema_state, path)
|
| 352 |
-
|
| 353 |
-
|
| 354 |
-
def load_ema_state(path, device):
|
| 355 |
-
return torch.load(path, map_location=device, weights_only=True)
|
| 356 |
-
|
| 357 |
-
|
| 358 |
-
def get_raw_model(model, accelerator):
|
| 359 |
-
m = accelerator.unwrap_model(model)
|
| 360 |
-
if hasattr(m, "_orig_mod"):
|
| 361 |
-
m = m._orig_mod
|
| 362 |
-
return m
|
| 363 |
-
|
| 364 |
-
|
| 365 |
-
def cleanup_old_checkpoints(checkpoint_dir, max_checkpoints):
|
| 366 |
-
checkpoints = sorted(
|
| 367 |
-
[d for d in Path(checkpoint_dir).iterdir() if d.is_dir() and d.name.startswith("checkpoint_step_")],
|
| 368 |
-
key=lambda x: int(x.name.split("_")[-1]),
|
| 369 |
-
)
|
| 370 |
-
while len(checkpoints) > max_checkpoints:
|
| 371 |
-
shutil.rmtree(checkpoints.pop(0))
|
| 372 |
-
|
| 373 |
-
def main():
|
| 374 |
-
default_data = Path(__file__).parents[3] / "data" / "elite_db.h5"
|
| 375 |
-
|
| 376 |
-
parser = argparse.ArgumentParser(description="Train Pos2MoveV2 (clean)")
|
| 377 |
-
# Data
|
| 378 |
-
parser.add_argument("--data", type=str, default=str(default_data))
|
| 379 |
-
parser.add_argument("--shards", type=str, default=None,
|
| 380 |
-
help="Directory of flat shards from scripts/build_shards.py. Bypasses "
|
| 381 |
-
"--data/--min-elo/--sample-weighting entirely: the sampling "
|
| 382 |
-
"distribution is frozen into the shard at build time.")
|
| 383 |
-
parser.add_argument("--min-elo", type=int, default=None)
|
| 384 |
-
parser.add_argument("--max-elo", type=int, default=None)
|
| 385 |
-
parser.add_argument("--sample-weighting", type=str, default="middlegame", choices=["uniform", "middlegame"])
|
| 386 |
-
parser.add_argument("--skip-opening-plies", type=int, default=0)
|
| 387 |
-
parser.add_argument("--max-val-samples", type=int, default=50_000,
|
| 388 |
-
help="Cap on val/test size. The HDF5 path has always applied this; "
|
| 389 |
-
"the shard path needs it too since a reserved shard is much larger.")
|
| 390 |
-
# Training
|
| 391 |
-
parser.add_argument("--batch-size", type=int, default=1024)
|
| 392 |
-
parser.add_argument("--grad-accum", type=int, default=4)
|
| 393 |
-
parser.add_argument("--epochs", type=int, default=None,
|
| 394 |
-
help="DEPRECATED and ignored. Training length is --max-steps. Kept so "
|
| 395 |
-
"existing invocations do not hard-fail; it warns and does nothing.")
|
| 396 |
-
parser.add_argument("--lr", type=float, default=1e-3)
|
| 397 |
-
parser.add_argument("--lr-muon", type=float, default=2e-3)
|
| 398 |
-
parser.add_argument("--lr-embedding", type=float, default=4e-4)
|
| 399 |
-
parser.add_argument("--lr-head", type=float, default=None)
|
| 400 |
-
parser.add_argument("--warmup-steps", type=int, default=2000)
|
| 401 |
-
parser.add_argument("--final-lr-ratio", type=float, default=0.05)
|
| 402 |
-
parser.add_argument("--lr-schedule", type=str, default="wsd", choices=["wsd", "linear"],
|
| 403 |
-
help="'wsd' = warmup / stable at peak / linear warmdown over the last "
|
| 404 |
-
"--warmdown-ratio of the run. 'linear' = the previous behaviour, "
|
| 405 |
-
"decay from peak across the whole remainder.")
|
| 406 |
-
parser.add_argument("--warmdown-ratio", type=float, default=0.2,
|
| 407 |
-
help="Fraction of the run spent decaying, for --lr-schedule wsd. Muon's "
|
| 408 |
-
"momentum warmdown is aligned to the same window.")
|
| 409 |
-
parser.add_argument("--max-grad-norm", type=float, default=1.0)
|
| 410 |
-
parser.add_argument("--weight-decay", type=float, default=0.1)
|
| 411 |
-
# Loss weights
|
| 412 |
-
parser.add_argument("--value-loss-weight", type=float, default=5.0)
|
| 413 |
-
parser.add_argument("--label-smoothing", type=float, default=0.1)
|
| 414 |
-
# Model
|
| 415 |
-
parser.add_argument("--preset", type=str, default="base", choices=list(PRESETS),
|
| 416 |
-
help="Model size. See PRESETS for per-GPU DDP memory. Individual "
|
| 417 |
-
"--embed-dim/--num-layers/--num-heads override the preset.")
|
| 418 |
-
parser.add_argument("--embed-dim", type=int, default=None)
|
| 419 |
-
parser.add_argument("--num-layers", type=int, default=None)
|
| 420 |
-
parser.add_argument("--num-heads", type=int, default=None)
|
| 421 |
-
parser.add_argument("--dropout", type=float, default=0.05)
|
| 422 |
-
parser.add_argument("--layer-drop", type=float, default=0.1,
|
| 423 |
-
help="Stochastic depth rate. Linearly scales from 0 (first layer) to this value (last layer). Try 0.1.") # Checkpointing
|
| 424 |
-
parser.add_argument("--max-steps", type=int, default=50_000,
|
| 425 |
-
help="Total optimizer steps to train for. This is THE training-length "
|
| 426 |
-
"knob: it bounds the loop and sets the LR decay horizon, and it "
|
| 427 |
-
"means the same thing on both data paths and at any world size.")
|
| 428 |
-
parser.add_argument("--save-every", type=int, default=None,
|
| 429 |
-
help="DEPRECATED and ignored (was epoch-based checkpointing). "
|
| 430 |
-
"Use --save-steps.")
|
| 431 |
-
parser.add_argument("--save-steps", type=int, default=5000)
|
| 432 |
-
parser.add_argument("--eval-steps", type=int, default=1000,
|
| 433 |
-
help="Validate every N optimizer steps. Val loss selects the best "
|
| 434 |
-
"checkpoint, so this sets the resolution of that choice. 0 "
|
| 435 |
-
"disables periodic validation (a final one still runs).")
|
| 436 |
-
parser.add_argument("--max-checkpoints", type=int, default=5)
|
| 437 |
-
parser.add_argument("--resume-from", type=str, default=None)
|
| 438 |
-
# EMA
|
| 439 |
-
parser.add_argument("--ema-decay", type=float, default=0.999)
|
| 440 |
-
# Muon
|
| 441 |
-
parser.add_argument("--distributed-muon", action=argparse.BooleanOptionalAction, default=True,
|
| 442 |
-
help="Partition Newton-Schulz across ranks by whole parameter, so the "
|
| 443 |
-
"NS work and momentum state are divided instead of every rank "
|
| 444 |
-
"redundantly computing the same thing. No-op at world size 1.")
|
| 445 |
-
parser.add_argument("--muon-schedules", action=argparse.BooleanOptionalAction, default=True,
|
| 446 |
-
help="Schedule Muon momentum (0.85->0.97, down to 0.90 during warmdown) "
|
| 447 |
-
"and cosine-decay its weight decay to zero, as nanochat does.")
|
| 448 |
-
# Misc
|
| 449 |
-
parser.add_argument("--seed", type=int, default=42)
|
| 450 |
-
parser.add_argument("--precision", type=str, default="bf16", choices=["bf16", "fp16", "fp32"])
|
| 451 |
-
parser.add_argument("--num-workers", type=int, default=12)
|
| 452 |
-
# store_true with default=True could never be turned off. BooleanOptionalAction
|
| 453 |
-
# gives a real --no-compile, which CPU/gloo testing and debugging both need.
|
| 454 |
-
parser.add_argument("--compile", action=argparse.BooleanOptionalAction, default=True)
|
| 455 |
-
parser.add_argument("--compile-mode", type=str, default="default",
|
| 456 |
-
choices=["default", "reduce-overhead", "max-autotune"])
|
| 457 |
-
parser.add_argument("--compile-order", type=str, default="auto",
|
| 458 |
-
choices=["auto", "before-prepare", "after-prepare"],
|
| 459 |
-
help="Whether torch.compile runs before or after accelerator.prepare(). "
|
| 460 |
-
"'auto' = before on a single process (the ordering the 2.2x "
|
| 461 |
-
"single-GPU speedup was measured with), after under DDP so "
|
| 462 |
-
"Dynamo's DDPOptimizer engages. See the note at the call site.")
|
| 463 |
-
args = parser.parse_args()
|
| 464 |
-
|
| 465 |
-
# Epochs are gone as a unit of training length. They were never comparable between the
|
| 466 |
-
# two data paths — the HDF5 dataset is indexed by *game* (738 optimizer steps/epoch) and
|
| 467 |
-
# the shard dataset by *sample* (11,495 for elite_k16, 50,505 for full_k4) — so the same
|
| 468 |
-
# --epochs N bought a 15-68x different amount of training, LR decay horizon and
|
| 469 |
-
# validation cadence depending only on which --data/--shards flag you passed.
|
| 470 |
-
for dead, replacement in (("epochs", "--max-steps"), ("save_every", "--save-steps")):
|
| 471 |
-
if getattr(args, dead) is not None:
|
| 472 |
-
rank0_print(f"WARNING: --{dead.replace('_', '-')} is deprecated and ignored; "
|
| 473 |
-
f"use {replacement}.")
|
| 474 |
-
if args.max_steps <= 0:
|
| 475 |
-
parser.error("--max-steps must be > 0")
|
| 476 |
-
|
| 477 |
-
# Preset supplies the geometry; an explicit flag always wins over it.
|
| 478 |
-
preset = PRESETS[args.preset]
|
| 479 |
-
for flag, key in (("embed_dim", "embed_dim"), ("num_layers", "num_layers"),
|
| 480 |
-
("num_heads", "num_heads")):
|
| 481 |
-
if getattr(args, flag) is None:
|
| 482 |
-
setattr(args, flag, preset[key])
|
| 483 |
-
|
| 484 |
-
set_seed(args.seed)
|
| 485 |
-
random.seed(args.seed)
|
| 486 |
-
np.random.seed(args.seed)
|
| 487 |
-
torch.manual_seed(args.seed)
|
| 488 |
-
torch.cuda.manual_seed_all(args.seed)
|
| 489 |
-
|
| 490 |
-
# ── Logging ──────────────────────────────────────────────────────────
|
| 491 |
-
# The run directory has to be decided by ONE rank and told to the others. Every rank
|
| 492 |
-
# scanning logs/ for the next free run number is a race: they can pick the same number
|
| 493 |
-
# (two ranks writing one run) or different ones (N runs, N-1 of them silently empty),
|
| 494 |
-
# and mkdir(exist_ok=False) then crashes whichever rank loses. The timestamp in the
|
| 495 |
-
# name makes independent derivation impossible too -- ranks start milliseconds apart.
|
| 496 |
-
#
|
| 497 |
-
# Ordering note: the Accelerator must exist before the broadcast (it is what sets up
|
| 498 |
-
# the process group), but init_trackers needs the run dir. Hence build it against the
|
| 499 |
-
# parent, then point it at the resolved run dir with set_directories().
|
| 500 |
-
log_dir = Path("logs") / "pos2move_v2"
|
| 501 |
-
log_dir.mkdir(parents=True, exist_ok=True)
|
| 502 |
-
|
| 503 |
-
precision = resolve_precision(args.precision)
|
| 504 |
-
accelerator = accelerate.Accelerator(
|
| 505 |
-
log_with="tensorboard",
|
| 506 |
-
project_dir=str(log_dir),
|
| 507 |
-
mixed_precision=precision,
|
| 508 |
-
gradient_accumulation_steps=args.grad_accum,
|
| 509 |
-
)
|
| 510 |
-
|
| 511 |
-
if accelerator.is_main_process:
|
| 512 |
-
run_number = get_next_run_number(str(log_dir))
|
| 513 |
-
run_name = f"run_{run_number:03d}_{datetime.now().strftime('%Y%m%d_%H%M%S')}"
|
| 514 |
-
else:
|
| 515 |
-
run_name = None
|
| 516 |
-
run_name = broadcast_object_list([run_name], from_process=0)[0]
|
| 517 |
-
|
| 518 |
-
log_path = log_dir / run_name
|
| 519 |
-
checkpoint_dir = log_path / "checkpoints"
|
| 520 |
-
if accelerator.is_main_process:
|
| 521 |
-
checkpoint_dir.mkdir(parents=True, exist_ok=False)
|
| 522 |
-
# Every rank must see the directory before anyone writes a checkpoint into it.
|
| 523 |
-
accelerator.wait_for_everyone()
|
| 524 |
-
point_project_config_at(accelerator.project_configuration, log_path)
|
| 525 |
-
|
| 526 |
-
# ── Dataset ──────────────────────────────────────────────────────────
|
| 527 |
-
use_shards = args.shards is not None
|
| 528 |
-
if use_shards:
|
| 529 |
-
# Flat memmap shards: the replay, tokenize and legal-move enumeration were all
|
| 530 |
-
# paid once at build time, and the val/test split is by whole shard (== whole
|
| 531 |
-
# game range) rather than by sample, so no game straddles the split.
|
| 532 |
-
train_set = FlatShardDataset(args.shards, "train")
|
| 533 |
-
val_set = FlatShardDataset(args.shards, "val")
|
| 534 |
-
test_set = FlatShardDataset(args.shards, "test")
|
| 535 |
-
|
| 536 |
-
# Whole reserved shards hold far more than validation needs (478k samples at
|
| 537 |
-
# K=16). Cap them the way the HDF5 path does, with a fixed seed so val loss stays
|
| 538 |
-
# comparable across runs — it is what selects the best checkpoint.
|
| 539 |
-
def cap(ds, name):
|
| 540 |
-
if len(ds) <= args.max_val_samples:
|
| 541 |
-
return ds
|
| 542 |
-
g = np.random.default_rng(0xC0FFEE)
|
| 543 |
-
keep = np.sort(g.choice(len(ds), size=args.max_val_samples, replace=False))
|
| 544 |
-
rank0_print(f" {name}: capped {len(ds):,} -> {args.max_val_samples:,} samples")
|
| 545 |
-
return Subset(ds, keep.tolist())
|
| 546 |
-
|
| 547 |
-
val_set = cap(val_set, "val")
|
| 548 |
-
test_set = cap(test_set, "test")
|
| 549 |
-
|
| 550 |
-
# The shard loaders bypass accelerator.prepare() (see the note at prepare()), so
|
| 551 |
-
# rank splitting is NOT automatic here the way it is for the HDF5 path -- it has to
|
| 552 |
-
# be asked for. Without this every rank draws the identical batches from the same
|
| 553 |
-
# seeded RandomSampler, and DDP degenerates into N ranks computing one gradient N
|
| 554 |
-
# times: the loss curve looks healthy, the effective batch never grows, and the
|
| 555 |
-
# extra GPUs buy nothing.
|
| 556 |
-
# This is exactly why the Accelerator is constructed above the dataset section:
|
| 557 |
-
# DistributedSampler requires an initialised process group, and the Accelerator is
|
| 558 |
-
# what initialises it. Building the loaders first raises "Default process group has
|
| 559 |
-
# not been initialized".
|
| 560 |
-
world_size = accelerator.num_processes
|
| 561 |
-
train_loader = make_shard_dataloader(train_set, batch_size=args.batch_size, shuffle=True,
|
| 562 |
-
num_workers=args.num_workers, drop_last=True,
|
| 563 |
-
seed=args.seed, distributed=world_size > 1)
|
| 564 |
-
# Val/test are deliberately NOT sharded. Each rank evaluates the whole (capped)
|
| 565 |
-
# set and therefore computes an identical val loss, so best-model selection agrees
|
| 566 |
-
# on every rank by construction and needs no gather or padding-trim. The cost is
|
| 567 |
-
# N x redundant work on <=10k samples -- about two seconds -- against a class of
|
| 568 |
-
# bug where ranks disagree about which checkpoint is best and race to write it.
|
| 569 |
-
val_loader = make_shard_dataloader(val_set, batch_size=args.batch_size, shuffle=False,
|
| 570 |
-
num_workers=args.num_workers)
|
| 571 |
-
test_loader = make_shard_dataloader(test_set, batch_size=args.batch_size, shuffle=False,
|
| 572 |
-
num_workers=args.num_workers)
|
| 573 |
-
vocab_size = PostionTokenizer().vocab_size
|
| 574 |
-
rank0_print(f"Shards: {args.shards} (K={train_set.meta['k']}, "
|
| 575 |
-
f"mode={train_set.meta['sample_mode']}, seed={train_set.meta['seed']})")
|
| 576 |
-
else:
|
| 577 |
-
dataset = HDF5ChessDataset(
|
| 578 |
-
hdf5_path=args.data,
|
| 579 |
-
min_elo=args.min_elo,
|
| 580 |
-
max_elo=args.max_elo,
|
| 581 |
-
sample_weighting=args.sample_weighting,
|
| 582 |
-
skip_opening_plies=args.skip_opening_plies,
|
| 583 |
-
)
|
| 584 |
-
|
| 585 |
-
val_size = min(int(0.1 * len(dataset)), args.max_val_samples)
|
| 586 |
-
test_size = min(int(0.1 * len(dataset)), args.max_val_samples)
|
| 587 |
-
train_size = len(dataset) - val_size - test_size
|
| 588 |
-
train_set, val_set, test_set = random_split(dataset, [train_size, val_size, test_size])
|
| 589 |
-
|
| 590 |
-
# pin_memory lets the H2D copy run on the DMA engine and overlap with compute;
|
| 591 |
-
# without it `.to(device, non_blocking=True)` is silently synchronous.
|
| 592 |
-
train_loader = DataLoader(train_set, batch_size=args.batch_size, shuffle=True,
|
| 593 |
-
num_workers=args.num_workers, drop_last=True,
|
| 594 |
-
persistent_workers=True, prefetch_factor=4, pin_memory=True,
|
| 595 |
-
worker_init_fn=shard_worker_init_fn)
|
| 596 |
-
val_loader = DataLoader(val_set, batch_size=args.batch_size, shuffle=False,
|
| 597 |
-
num_workers=args.num_workers, pin_memory=True,
|
| 598 |
-
worker_init_fn=shard_worker_init_fn)
|
| 599 |
-
test_loader = DataLoader(test_set, batch_size=args.batch_size, shuffle=False,
|
| 600 |
-
num_workers=args.num_workers, pin_memory=True,
|
| 601 |
-
worker_init_fn=shard_worker_init_fn)
|
| 602 |
-
vocab_size = dataset.position_tokenizer.vocab_size
|
| 603 |
-
|
| 604 |
-
rank0_print(f"Train: {len(train_set):,} | Val: {len(val_set):,} | Test: {len(test_set):,}")
|
| 605 |
-
|
| 606 |
-
# ── Model ────────────────────────────────────────────────────────────
|
| 607 |
-
model_config = {
|
| 608 |
-
"vocab_size": vocab_size,
|
| 609 |
-
"embed_dim": args.embed_dim,
|
| 610 |
-
"nb_transformer_layers": args.num_layers,
|
| 611 |
-
"num_heads": args.num_heads,
|
| 612 |
-
"dropout": args.dropout,
|
| 613 |
-
"kvq_bias": False,
|
| 614 |
-
"layer_drop": args.layer_drop,
|
| 615 |
-
}
|
| 616 |
-
if accelerator.is_main_process:
|
| 617 |
-
with (log_path / "model_config.json").open("w") as f:
|
| 618 |
-
json.dump(model_config, f, indent=2)
|
| 619 |
-
|
| 620 |
-
# Effective batch scales with world size: every rank contributes its own micro-batch
|
| 621 |
-
# to the same all-reduced gradient. Reporting it without num_processes understates the
|
| 622 |
-
# real batch by N, which is exactly the number linear LR scaling is derived from.
|
| 623 |
-
effective_bs = args.batch_size * args.grad_accum * accelerator.num_processes
|
| 624 |
-
rank0_print(f"Effective batch size: {effective_bs} (micro={args.batch_size} × "
|
| 625 |
-
f"accum={args.grad_accum} × ranks={accelerator.num_processes})")
|
| 626 |
-
accelerator.init_trackers(
|
| 627 |
-
project_name="pos2move_v2",
|
| 628 |
-
config={
|
| 629 |
-
"batch_size": args.batch_size,
|
| 630 |
-
"grad_accum": args.grad_accum,
|
| 631 |
-
"effective_batch_size": effective_bs,
|
| 632 |
-
"max_steps": args.max_steps,
|
| 633 |
-
"lr": args.lr,
|
| 634 |
-
"weight_decay": args.weight_decay,
|
| 635 |
-
"value_loss_weight": args.value_loss_weight,
|
| 636 |
-
**model_config,
|
| 637 |
-
},
|
| 638 |
-
)
|
| 639 |
-
|
| 640 |
-
device = accelerator.device
|
| 641 |
-
rank0_print(f"Device: {device} | Precision: {precision} | Logging to: {log_path}")
|
| 642 |
-
|
| 643 |
-
model = Pos2MoveV2(**model_config)
|
| 644 |
-
total_params = sum(p.numel() for p in model.parameters())
|
| 645 |
-
rank0_print(f"Model parameters: {total_params:,}")
|
| 646 |
-
model.to(device)
|
| 647 |
-
|
| 648 |
-
# torch.compile vs accelerator.prepare() ordering is not cosmetic under DDP.
|
| 649 |
-
#
|
| 650 |
-
# Dynamo's DDPOptimizer (torch._dynamo.backends.distributed, on by default) splits the
|
| 651 |
-
# graph at DDP's gradient all-reduce bucket boundaries so the collectives can overlap
|
| 652 |
-
# with backward compute. Its own docs: "DDPOptimizer applies when dynamo compiles
|
| 653 |
-
# models wrapped in DistributedDataParallel". Compiling first gives DDP(OptimizedModule)
|
| 654 |
-
# -- the wrap happens outside the compiled graph, DDPOptimizer never sees a DDP module,
|
| 655 |
-
# and the overlap is lost. Compiling after prepare() gives compile(DDP(model)), which is
|
| 656 |
-
# the shape it wants.
|
| 657 |
-
#
|
| 658 |
-
# Single process has no buckets to align to, so 'before' is kept there: it is the
|
| 659 |
-
# ordering the measured 2.2x speedup and the compile-time numbers come from.
|
| 660 |
-
compile_order = args.compile_order
|
| 661 |
-
if compile_order == "auto":
|
| 662 |
-
compile_order = "after-prepare" if accelerator.num_processes > 1 else "before-prepare"
|
| 663 |
-
|
| 664 |
-
def maybe_compile(m, when):
|
| 665 |
-
if args.compile and compile_order == when:
|
| 666 |
-
rank0_print(f"torch.compile (mode={args.compile_mode}, {when})")
|
| 667 |
-
return torch.compile(m, mode=args.compile_mode, dynamic=False) # shapes are static
|
| 668 |
-
return m
|
| 669 |
-
|
| 670 |
-
model = maybe_compile(model, "before-prepare")
|
| 671 |
-
|
| 672 |
-
# ── Optimizer (Muon for 2D weights, AdamW for embeddings/heads/1D) ──
|
| 673 |
-
lr_emb = args.lr_embedding or args.lr
|
| 674 |
-
lr_head = args.lr_head or args.lr
|
| 675 |
-
|
| 676 |
-
muon_params = []
|
| 677 |
-
adamw_emb_params = []
|
| 678 |
-
adamw_head_params = []
|
| 679 |
-
adamw_other_params = []
|
| 680 |
-
|
| 681 |
-
for name, param in model.named_parameters():
|
| 682 |
-
if "embedding" in name:
|
| 683 |
-
adamw_emb_params.append(param)
|
| 684 |
-
elif "move_head" in name or "value_head" in name:
|
| 685 |
-
adamw_head_params.append(param)
|
| 686 |
-
elif param.ndim == 2:
|
| 687 |
-
muon_params.append(param)
|
| 688 |
-
else:
|
| 689 |
-
adamw_other_params.append(param)
|
| 690 |
-
|
| 691 |
-
rank0_print(f"LR: emb={lr_emb:.2e} | muon={args.lr_muon:.2e} | head={lr_head:.2e}")
|
| 692 |
-
rank0_print(
|
| 693 |
-
f"Params: emb={sum(p.numel() for p in adamw_emb_params):,} | "
|
| 694 |
-
f"muon={sum(p.numel() for p in muon_params):,} | "
|
| 695 |
-
f"adamw_other={sum(p.numel() for p in adamw_other_params):,} | "
|
| 696 |
-
f"head={sum(p.numel() for p in adamw_head_params):,}"
|
| 697 |
-
)
|
| 698 |
-
|
| 699 |
-
# DistributedMuon degenerates to exactly torch.optim.Muon at world size 1 (verified in
|
| 700 |
-
# tests/test_distributed_muon.py), so there is one code path rather than two.
|
| 701 |
-
muon_cls = DistributedMuon if args.distributed_muon else torch.optim.Muon
|
| 702 |
-
muon_optimizer = muon_cls(
|
| 703 |
-
muon_params,
|
| 704 |
-
lr=args.lr_muon,
|
| 705 |
-
momentum=0.95,
|
| 706 |
-
weight_decay=args.weight_decay,
|
| 707 |
-
)
|
| 708 |
-
if args.distributed_muon and accelerator.num_processes > 1:
|
| 709 |
-
rank0_print(muon_optimizer.sharding_summary().split(" | ")[0])
|
| 710 |
-
adamw_optimizer = torch.optim.AdamW([
|
| 711 |
-
{"params": adamw_emb_params, "lr": lr_emb, "weight_decay": args.weight_decay},
|
| 712 |
-
{"params": adamw_other_params, "lr": args.lr, "weight_decay": 0.0},
|
| 713 |
-
{"params": adamw_head_params, "lr": lr_head, "weight_decay": 0.0},
|
| 714 |
-
])
|
| 715 |
-
|
| 716 |
-
if use_shards:
|
| 717 |
-
# The shard loaders are deliberately kept out of prepare(). They run with
|
| 718 |
-
# batch_size=None and a BatchSampler so that a whole index list reaches
|
| 719 |
-
# __getitem__ in one call; Accelerate reads batch_size=None as "this iterable
|
| 720 |
-
# already yields batches" and would re-shard at the wrong granularity. Rank
|
| 721 |
-
# splitting is done by the DistributedSampler inside make_shard_dataloader, and
|
| 722 |
-
# the H2D copy by unpack_batch.
|
| 723 |
-
model, muon_optimizer, adamw_optimizer = accelerator.prepare(
|
| 724 |
-
model, muon_optimizer, adamw_optimizer
|
| 725 |
-
)
|
| 726 |
-
else:
|
| 727 |
-
model, muon_optimizer, adamw_optimizer, train_loader, val_loader, test_loader = accelerator.prepare(
|
| 728 |
-
model, muon_optimizer, adamw_optimizer, train_loader, val_loader, test_loader
|
| 729 |
-
)
|
| 730 |
-
|
| 731 |
-
model = maybe_compile(model, "after-prepare")
|
| 732 |
-
|
| 733 |
-
# ── EMA ──────────────────────────────────────────────────────────────
|
| 734 |
-
use_ema = args.ema_decay > 0
|
| 735 |
-
ema_state = None
|
| 736 |
-
if use_ema:
|
| 737 |
-
ema_state = create_ema_state(get_raw_model(model, accelerator))
|
| 738 |
-
rank0_print(f"EMA enabled (decay={args.ema_decay})")
|
| 739 |
-
|
| 740 |
-
# ── Resume ───────────────────────────────────────────────────────────
|
| 741 |
-
best_val_loss = float("inf")
|
| 742 |
-
start_pass = 0
|
| 743 |
-
global_step = 0
|
| 744 |
-
trainer_state = None
|
| 745 |
-
|
| 746 |
-
if args.resume_from:
|
| 747 |
-
trainer_state = load_trainer_state(args.resume_from)
|
| 748 |
-
if trainer_state:
|
| 749 |
-
# "epoch" is the pre-step-paradigm key; read it so old checkpoints still resume.
|
| 750 |
-
start_pass = trainer_state.get("data_pass", trainer_state.get("epoch", 0)) + 1
|
| 751 |
-
global_step = trainer_state["global_step"]
|
| 752 |
-
best_val_loss = trainer_state["best_val_loss"]
|
| 753 |
-
|
| 754 |
-
# ── Scheduler ────────────────────────────────────────────────────────
|
| 755 |
-
total_steps = args.max_steps
|
| 756 |
-
scheduler_config = {
|
| 757 |
-
"warmup_steps": args.warmup_steps,
|
| 758 |
-
"total_steps": total_steps,
|
| 759 |
-
"final_lr_ratio": args.final_lr_ratio,
|
| 760 |
-
"schedule": args.lr_schedule,
|
| 761 |
-
"warmdown_ratio": args.warmdown_ratio,
|
| 762 |
-
}
|
| 763 |
-
if trainer_state and "scheduler_config" in trainer_state:
|
| 764 |
-
# A resume keeps the schedule it started with, so the LR curve stays continuous.
|
| 765 |
-
# That means --max-steps still bounds the *loop* but no longer sets the LR horizon,
|
| 766 |
-
# and the two can silently disagree. Say so rather than let it be discovered later.
|
| 767 |
-
scheduler_config = trainer_state["scheduler_config"]
|
| 768 |
-
resumed_total = scheduler_config.get("total_steps")
|
| 769 |
-
if resumed_total != args.max_steps:
|
| 770 |
-
rank0_print(f"WARNING: resuming with the checkpoint's LR schedule "
|
| 771 |
-
f"(total_steps={resumed_total:,}) while --max-steps={args.max_steps:,} "
|
| 772 |
-
f"bounds the loop. The LR decays over {resumed_total:,} steps regardless.")
|
| 773 |
-
total_steps = resumed_total
|
| 774 |
-
|
| 775 |
-
# An "epoch" is not comparable between the two data paths: the HDF5 dataset is indexed
|
| 776 |
-
# by *game* (738 optimizer steps/epoch) and the shard dataset by *sample* (11.5k for
|
| 777 |
-
# elite_k16, 50.5k for full_k4). So --epochs N silently buys a 15-68x longer LR decay
|
| 778 |
-
# on shards, and epoch-end validation fires that much less often. Print both so the
|
| 779 |
-
# schedule is never a surprise, and say so out loud when --max-steps is not pinning it.
|
| 780 |
-
steps_per_pass = len(train_loader) // max(1, args.grad_accum)
|
| 781 |
-
samples_seen = args.max_steps * effective_bs
|
| 782 |
-
warmdown_note = (f"warmdown {round(args.warmdown_ratio * total_steps):,} | "
|
| 783 |
-
if args.lr_schedule == "wsd" else "")
|
| 784 |
-
rank0_print(f"Schedule: {total_steps:,} optimizer steps | {args.lr_schedule} | "
|
| 785 |
-
f"warmup {args.warmup_steps:,} | {warmdown_note}"
|
| 786 |
-
f"eval every {args.eval_steps:,} | save every {args.save_steps:,}")
|
| 787 |
-
rank0_print(f" one pass over the training data = {steps_per_pass:,} steps/rank "
|
| 788 |
-
f"({args.max_steps / max(1, steps_per_pass):.2f} passes, "
|
| 789 |
-
f"{samples_seen:,} samples consumed)")
|
| 790 |
-
|
| 791 |
-
muon_scheduler = create_lr_scheduler(muon_optimizer, **scheduler_config)
|
| 792 |
-
adamw_scheduler = create_lr_scheduler(adamw_optimizer, **scheduler_config)
|
| 793 |
-
accelerator.register_for_checkpointing(muon_scheduler, adamw_scheduler)
|
| 794 |
-
|
| 795 |
-
if args.resume_from:
|
| 796 |
-
accelerator.load_state(args.resume_from)
|
| 797 |
-
if use_ema:
|
| 798 |
-
ema_path = Path(args.resume_from) / "ema_state.pt"
|
| 799 |
-
if ema_path.exists():
|
| 800 |
-
ema_state = load_ema_state(ema_path, device)
|
| 801 |
-
rank0_print(f"Restored EMA state from {ema_path}")
|
| 802 |
-
else:
|
| 803 |
-
ema_state = create_ema_state(get_raw_model(model, accelerator))
|
| 804 |
-
rank0_print("EMA state not found in checkpoint, re-initialized from model")
|
| 805 |
-
rank0_print(f"Resumed from {args.resume_from} (step {global_step:,}/{args.max_steps:,}, "
|
| 806 |
-
f"data pass {start_pass})")
|
| 807 |
-
|
| 808 |
-
# ── Validation ───────────────────────────────────────────────────────
|
| 809 |
-
# Runs on the --eval-steps counter. It used to run only at epoch end, which was a fine
|
| 810 |
-
# cadence on the HDF5 path (738 steps/epoch) and a bad one on shards (11,495 steps for
|
| 811 |
-
# elite_k16, 50,505 for full_k4) — the dataset index is a sample there, not a game. Val
|
| 812 |
-
# loss selects the best checkpoint, so that gap mattered. Hence the step paradigm.
|
| 813 |
-
|
| 814 |
-
def run_validation():
|
| 815 |
-
"""Evaluate on val_loader (under EMA weights if enabled). Restores train mode."""
|
| 816 |
-
model.eval()
|
| 817 |
-
unwrapped = get_raw_model(model, accelerator)
|
| 818 |
-
if use_ema:
|
| 819 |
-
swap_ema_weights(unwrapped, ema_state)
|
| 820 |
-
|
| 821 |
-
# No cross-rank reduction here on purpose: val_loader is replicated, not sharded,
|
| 822 |
-
# so every rank walks the identical set and arrives at the identical number. That
|
| 823 |
-
# is what keeps best-model selection consistent across ranks without a gather.
|
| 824 |
-
sums = dict(loss=0.0, ce=0.0, legal_ce=0.0, value=0.0, correct=0, legal_correct=0, total=0)
|
| 825 |
-
with torch.no_grad():
|
| 826 |
-
for batch in tqdm(val_loader, desc="Validation", leave=False,
|
| 827 |
-
disable=not accelerator.is_main_process):
|
| 828 |
-
b = unpack_batch(batch, device)
|
| 829 |
-
move_logits, value = model(b["board"], b["player"], b["castling"], b["en_passant"])
|
| 830 |
-
loss, metrics = compute_loss(
|
| 831 |
-
move_logits, value,
|
| 832 |
-
b["from_sq"], b["action_plane"],
|
| 833 |
-
b["legal_planes"], b["result"], b["is_white"], b["move_number"],
|
| 834 |
-
args.value_loss_weight,
|
| 835 |
-
)
|
| 836 |
-
B = move_logits.size(0)
|
| 837 |
-
sums["loss"] += loss.item() * B
|
| 838 |
-
sums["ce"] += metrics["ce"].item() * B
|
| 839 |
-
sums["legal_ce"] += metrics["legal_ce"].item() * B
|
| 840 |
-
sums["value"] += metrics["value_loss"].item() * B
|
| 841 |
-
sums["correct"] += int(metrics["acc"].item() * B)
|
| 842 |
-
sums["legal_correct"] += int(metrics["legal_acc"].item() * B)
|
| 843 |
-
sums["total"] += B
|
| 844 |
-
|
| 845 |
-
if use_ema:
|
| 846 |
-
swap_ema_weights(unwrapped, ema_state)
|
| 847 |
-
model.train()
|
| 848 |
-
|
| 849 |
-
n = max(1, sums["total"])
|
| 850 |
-
return {
|
| 851 |
-
"val/loss": sums["loss"] / n,
|
| 852 |
-
"val/ce": sums["ce"] / n,
|
| 853 |
-
"val/legal_ce": sums["legal_ce"] / n,
|
| 854 |
-
"val/value_loss": sums["value"] / n,
|
| 855 |
-
"val/accuracy": sums["correct"] / n,
|
| 856 |
-
"val/legal_accuracy": sums["legal_correct"] / n,
|
| 857 |
-
}
|
| 858 |
-
|
| 859 |
-
def record_validation(vals, data_pass, extra_log=None, label=""):
|
| 860 |
-
"""Log, print, and update best-model selection. Returns True if it was a new best."""
|
| 861 |
-
nonlocal best_val_loss
|
| 862 |
-
log_dict = dict(extra_log or {})
|
| 863 |
-
log_dict.update(vals)
|
| 864 |
-
if use_ema:
|
| 865 |
-
log_dict["val/ema_loss"] = vals["val/loss"]
|
| 866 |
-
log_dict["val/ema_accuracy"] = vals["val/accuracy"]
|
| 867 |
-
accelerator.log(log_dict, step=global_step)
|
| 868 |
-
|
| 869 |
-
ema_tag = " (EMA)" if use_ema else ""
|
| 870 |
-
rank0_print(f"{label}val_loss={vals['val/loss']:.4f}{ema_tag} "
|
| 871 |
-
f"| val_acc={vals['val/accuracy']:.4f} "
|
| 872 |
-
f"| val_legal_acc={vals['val/legal_accuracy']:.4f}")
|
| 873 |
-
|
| 874 |
-
if vals["val/loss"] < best_val_loss:
|
| 875 |
-
best_val_loss = vals["val/loss"]
|
| 876 |
-
best_path = checkpoint_dir / "best_model"
|
| 877 |
-
accelerator.save_state(str(best_path))
|
| 878 |
-
save_trainer_state(best_path, data_pass, global_step, best_val_loss, scheduler_config)
|
| 879 |
-
if use_ema and accelerator.is_main_process:
|
| 880 |
-
save_ema_state(ema_state, best_path / "ema_state.pt")
|
| 881 |
-
rank0_print(f" -> New best model (val_loss={vals['val/loss']:.4f})")
|
| 882 |
-
return True
|
| 883 |
-
return False
|
| 884 |
-
|
| 885 |
-
# ── Training loop (step-driven) ──────────────────────────────────────
|
| 886 |
-
# The loop is bounded by optimizer steps and cycles the dataloader as many times as
|
| 887 |
-
# that takes. Nothing downstream keys off "epoch" any more: LR decay, validation and
|
| 888 |
-
# checkpointing are all on step counters, which mean the same thing whichever dataset
|
| 889 |
-
# is mounted and at any world size.
|
| 890 |
-
|
| 891 |
-
def start_pass_shuffle(loader, n):
|
| 892 |
-
"""Reshuffle for a fresh pass over the data.
|
| 893 |
-
|
| 894 |
-
DistributedSampler needs set_epoch or every rank replays the identical order on
|
| 895 |
-
every pass. RandomSampler carries its generator across iterators and reshuffles on
|
| 896 |
-
its own, so this is a no-op there.
|
| 897 |
-
"""
|
| 898 |
-
for obj in (loader,
|
| 899 |
-
getattr(loader, "sampler", None),
|
| 900 |
-
getattr(getattr(loader, "sampler", None), "sampler", None)):
|
| 901 |
-
if obj is not None and hasattr(obj, "set_epoch"):
|
| 902 |
-
obj.set_epoch(n)
|
| 903 |
-
return
|
| 904 |
-
|
| 905 |
-
def new_window():
|
| 906 |
-
"""Train metrics accumulated since the last validation, sample-weighted."""
|
| 907 |
-
return dict(loss=0.0, ce=0.0, legal_ce=0.0, value=0.0, acc=0.0, legal_acc=0.0, n=0)
|
| 908 |
-
|
| 909 |
-
WINDOW_KEYS = ("loss", "ce", "legal_ce", "value", "acc", "legal_acc")
|
| 910 |
-
|
| 911 |
-
def flush_window(w):
|
| 912 |
-
"""Average the window across ranks, not just within one.
|
| 913 |
-
|
| 914 |
-
Each rank trains on a different slice, so these sums are rank-local: rank 0's
|
| 915 |
-
numbers alone are a 1/N sample of the step. Reducing sum-of-(metric x batch) and
|
| 916 |
-
sum-of-batch separately, then dividing, is exact regardless of how unevenly the
|
| 917 |
-
last batches fall -- which is why this does not need gather_for_metrics and its
|
| 918 |
-
padding-trim bookkeeping (that machinery also would not work here, since the shard
|
| 919 |
-
loaders never went through prepare()).
|
| 920 |
-
"""
|
| 921 |
-
vals = torch.tensor([w[k] for k in WINDOW_KEYS] + [float(w["n"])],
|
| 922 |
-
dtype=torch.float64, device=accelerator.device)
|
| 923 |
-
if accelerator.num_processes > 1:
|
| 924 |
-
vals = accelerator.reduce(vals, reduction="sum")
|
| 925 |
-
total = max(1.0, vals[-1].item())
|
| 926 |
-
return {f"train/{'value_loss' if k == 'value' else k}": vals[i].item() / total
|
| 927 |
-
for i, k in enumerate(WINDOW_KEYS)}
|
| 928 |
-
|
| 929 |
-
last_val_step = -1
|
| 930 |
-
|
| 931 |
-
def validate_now(window, data_pass, label_prefix):
|
| 932 |
-
"""Flush the train window, validate, log both against the same step.
|
| 933 |
-
|
| 934 |
-
No-ops if this step has already been validated, which happens whenever max_steps is
|
| 935 |
-
an exact multiple of eval_steps and the final validation lands on the same step as
|
| 936 |
-
the last periodic one. Worth guarding rather than tolerating: on the HDF5 path the
|
| 937 |
-
two calls do not even agree, because ``HDF5ChessDataset.__getitem__`` samples a
|
| 938 |
-
fresh ply per call, so its val set re-rolls on every pass. (The shard val set is
|
| 939 |
-
fixed, so shard val loss is comparable across evaluations and HDF5 val loss is not
|
| 940 |
-
— worth remembering when reading either curve.)
|
| 941 |
-
"""
|
| 942 |
-
nonlocal last_val_step
|
| 943 |
-
if global_step == last_val_step:
|
| 944 |
-
return
|
| 945 |
-
last_val_step = global_step
|
| 946 |
-
extra = flush_window(window) if window["n"] else {}
|
| 947 |
-
extra["data_pass"] = data_pass
|
| 948 |
-
train_tag = f"train_loss={extra['train/loss']:.4f} | " if window["n"] else ""
|
| 949 |
-
record_validation(run_validation(), data_pass, extra_log=extra,
|
| 950 |
-
label=f"{label_prefix}{train_tag}")
|
| 951 |
-
|
| 952 |
-
model.train()
|
| 953 |
-
window = new_window()
|
| 954 |
-
data_pass = start_pass
|
| 955 |
-
pbar = tqdm(total=args.max_steps, initial=global_step, desc="train", unit="step",
|
| 956 |
-
disable=not accelerator.is_main_process)
|
| 957 |
-
|
| 958 |
-
while global_step < args.max_steps:
|
| 959 |
-
start_pass_shuffle(train_loader, data_pass)
|
| 960 |
-
for batch in train_loader:
|
| 961 |
-
with accelerator.accumulate(model):
|
| 962 |
-
b = unpack_batch(batch, device)
|
| 963 |
-
board, player = b["board"], b["player"]
|
| 964 |
-
castling, en_passant = b["castling"], b["en_passant"]
|
| 965 |
-
from_sq, action_plane = b["from_sq"], b["action_plane"]
|
| 966 |
-
legal_planes, result = b["legal_planes"], b["result"]
|
| 967 |
-
is_white, move_number = b["is_white"], b["move_number"]
|
| 968 |
-
|
| 969 |
-
move_logits, value = model(board, player, castling, en_passant)
|
| 970 |
-
loss, metrics = compute_loss(
|
| 971 |
-
move_logits, value,
|
| 972 |
-
from_sq, action_plane,
|
| 973 |
-
legal_planes, result, is_white, move_number,
|
| 974 |
-
args.value_loss_weight,
|
| 975 |
-
label_smoothing=args.label_smoothing,
|
| 976 |
-
)
|
| 977 |
-
|
| 978 |
-
accelerator.backward(loss)
|
| 979 |
-
if accelerator.sync_gradients and args.max_grad_norm > 0:
|
| 980 |
-
accelerator.clip_grad_norm_(model.parameters(), args.max_grad_norm)
|
| 981 |
-
if args.muon_schedules:
|
| 982 |
-
# Momentum is not a constant in a well-tuned Muon run: a long memory
|
| 983 |
-
# early averages over a fast-moving gradient, and a slightly shorter one
|
| 984 |
-
# late helps the run settle. Weight decay cosine-decays to zero.
|
| 985 |
-
mom = muon_momentum(global_step, args.max_steps,
|
| 986 |
-
warmdown_ratio=args.warmdown_ratio)
|
| 987 |
-
wdk = muon_weight_decay(global_step, args.max_steps, args.weight_decay)
|
| 988 |
-
for g in muon_optimizer.param_groups:
|
| 989 |
-
g["momentum"], g["weight_decay"] = mom, wdk
|
| 990 |
-
muon_optimizer.step()
|
| 991 |
-
adamw_optimizer.step()
|
| 992 |
-
if accelerator.sync_gradients:
|
| 993 |
-
muon_scheduler.step()
|
| 994 |
-
adamw_scheduler.step()
|
| 995 |
-
muon_optimizer.zero_grad()
|
| 996 |
-
adamw_optimizer.zero_grad()
|
| 997 |
-
|
| 998 |
-
# EMA is redundant under DDP but not wrong, and that is worth stating.
|
| 999 |
-
# Gradients are all-reduced before the step, so every rank holds identical
|
| 1000 |
-
# params and computes an identical EMA from an identical decay -- no drift.
|
| 1001 |
-
# swap_ema_weights in run_validation is likewise symmetric because every
|
| 1002 |
-
# rank validates (val_loader is replicated, not sharded). The failure mode
|
| 1003 |
-
# to avoid is any rank taking a different branch here; sync_gradients is
|
| 1004 |
-
# world-uniform, so none does. tests/test_distributed.py asserts it.
|
| 1005 |
-
if use_ema and accelerator.sync_gradients:
|
| 1006 |
-
update_ema(get_raw_model(model, accelerator), ema_state, args.ema_decay)
|
| 1007 |
-
|
| 1008 |
-
bs = board.size(0)
|
| 1009 |
-
window["loss"] += loss.item() * bs
|
| 1010 |
-
window["ce"] += metrics["ce"].item() * bs
|
| 1011 |
-
window["legal_ce"] += metrics["legal_ce"].item() * bs
|
| 1012 |
-
window["value"] += metrics["value_loss"].item() * bs
|
| 1013 |
-
window["acc"] += metrics["acc"].item() * bs
|
| 1014 |
-
window["legal_acc"] += metrics["legal_acc"].item() * bs
|
| 1015 |
-
window["n"] += bs
|
| 1016 |
-
|
| 1017 |
-
pbar.set_postfix(
|
| 1018 |
-
loss=f"{loss.item():.4f}",
|
| 1019 |
-
ce=f"{metrics['ce'].item():.4f}",
|
| 1020 |
-
acc=f"{metrics['acc'].item():.3f}",
|
| 1021 |
-
val_l=f"{metrics['value_loss'].item():.4f}",
|
| 1022 |
-
)
|
| 1023 |
-
|
| 1024 |
-
if not accelerator.sync_gradients:
|
| 1025 |
-
continue
|
| 1026 |
-
|
| 1027 |
-
muon_lrs = muon_scheduler.get_last_lr()
|
| 1028 |
-
adamw_lrs = adamw_scheduler.get_last_lr()
|
| 1029 |
-
accelerator.log(
|
| 1030 |
-
{
|
| 1031 |
-
"train/step_loss": loss.item(),
|
| 1032 |
-
"train/step_ce": metrics["ce"].item(),
|
| 1033 |
-
"train/step_legal_ce": metrics["legal_ce"].item(),
|
| 1034 |
-
"train/step_value_loss": metrics["value_loss"].item(),
|
| 1035 |
-
"train/step_acc": metrics["acc"].item(),
|
| 1036 |
-
"train/step_legal_acc": metrics["legal_acc"].item(),
|
| 1037 |
-
"train/lr_muon": muon_lrs[0],
|
| 1038 |
-
"train/lr_emb": adamw_lrs[0],
|
| 1039 |
-
"train/lr_head": adamw_lrs[2],
|
| 1040 |
-
**({"train/muon_momentum": muon_optimizer.param_groups[0]["momentum"],
|
| 1041 |
-
"train/muon_wd": muon_optimizer.param_groups[0]["weight_decay"]}
|
| 1042 |
-
if args.muon_schedules else {}),
|
| 1043 |
-
},
|
| 1044 |
-
step=global_step,
|
| 1045 |
-
)
|
| 1046 |
-
|
| 1047 |
-
global_step += 1
|
| 1048 |
-
pbar.update(1)
|
| 1049 |
-
|
| 1050 |
-
if args.save_steps > 0 and global_step % args.save_steps == 0:
|
| 1051 |
-
ckpt = checkpoint_dir / f"checkpoint_step_{global_step:07d}"
|
| 1052 |
-
accelerator.save_state(str(ckpt))
|
| 1053 |
-
save_trainer_state(ckpt, data_pass, global_step, best_val_loss, scheduler_config)
|
| 1054 |
-
if use_ema and accelerator.is_main_process:
|
| 1055 |
-
save_ema_state(ema_state, ckpt / "ema_state.pt")
|
| 1056 |
-
if accelerator.is_main_process:
|
| 1057 |
-
rank0_print(f"\n Saved step checkpoint at step {global_step}")
|
| 1058 |
-
cleanup_old_checkpoints(checkpoint_dir, args.max_checkpoints)
|
| 1059 |
-
|
| 1060 |
-
# Validation is on a step counter, and global_step is identical on every rank,
|
| 1061 |
-
# so all ranks enter the collective forward together.
|
| 1062 |
-
if args.eval_steps > 0 and global_step % args.eval_steps == 0:
|
| 1063 |
-
validate_now(window, data_pass,
|
| 1064 |
-
f"\nStep {global_step:,}/{args.max_steps:,}: ")
|
| 1065 |
-
window = new_window()
|
| 1066 |
-
|
| 1067 |
-
if global_step >= args.max_steps:
|
| 1068 |
-
break
|
| 1069 |
-
|
| 1070 |
-
data_pass += 1
|
| 1071 |
-
|
| 1072 |
-
pbar.close()
|
| 1073 |
-
|
| 1074 |
-
# Always finish on a validation, so the final steps can still win best_model even when
|
| 1075 |
-
# max_steps is not a multiple of eval_steps.
|
| 1076 |
-
rank0_print(f"\nReached max-steps ({args.max_steps:,}) after {data_pass} pass(es) over the data.")
|
| 1077 |
-
validate_now(window, data_pass, f"Final (step {global_step:,}): ")
|
| 1078 |
-
|
| 1079 |
-
# ── Final test (using EMA weights) ───────────────────────────────────
|
| 1080 |
-
rank0_print("\nFinal test evaluation...")
|
| 1081 |
-
model.eval()
|
| 1082 |
-
if use_ema:
|
| 1083 |
-
swap_ema_weights(get_raw_model(model, accelerator), ema_state)
|
| 1084 |
-
test_loss_sum = 0.0
|
| 1085 |
-
test_correct = 0
|
| 1086 |
-
test_legal_correct = 0
|
| 1087 |
-
test_total = 0
|
| 1088 |
-
|
| 1089 |
-
with torch.no_grad():
|
| 1090 |
-
for batch in tqdm(test_loader, desc="Testing", leave=False,
|
| 1091 |
-
disable=not accelerator.is_main_process):
|
| 1092 |
-
b = unpack_batch(batch, device)
|
| 1093 |
-
board, player = b["board"], b["player"]
|
| 1094 |
-
castling, en_passant = b["castling"], b["en_passant"]
|
| 1095 |
-
from_sq, action_plane = b["from_sq"], b["action_plane"]
|
| 1096 |
-
legal_planes, result = b["legal_planes"], b["result"]
|
| 1097 |
-
is_white, move_number = b["is_white"], b["move_number"]
|
| 1098 |
-
|
| 1099 |
-
move_logits, value = model(board, player, castling, en_passant)
|
| 1100 |
-
loss, metrics = compute_loss(
|
| 1101 |
-
move_logits, value,
|
| 1102 |
-
from_sq, action_plane,
|
| 1103 |
-
legal_planes, result, is_white, move_number,
|
| 1104 |
-
args.value_loss_weight,
|
| 1105 |
-
)
|
| 1106 |
-
|
| 1107 |
-
B = move_logits.size(0)
|
| 1108 |
-
test_loss_sum += loss.item() * B
|
| 1109 |
-
test_correct += int(metrics["acc"].item() * B)
|
| 1110 |
-
test_legal_correct += int(metrics["legal_acc"].item() * B)
|
| 1111 |
-
test_total += B
|
| 1112 |
-
|
| 1113 |
-
test_loss = test_loss_sum / test_total
|
| 1114 |
-
test_acc = test_correct / test_total
|
| 1115 |
-
test_legal_acc = test_legal_correct / test_total
|
| 1116 |
-
|
| 1117 |
-
accelerator.log(
|
| 1118 |
-
{"test/loss": test_loss, "test/accuracy": test_acc, "test/legal_accuracy": test_legal_acc},
|
| 1119 |
-
step=global_step,
|
| 1120 |
-
)
|
| 1121 |
-
rank0_print(f"Test: loss={test_loss:.4f} | accuracy={test_acc:.4f} | legal_accuracy={test_legal_acc:.4f}")
|
| 1122 |
-
|
| 1123 |
-
accelerator.end_training()
|
| 1124 |
-
rank0_print(f"\nTraining complete. Logs: {log_path}")
|
| 1125 |
-
|
| 1126 |
-
|
| 1127 |
-
if __name__ == "__main__":
|
| 1128 |
-
main()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
chesstransformer/trainers/position2move_trainer.py
DELETED
|
@@ -1,682 +0,0 @@
|
|
| 1 |
-
#!/usr/bin/env python3
|
| 2 |
-
"""
|
| 3 |
-
Training script for Position2Move transformer model.
|
| 4 |
-
|
| 5 |
-
This script trains a transformer to predict chess moves from board positions.
|
| 6 |
-
Uses the new efficient HDF5 dataset format with game-based storage and random sampling.
|
| 7 |
-
|
| 8 |
-
Usage:
|
| 9 |
-
python src/chesstransformer/trainers/position2move_trainer.py --data data/Lichess2017.h5
|
| 10 |
-
|
| 11 |
-
Options:
|
| 12 |
-
--data: Path to HDF5 dataset
|
| 13 |
-
--min-elo: Minimum average ELO for filtering games
|
| 14 |
-
--max-elo: Maximum average ELO for filtering games
|
| 15 |
-
--batch-size: Batch size (default: 32)
|
| 16 |
-
--epochs: Number of epochs (default: 10)
|
| 17 |
-
--lr: Learning rate (default: 3e-4)
|
| 18 |
-
--save-every: Save checkpoint every N epochs (default: 5)
|
| 19 |
-
"""
|
| 20 |
-
|
| 21 |
-
from pathlib import Path
|
| 22 |
-
from datetime import datetime
|
| 23 |
-
import argparse
|
| 24 |
-
import copy
|
| 25 |
-
import random
|
| 26 |
-
import numpy as np
|
| 27 |
-
import json
|
| 28 |
-
|
| 29 |
-
from tqdm.auto import tqdm
|
| 30 |
-
import torch
|
| 31 |
-
from torch.nn import functional as F
|
| 32 |
-
from torch.utils.data import DataLoader, random_split
|
| 33 |
-
import accelerate
|
| 34 |
-
from accelerate.utils import set_seed
|
| 35 |
-
from torch.optim.lr_scheduler import LambdaLR
|
| 36 |
-
|
| 37 |
-
from chesstransformer.datasets.h5_lichess_dataset import HDF5ChessDataset
|
| 38 |
-
from chesstransformer.models.transformer.position2move import Position2MoveModel
|
| 39 |
-
|
| 40 |
-
|
| 41 |
-
def get_next_run_number(log_dir="logs"):
|
| 42 |
-
"""Get the next run number by checking existing run directories."""
|
| 43 |
-
log_path = Path(log_dir)
|
| 44 |
-
if not log_path.exists():
|
| 45 |
-
return 1
|
| 46 |
-
|
| 47 |
-
existing_runs = [d.name for d in log_path.iterdir() if d.is_dir() and d.name.startswith("run_")]
|
| 48 |
-
|
| 49 |
-
if not existing_runs:
|
| 50 |
-
return 1
|
| 51 |
-
|
| 52 |
-
run_numbers = []
|
| 53 |
-
for run_name in existing_runs:
|
| 54 |
-
try:
|
| 55 |
-
num = int(run_name.split("_")[1])
|
| 56 |
-
run_numbers.append(num)
|
| 57 |
-
except (IndexError, ValueError):
|
| 58 |
-
continue
|
| 59 |
-
|
| 60 |
-
return max(run_numbers) + 1 if run_numbers else 1
|
| 61 |
-
|
| 62 |
-
|
| 63 |
-
def compute_loss(logits, moves, legal_moves_mask, beta, legal_move_loss_weight):
|
| 64 |
-
llm_loss = F.cross_entropy(logits, moves)
|
| 65 |
-
illegal_prob = F.softmax(logits, dim=-1) * (1.0 - legal_moves_mask.float())
|
| 66 |
-
legal_move_loss = torch.sum(illegal_prob**2, dim=-1).mean()
|
| 67 |
-
filtered_logits = torch.where(legal_moves_mask.bool(), logits, -torch.inf)
|
| 68 |
-
legal_move_cross_entropy = F.cross_entropy(filtered_logits, moves)
|
| 69 |
-
loss = beta * llm_loss + legal_move_loss_weight * legal_move_loss + (1 - beta) * legal_move_cross_entropy
|
| 70 |
-
return loss, llm_loss, legal_move_loss, legal_move_cross_entropy
|
| 71 |
-
|
| 72 |
-
|
| 73 |
-
def save_trainer_state(checkpoint_dir, epoch, global_step, best_val_loss, scheduler_config=None):
|
| 74 |
-
state = {
|
| 75 |
-
"epoch": epoch,
|
| 76 |
-
"global_step": global_step,
|
| 77 |
-
"best_val_loss": best_val_loss,
|
| 78 |
-
}
|
| 79 |
-
if scheduler_config:
|
| 80 |
-
state["scheduler_config"] = scheduler_config
|
| 81 |
-
state_path = Path(checkpoint_dir) / "trainer_state.json"
|
| 82 |
-
with state_path.open("w", encoding="utf-8") as fp:
|
| 83 |
-
json.dump(state, fp)
|
| 84 |
-
|
| 85 |
-
|
| 86 |
-
def load_trainer_state(checkpoint_dir):
|
| 87 |
-
state_path = Path(checkpoint_dir) / "trainer_state.json"
|
| 88 |
-
if not state_path.exists():
|
| 89 |
-
return None
|
| 90 |
-
with state_path.open("r", encoding="utf-8") as fp:
|
| 91 |
-
return json.load(fp)
|
| 92 |
-
|
| 93 |
-
|
| 94 |
-
def cleanup_old_step_checkpoints(checkpoint_dir, max_checkpoints):
|
| 95 |
-
"""Remove old step checkpoints, keeping only the most recent ones."""
|
| 96 |
-
checkpoint_path = Path(checkpoint_dir)
|
| 97 |
-
step_checkpoints = sorted(
|
| 98 |
-
[d for d in checkpoint_path.iterdir() if d.is_dir() and d.name.startswith("checkpoint_step_")],
|
| 99 |
-
key=lambda x: int(x.name.split("_")[-1]),
|
| 100 |
-
)
|
| 101 |
-
|
| 102 |
-
# Remove oldest checkpoints if we exceed max_checkpoints
|
| 103 |
-
while len(step_checkpoints) > max_checkpoints:
|
| 104 |
-
oldest = step_checkpoints.pop(0)
|
| 105 |
-
import shutil
|
| 106 |
-
|
| 107 |
-
shutil.rmtree(oldest)
|
| 108 |
-
print(f" Removed old checkpoint: {oldest.name}")
|
| 109 |
-
|
| 110 |
-
|
| 111 |
-
def create_linear_warmup_decay_scheduler(optimizer, warmup_steps, total_steps, final_lr_ratio):
|
| 112 |
-
final_lr_ratio = max(0.0, final_lr_ratio)
|
| 113 |
-
warmup_steps = max(1, warmup_steps)
|
| 114 |
-
total_steps = max(warmup_steps + 1, total_steps)
|
| 115 |
-
|
| 116 |
-
def lr_lambda(current_step):
|
| 117 |
-
if current_step < warmup_steps:
|
| 118 |
-
return float(current_step + 1) / float(warmup_steps)
|
| 119 |
-
progress = (current_step - warmup_steps + 1) / float(max(1, total_steps - warmup_steps))
|
| 120 |
-
return 1.0 - (1.0 - final_lr_ratio) * min(1.0, progress)
|
| 121 |
-
|
| 122 |
-
return LambdaLR(optimizer, lr_lambda)
|
| 123 |
-
|
| 124 |
-
|
| 125 |
-
def find_best_learning_rate(
|
| 126 |
-
model, train_loader, accelerator, lr_min=1e-6, lr_max=1e-2, num_steps=100, beta=0.7, legal_move_loss_weight=10.0
|
| 127 |
-
):
|
| 128 |
-
model.train()
|
| 129 |
-
device = accelerator.device
|
| 130 |
-
initial_state = copy.deepcopy(model.state_dict())
|
| 131 |
-
# Use base learning rate for LR finder
|
| 132 |
-
optimizer = torch.optim.AdamW(model.parameters(), lr=lr_min)
|
| 133 |
-
multiplier = (lr_max / lr_min) ** (1 / max(num_steps - 1, 1))
|
| 134 |
-
best_lr = lr_min
|
| 135 |
-
best_loss = float("inf")
|
| 136 |
-
lr = lr_min
|
| 137 |
-
data_iter = iter(train_loader)
|
| 138 |
-
|
| 139 |
-
for step in range(num_steps):
|
| 140 |
-
try:
|
| 141 |
-
batch = next(data_iter)
|
| 142 |
-
except StopIteration:
|
| 143 |
-
data_iter = iter(train_loader)
|
| 144 |
-
batch = next(data_iter)
|
| 145 |
-
|
| 146 |
-
optimizer.zero_grad()
|
| 147 |
-
positions = batch["position"].to(device)
|
| 148 |
-
moves = batch["move"].to(device)
|
| 149 |
-
is_white = batch["is_white"].to(device)
|
| 150 |
-
legal_moves_mask = batch["legal_moves_mask"].to(device)
|
| 151 |
-
castling_rights = batch["castling_rights"].to(device)
|
| 152 |
-
en_passant_file = batch["en_passant_file"].to(device)
|
| 153 |
-
halfmove_clock = batch["halfmove_clock"].to(device)
|
| 154 |
-
|
| 155 |
-
logits = model(positions, is_white, castling_rights, en_passant_file, halfmove_clock)
|
| 156 |
-
loss, llm_loss, legal_move_loss, legal_move_cross_entropy = compute_loss(
|
| 157 |
-
logits, moves, legal_moves_mask, beta, legal_move_loss_weight
|
| 158 |
-
)
|
| 159 |
-
|
| 160 |
-
accelerator.backward(loss)
|
| 161 |
-
optimizer.step()
|
| 162 |
-
|
| 163 |
-
loss_value = accelerator.gather_for_metrics(loss.detach()).mean().item()
|
| 164 |
-
if loss_value < best_loss:
|
| 165 |
-
best_loss = loss_value
|
| 166 |
-
best_lr = lr
|
| 167 |
-
|
| 168 |
-
lr *= multiplier
|
| 169 |
-
for param_group in optimizer.param_groups:
|
| 170 |
-
param_group["lr"] = lr
|
| 171 |
-
|
| 172 |
-
model.load_state_dict(initial_state)
|
| 173 |
-
model.zero_grad(set_to_none=True)
|
| 174 |
-
accelerator.free_memory()
|
| 175 |
-
return best_lr, best_loss
|
| 176 |
-
|
| 177 |
-
|
| 178 |
-
def main():
|
| 179 |
-
default_data_path = Path(__file__).parents[2] / "data" / "Lichess2017.h5"
|
| 180 |
-
|
| 181 |
-
parser = argparse.ArgumentParser(description="Train Position2Move Transformer")
|
| 182 |
-
parser.add_argument("--data", type=str, default=default_data_path, help="Path to HDF5 dataset")
|
| 183 |
-
parser.add_argument("--min-elo", type=int, default=None, help="Minimum average ELO for filtering games")
|
| 184 |
-
parser.add_argument("--max-elo", type=int, default=None, help="Maximum average ELO for filtering games")
|
| 185 |
-
parser.add_argument("--batch-size", type=int, default=32, help="Batch size")
|
| 186 |
-
parser.add_argument("--epochs", type=int, default=10, help="Number of epochs")
|
| 187 |
-
parser.add_argument("--lr", type=float, default=3e-4, help="Learning rate for transformers (base LR)")
|
| 188 |
-
parser.add_argument(
|
| 189 |
-
"--lr-embedding", type=float, default=None, help="Learning rate for embeddings (default: same as --lr)"
|
| 190 |
-
)
|
| 191 |
-
parser.add_argument(
|
| 192 |
-
"--lr-head", type=float, default=None, help="Learning rate for output head (default: same as --lr)"
|
| 193 |
-
)
|
| 194 |
-
parser.add_argument("--save-every", type=int, default=5, help="Save checkpoint every N epochs")
|
| 195 |
-
parser.add_argument("--lr-find", action="store_true", help="Run learning rate finder before training")
|
| 196 |
-
parser.add_argument("--lr-min", type=float, default=1e-6, help="Minimum LR for finder sweep")
|
| 197 |
-
parser.add_argument("--lr-max", type=float, default=1e-2, help="Maximum LR for finder sweep")
|
| 198 |
-
parser.add_argument("--lr-steps", type=int, default=100, help="Number of steps for LR finder sweep")
|
| 199 |
-
parser.add_argument("--beta", type=float, default=0.7, help="Weight for LLM loss in combined loss function")
|
| 200 |
-
parser.add_argument(
|
| 201 |
-
"--legal-move-loss-weight",
|
| 202 |
-
type=float,
|
| 203 |
-
default=10.0,
|
| 204 |
-
help="Weight for legal move loss in combined loss function",
|
| 205 |
-
)
|
| 206 |
-
parser.add_argument("--random-seed", type=int, default=42, help="Random seed for reproducibility")
|
| 207 |
-
parser.add_argument("--warmup-steps", type=int, default=1000, help="Number of warmup steps for LR scheduler")
|
| 208 |
-
parser.add_argument("--final-lr-ratio", type=float, default=0.01, help="Final LR multiplier after linear decay")
|
| 209 |
-
parser.add_argument("--resume-from", type=str, default=None, help="Path to checkpoint directory to resume from")
|
| 210 |
-
parser.add_argument(
|
| 211 |
-
"--max-grad-norm", type=float, default=1.0, help="Maximum gradient norm for clipping (0 to disable)"
|
| 212 |
-
)
|
| 213 |
-
parser.add_argument(
|
| 214 |
-
"--max_examples", type=int, default=None, help="Maximum number of examples to use from the dataset"
|
| 215 |
-
)
|
| 216 |
-
parser.add_argument("--save-steps", type=int, default=5000, help="Save checkpoint every N steps")
|
| 217 |
-
parser.add_argument(
|
| 218 |
-
"--max-checkpoints",
|
| 219 |
-
type=int,
|
| 220 |
-
default=5,
|
| 221 |
-
help="Maximum number of step checkpoints to keep (excluding best model)",
|
| 222 |
-
)
|
| 223 |
-
parser.add_argument(
|
| 224 |
-
"--precision", type=str, default="fp16", choices=["bf16", "fp16", "fp32"], help="Precision for training"
|
| 225 |
-
)
|
| 226 |
-
args = parser.parse_args()
|
| 227 |
-
|
| 228 |
-
set_seed(args.random_seed)
|
| 229 |
-
random.seed(args.random_seed)
|
| 230 |
-
np.random.seed(args.random_seed)
|
| 231 |
-
torch.manual_seed(args.random_seed)
|
| 232 |
-
torch.cuda.manual_seed_all(args.random_seed)
|
| 233 |
-
|
| 234 |
-
# Setup dataset and dataloaders
|
| 235 |
-
dataset = HDF5ChessDataset(
|
| 236 |
-
hdf5_path=args.data,
|
| 237 |
-
min_elo=args.min_elo,
|
| 238 |
-
max_elo=args.max_elo,
|
| 239 |
-
)
|
| 240 |
-
|
| 241 |
-
MAX_VALIDATION_SAMPLES = 10000
|
| 242 |
-
|
| 243 |
-
val_size = min(int(0.1 * len(dataset)), MAX_VALIDATION_SAMPLES)
|
| 244 |
-
test_size = min(int(0.1 * len(dataset)), MAX_VALIDATION_SAMPLES)
|
| 245 |
-
train_size = len(dataset) - val_size - test_size
|
| 246 |
-
train_set, val_set, test_set = random_split(dataset, [train_size, val_size, test_size])
|
| 247 |
-
|
| 248 |
-
train_loader = DataLoader(train_set, batch_size=args.batch_size, shuffle=True, num_workers=12, drop_last=True)
|
| 249 |
-
val_loader = DataLoader(val_set, batch_size=args.batch_size, shuffle=False, num_workers=12)
|
| 250 |
-
test_loader = DataLoader(test_set, batch_size=args.batch_size, shuffle=False, num_workers=12)
|
| 251 |
-
|
| 252 |
-
print(f"Train samples: {len(train_set)}, Val samples: {len(val_set)}, Test samples: {len(test_set)}")
|
| 253 |
-
|
| 254 |
-
# Model configuration
|
| 255 |
-
model_config = {
|
| 256 |
-
"vocab_size": dataset.position_tokenizer.vocab_size,
|
| 257 |
-
"move_vocab_size": dataset.move_tokenizer.vocab_size,
|
| 258 |
-
"embed_dim": 512,
|
| 259 |
-
"nb_transformer_layers": 8,
|
| 260 |
-
"num_heads": 8,
|
| 261 |
-
"num_kv_groups": 4, # GQA: 8 heads / 4 groups = 2 heads per KV group
|
| 262 |
-
"dropout": 0.1,
|
| 263 |
-
"kvq_bias": False,
|
| 264 |
-
"mask_future": False,
|
| 265 |
-
"rope": True, # Use RoPE for positional encoding
|
| 266 |
-
"use_swiglu": True, # Use SwiGLU activation (Llama-style)
|
| 267 |
-
"use_col_row_emb": True, # Use separate column and row embeddings
|
| 268 |
-
}
|
| 269 |
-
|
| 270 |
-
# Setup logging
|
| 271 |
-
log_dir = Path("logs") / "position2move"
|
| 272 |
-
log_dir.mkdir(parents=True, exist_ok=True)
|
| 273 |
-
run_number = get_next_run_number(str(log_dir))
|
| 274 |
-
log_dir_full = f"run_{run_number:03d}_{datetime.now().strftime('%Y%m%d_%H%M%S')}"
|
| 275 |
-
log_path = log_dir / log_dir_full
|
| 276 |
-
log_path.mkdir(parents=True, exist_ok=False)
|
| 277 |
-
print(f"Logging to: {log_path}")
|
| 278 |
-
|
| 279 |
-
# Create checkpoints directory
|
| 280 |
-
checkpoint_dir = log_path / "checkpoints"
|
| 281 |
-
checkpoint_dir.mkdir(exist_ok=True)
|
| 282 |
-
|
| 283 |
-
# Save model config to JSON
|
| 284 |
-
config_path = log_path / "model_config.json"
|
| 285 |
-
with config_path.open("w", encoding="utf-8") as f:
|
| 286 |
-
json.dump(model_config, f, indent=2)
|
| 287 |
-
print(f"Model config saved to: {config_path}")
|
| 288 |
-
|
| 289 |
-
precision = args.precision if torch.cuda.is_available() or torch.backends.mps.is_available() else "fp32"
|
| 290 |
-
print(f"Using precision: {precision}")
|
| 291 |
-
|
| 292 |
-
accelerator = accelerate.Accelerator(log_with="tensorboard", project_dir=str(log_path), mixed_precision=precision)
|
| 293 |
-
|
| 294 |
-
# Initialize trackers
|
| 295 |
-
accelerator.init_trackers(
|
| 296 |
-
project_name="position2move",
|
| 297 |
-
config={
|
| 298 |
-
"batch_size": args.batch_size,
|
| 299 |
-
"epochs": args.epochs,
|
| 300 |
-
"learning_rate": args.lr,
|
| 301 |
-
"lr_embedding": args.lr_embedding if args.lr_embedding is not None else args.lr,
|
| 302 |
-
"lr_head": args.lr_head if args.lr_head is not None else args.lr,
|
| 303 |
-
"min_elo": args.min_elo,
|
| 304 |
-
"max_elo": args.max_elo,
|
| 305 |
-
"max_grad_norm": args.max_grad_norm,
|
| 306 |
-
**model_config,
|
| 307 |
-
"warmup_steps": args.warmup_steps,
|
| 308 |
-
"final_lr_ratio": args.final_lr_ratio,
|
| 309 |
-
"precision": args.precision,
|
| 310 |
-
},
|
| 311 |
-
)
|
| 312 |
-
|
| 313 |
-
device = accelerator.device
|
| 314 |
-
print(f"Using device: {device}")
|
| 315 |
-
|
| 316 |
-
model = Position2MoveModel(**model_config)
|
| 317 |
-
|
| 318 |
-
# torch.compile has limited support on MPS backend - disable if compilation fails
|
| 319 |
-
if device.type == "mps":
|
| 320 |
-
print("Warning: Skipping torch.compile on MPS device (Mac GPU) due to backend limitations")
|
| 321 |
-
print("Training will proceed without compilation optimizations")
|
| 322 |
-
else:
|
| 323 |
-
try:
|
| 324 |
-
model = torch.compile(
|
| 325 |
-
model,
|
| 326 |
-
)
|
| 327 |
-
print("Model compiled successfully")
|
| 328 |
-
except Exception as e:
|
| 329 |
-
print(f"Warning: torch.compile failed ({e}), proceeding without compilation")
|
| 330 |
-
|
| 331 |
-
model.to(device)
|
| 332 |
-
|
| 333 |
-
if args.lr_find:
|
| 334 |
-
if accelerator.is_main_process:
|
| 335 |
-
print("Running learning rate finder...")
|
| 336 |
-
best_lr, best_loss = find_best_learning_rate(
|
| 337 |
-
model,
|
| 338 |
-
train_loader,
|
| 339 |
-
accelerator,
|
| 340 |
-
lr_min=args.lr_min,
|
| 341 |
-
lr_max=args.lr_max,
|
| 342 |
-
num_steps=args.lr_steps,
|
| 343 |
-
beta=args.beta,
|
| 344 |
-
legal_move_loss_weight=args.legal_move_loss_weight,
|
| 345 |
-
)
|
| 346 |
-
accelerator.log({"lr_finder/best_lr": best_lr, "lr_finder/best_loss": best_loss}, step=0)
|
| 347 |
-
args.lr = best_lr
|
| 348 |
-
if accelerator.is_main_process:
|
| 349 |
-
print(f"Best LR found: {best_lr:.6g} (loss {best_loss:.4f})")
|
| 350 |
-
|
| 351 |
-
# Set up discriminative learning rates
|
| 352 |
-
lr_embedding = args.lr_embedding if args.lr_embedding is not None else args.lr
|
| 353 |
-
lr_head = args.lr_head if args.lr_head is not None else args.lr
|
| 354 |
-
lr_transformer = args.lr
|
| 355 |
-
|
| 356 |
-
# Create parameter groups
|
| 357 |
-
embedding_params = []
|
| 358 |
-
transformer_params = []
|
| 359 |
-
head_params = []
|
| 360 |
-
|
| 361 |
-
# Get underlying model (unwrap from accelerator if needed)
|
| 362 |
-
unwrapped_model = accelerator.unwrap_model(model)
|
| 363 |
-
|
| 364 |
-
# Categorize parameters
|
| 365 |
-
for name, param in unwrapped_model.named_parameters():
|
| 366 |
-
if "embedding" in name:
|
| 367 |
-
embedding_params.append(param)
|
| 368 |
-
elif "lm_head" in name:
|
| 369 |
-
head_params.append(param)
|
| 370 |
-
else:
|
| 371 |
-
transformer_params.append(param)
|
| 372 |
-
|
| 373 |
-
param_groups = [
|
| 374 |
-
{"params": embedding_params, "lr": lr_embedding, "name": "embeddings"},
|
| 375 |
-
{"params": transformer_params, "lr": lr_transformer, "name": "transformers"},
|
| 376 |
-
{"params": head_params, "lr": lr_head, "name": "head"},
|
| 377 |
-
]
|
| 378 |
-
|
| 379 |
-
if accelerator.is_main_process:
|
| 380 |
-
emb_count = sum(p.numel() for p in embedding_params)
|
| 381 |
-
trans_count = sum(p.numel() for p in transformer_params)
|
| 382 |
-
head_count = sum(p.numel() for p in head_params)
|
| 383 |
-
print(f"\nLearning rates:")
|
| 384 |
-
print(f" Embeddings: {lr_embedding:.6g}")
|
| 385 |
-
print(f" Transformers: {lr_transformer:.6g}")
|
| 386 |
-
print(f" Head: {lr_head:.6g}")
|
| 387 |
-
print(f" Param counts: Emb={emb_count:,}, Trans={trans_count:,}, Head={head_count:,}\n")
|
| 388 |
-
|
| 389 |
-
optimizer = torch.optim.AdamW(param_groups, lr=args.lr)
|
| 390 |
-
model, optimizer, train_loader, val_loader, test_loader = accelerator.prepare(
|
| 391 |
-
model, optimizer, train_loader, val_loader, test_loader
|
| 392 |
-
)
|
| 393 |
-
|
| 394 |
-
# Check if resuming to load scheduler config from checkpoint
|
| 395 |
-
best_val_loss = float("inf")
|
| 396 |
-
start_epoch = 1
|
| 397 |
-
global_step = 0
|
| 398 |
-
resume_path = Path(args.resume_from) if args.resume_from else None
|
| 399 |
-
trainer_state = None
|
| 400 |
-
|
| 401 |
-
if resume_path:
|
| 402 |
-
trainer_state = load_trainer_state(resume_path)
|
| 403 |
-
if trainer_state:
|
| 404 |
-
start_epoch = trainer_state.get("epoch", 0) + 1
|
| 405 |
-
global_step = trainer_state.get("global_step", 0)
|
| 406 |
-
best_val_loss = trainer_state.get("best_val_loss", float("inf"))
|
| 407 |
-
|
| 408 |
-
# Determine scheduler parameters - use saved config when resuming for consistency
|
| 409 |
-
has_scheduler_config = trainer_state and "scheduler_config" in trainer_state
|
| 410 |
-
|
| 411 |
-
if has_scheduler_config:
|
| 412 |
-
saved_config = trainer_state["scheduler_config"]
|
| 413 |
-
scheduler_warmup_steps = saved_config["warmup_steps"]
|
| 414 |
-
scheduler_total_steps = saved_config["total_steps"]
|
| 415 |
-
scheduler_final_lr_ratio = saved_config["final_lr_ratio"]
|
| 416 |
-
if accelerator.is_main_process:
|
| 417 |
-
print(
|
| 418 |
-
f"Using scheduler config from checkpoint: warmup={scheduler_warmup_steps}, total={scheduler_total_steps}, final_lr_ratio={scheduler_final_lr_ratio}"
|
| 419 |
-
)
|
| 420 |
-
else:
|
| 421 |
-
scheduler_warmup_steps = args.warmup_steps
|
| 422 |
-
scheduler_total_steps = max(1, len(train_loader)) * max(1, args.epochs)
|
| 423 |
-
scheduler_final_lr_ratio = args.final_lr_ratio
|
| 424 |
-
if resume_path and accelerator.is_main_process:
|
| 425 |
-
print(
|
| 426 |
-
f"Old checkpoint without scheduler_config - using new scheduler params: warmup={scheduler_warmup_steps}, total={scheduler_total_steps}"
|
| 427 |
-
)
|
| 428 |
-
|
| 429 |
-
# Store scheduler config for saving in checkpoints
|
| 430 |
-
scheduler_config = {
|
| 431 |
-
"warmup_steps": scheduler_warmup_steps,
|
| 432 |
-
"total_steps": scheduler_total_steps,
|
| 433 |
-
"final_lr_ratio": scheduler_final_lr_ratio,
|
| 434 |
-
}
|
| 435 |
-
|
| 436 |
-
scheduler = create_linear_warmup_decay_scheduler(
|
| 437 |
-
optimizer,
|
| 438 |
-
warmup_steps=scheduler_warmup_steps,
|
| 439 |
-
total_steps=scheduler_total_steps,
|
| 440 |
-
final_lr_ratio=scheduler_final_lr_ratio,
|
| 441 |
-
)
|
| 442 |
-
|
| 443 |
-
# Always register scheduler for checkpointing (required for load_state to work)
|
| 444 |
-
accelerator.register_for_checkpointing(scheduler)
|
| 445 |
-
|
| 446 |
-
# Load the full accelerator state (model, optimizer, scheduler) if resuming
|
| 447 |
-
if resume_path:
|
| 448 |
-
accelerator.load_state(str(resume_path))
|
| 449 |
-
|
| 450 |
-
# For old checkpoints without scheduler_config, the loaded scheduler state
|
| 451 |
-
# has a mismatched lr_lambda (old total_steps). We need to recreate it.
|
| 452 |
-
if not has_scheduler_config:
|
| 453 |
-
# Recreate scheduler with correct parameters
|
| 454 |
-
scheduler = create_linear_warmup_decay_scheduler(
|
| 455 |
-
optimizer,
|
| 456 |
-
warmup_steps=scheduler_warmup_steps,
|
| 457 |
-
total_steps=scheduler_total_steps,
|
| 458 |
-
final_lr_ratio=scheduler_final_lr_ratio,
|
| 459 |
-
)
|
| 460 |
-
# Manually advance to the correct step
|
| 461 |
-
scheduler.last_epoch = global_step - 1 # -1 because step() will increment it
|
| 462 |
-
scheduler.step() # This sets it to global_step and computes correct LR
|
| 463 |
-
if accelerator.is_main_process:
|
| 464 |
-
current_lrs = scheduler.get_last_lr()
|
| 465 |
-
print(f" Recreated scheduler and advanced to step {global_step}")
|
| 466 |
-
print(f" Current LRs: emb={current_lrs[0]:.6g}, trans={current_lrs[1]:.6g}, head={current_lrs[2]:.6g}")
|
| 467 |
-
|
| 468 |
-
if accelerator.is_main_process:
|
| 469 |
-
print(f"Resumed from checkpoint: {resume_path}")
|
| 470 |
-
print(f" Starting from epoch {start_epoch}, global_step {global_step}, best_val_loss {best_val_loss:.4f}")
|
| 471 |
-
|
| 472 |
-
for epoch in range(start_epoch, args.epochs + 1):
|
| 473 |
-
# Training phase
|
| 474 |
-
model.train()
|
| 475 |
-
train_loss = 0.0
|
| 476 |
-
train_llm_loss = 0.0
|
| 477 |
-
train_legal_move_loss = 0.0
|
| 478 |
-
train_legal_move_ce = 0.0
|
| 479 |
-
|
| 480 |
-
progress_bar = tqdm(train_loader, desc=f"Epoch {epoch}/{args.epochs} - Training", leave=False)
|
| 481 |
-
for batch in progress_bar:
|
| 482 |
-
optimizer.zero_grad()
|
| 483 |
-
positions = batch["position"]
|
| 484 |
-
moves = batch["move"]
|
| 485 |
-
is_white = batch["is_white"]
|
| 486 |
-
legal_moves_mask = batch["legal_moves_mask"]
|
| 487 |
-
castling_rights = batch["castling_rights"]
|
| 488 |
-
en_passant_file = batch["en_passant_file"]
|
| 489 |
-
halfmove_clock = batch["halfmove_clock"]
|
| 490 |
-
|
| 491 |
-
logits = model(positions, is_white, castling_rights, en_passant_file, halfmove_clock)
|
| 492 |
-
loss, llm_loss, legal_move_loss, legal_move_cross_entropy = compute_loss(
|
| 493 |
-
logits, moves, legal_moves_mask, args.beta, args.legal_move_loss_weight
|
| 494 |
-
)
|
| 495 |
-
|
| 496 |
-
accelerator.backward(loss)
|
| 497 |
-
|
| 498 |
-
# Gradient clipping
|
| 499 |
-
if args.max_grad_norm > 0:
|
| 500 |
-
accelerator.clip_grad_norm_(model.parameters(), args.max_grad_norm)
|
| 501 |
-
|
| 502 |
-
optimizer.step()
|
| 503 |
-
scheduler.step()
|
| 504 |
-
|
| 505 |
-
batch_size = positions.size(0)
|
| 506 |
-
train_loss += loss.item() * batch_size
|
| 507 |
-
train_llm_loss += llm_loss.item() * batch_size
|
| 508 |
-
train_legal_move_loss += legal_move_loss.item() * batch_size
|
| 509 |
-
train_legal_move_ce += legal_move_cross_entropy.item() * batch_size
|
| 510 |
-
|
| 511 |
-
progress_bar.set_postfix(
|
| 512 |
-
{
|
| 513 |
-
"loss": loss.item(),
|
| 514 |
-
"llm_loss": llm_loss.item(),
|
| 515 |
-
"legal_move_loss": legal_move_loss.item(),
|
| 516 |
-
"legal_move_ce": legal_move_cross_entropy.item(),
|
| 517 |
-
}
|
| 518 |
-
)
|
| 519 |
-
|
| 520 |
-
# Log step-level metrics
|
| 521 |
-
current_lrs = scheduler.get_last_lr()
|
| 522 |
-
log_dict = {
|
| 523 |
-
"train/step_loss": loss.item(),
|
| 524 |
-
"train/step_llm_loss": llm_loss.item(),
|
| 525 |
-
"train/step_legal_move_loss": legal_move_loss.item(),
|
| 526 |
-
"train/step_legal_move_ce": legal_move_cross_entropy.item(),
|
| 527 |
-
"train/lr_embeddings": current_lrs[0],
|
| 528 |
-
"train/lr_transformers": current_lrs[1],
|
| 529 |
-
"train/lr_head": current_lrs[2],
|
| 530 |
-
}
|
| 531 |
-
accelerator.log(log_dict, step=global_step)
|
| 532 |
-
|
| 533 |
-
global_step += 1
|
| 534 |
-
|
| 535 |
-
# Save step checkpoint
|
| 536 |
-
if args.save_steps > 0 and global_step % args.save_steps == 0:
|
| 537 |
-
step_checkpoint_path = checkpoint_dir / f"checkpoint_step_{global_step:07d}"
|
| 538 |
-
accelerator.save_state(str(step_checkpoint_path))
|
| 539 |
-
save_trainer_state(step_checkpoint_path, epoch, global_step, best_val_loss, scheduler_config)
|
| 540 |
-
if accelerator.is_main_process:
|
| 541 |
-
print(f"\n Saved step checkpoint at step {global_step}")
|
| 542 |
-
cleanup_old_step_checkpoints(checkpoint_dir, args.max_checkpoints)
|
| 543 |
-
|
| 544 |
-
# Calculate average training metrics
|
| 545 |
-
train_loss /= len(train_set)
|
| 546 |
-
train_llm_loss /= len(train_set)
|
| 547 |
-
train_legal_move_loss /= len(train_set)
|
| 548 |
-
train_legal_move_ce /= len(train_set)
|
| 549 |
-
|
| 550 |
-
# Validation phase
|
| 551 |
-
model.eval()
|
| 552 |
-
val_loss = 0.0
|
| 553 |
-
val_llm_loss = 0.0
|
| 554 |
-
val_legal_move_loss = 0.0
|
| 555 |
-
val_legal_move_ce = 0.0
|
| 556 |
-
val_correct = 0
|
| 557 |
-
val_total = 0
|
| 558 |
-
|
| 559 |
-
with torch.no_grad():
|
| 560 |
-
progress_bar = tqdm(val_loader, desc=f"Epoch {epoch}/{args.epochs} - Validation", leave=False)
|
| 561 |
-
for batch in progress_bar:
|
| 562 |
-
positions = batch["position"]
|
| 563 |
-
moves = batch["move"]
|
| 564 |
-
is_white = batch["is_white"]
|
| 565 |
-
legal_moves_mask = batch["legal_moves_mask"]
|
| 566 |
-
castling_rights = batch["castling_rights"]
|
| 567 |
-
en_passant_file = batch["en_passant_file"]
|
| 568 |
-
halfmove_clock = batch["halfmove_clock"]
|
| 569 |
-
|
| 570 |
-
logits = model(positions, is_white, castling_rights, en_passant_file, halfmove_clock)
|
| 571 |
-
loss, llm_loss, legal_move_loss, legal_move_cross_entropy = compute_loss(
|
| 572 |
-
logits, moves, legal_moves_mask, args.beta, args.legal_move_loss_weight
|
| 573 |
-
)
|
| 574 |
-
|
| 575 |
-
batch_size = positions.size(0)
|
| 576 |
-
val_loss += loss.item() * batch_size
|
| 577 |
-
val_llm_loss += llm_loss.item() * batch_size
|
| 578 |
-
val_legal_move_loss += legal_move_loss.item() * batch_size
|
| 579 |
-
val_legal_move_ce += legal_move_cross_entropy.item() * batch_size
|
| 580 |
-
|
| 581 |
-
# Calculate accuracy
|
| 582 |
-
predictions = torch.argmax(logits, dim=-1)
|
| 583 |
-
val_correct += (predictions == moves).sum().item()
|
| 584 |
-
val_total += batch_size
|
| 585 |
-
|
| 586 |
-
progress_bar.set_postfix({"val_loss": loss.item()})
|
| 587 |
-
|
| 588 |
-
val_loss /= len(val_set)
|
| 589 |
-
val_llm_loss /= len(val_set)
|
| 590 |
-
val_legal_move_loss /= len(val_set)
|
| 591 |
-
val_legal_move_ce /= len(val_set)
|
| 592 |
-
val_accuracy = val_correct / val_total if val_total > 0 else 0.0
|
| 593 |
-
|
| 594 |
-
# Log epoch-level metrics
|
| 595 |
-
accelerator.log(
|
| 596 |
-
{
|
| 597 |
-
"train/epoch_loss": train_loss,
|
| 598 |
-
"train/epoch_llm_loss": train_llm_loss,
|
| 599 |
-
"train/epoch_legal_move_loss": train_legal_move_loss,
|
| 600 |
-
"train/epoch_legal_move_ce": train_legal_move_ce,
|
| 601 |
-
"val/loss": val_loss,
|
| 602 |
-
"val/llm_loss": val_llm_loss,
|
| 603 |
-
"val/legal_move_loss": val_legal_move_loss,
|
| 604 |
-
"val/legal_move_ce": val_legal_move_ce,
|
| 605 |
-
"val/accuracy": val_accuracy,
|
| 606 |
-
"epoch": epoch,
|
| 607 |
-
"train/lr_embeddings_epoch_end": scheduler.get_last_lr()[0],
|
| 608 |
-
"train/lr_transformers_epoch_end": scheduler.get_last_lr()[1],
|
| 609 |
-
"train/lr_head_epoch_end": scheduler.get_last_lr()[2],
|
| 610 |
-
},
|
| 611 |
-
step=global_step,
|
| 612 |
-
)
|
| 613 |
-
|
| 614 |
-
print(f"\nEpoch {epoch}/{args.epochs}:")
|
| 615 |
-
print(f" Train Loss: {train_loss:.4f} | Val Loss: {val_loss:.4f} | Val Accuracy: {val_accuracy:.4f}")
|
| 616 |
-
|
| 617 |
-
# Save best model
|
| 618 |
-
if val_loss < best_val_loss:
|
| 619 |
-
best_val_loss = val_loss
|
| 620 |
-
best_checkpoint_path = checkpoint_dir / "best_model"
|
| 621 |
-
accelerator.save_state(str(best_checkpoint_path))
|
| 622 |
-
save_trainer_state(best_checkpoint_path, epoch, global_step, best_val_loss, scheduler_config)
|
| 623 |
-
print(f" Saved best model (val_loss: {val_loss:.4f})")
|
| 624 |
-
|
| 625 |
-
# Periodic checkpoint saving
|
| 626 |
-
if epoch % args.save_every == 0:
|
| 627 |
-
checkpoint_path = checkpoint_dir / f"checkpoint_epoch_{epoch:03d}"
|
| 628 |
-
accelerator.save_state(str(checkpoint_path))
|
| 629 |
-
save_trainer_state(checkpoint_path, epoch, global_step, best_val_loss, scheduler_config)
|
| 630 |
-
print(f" Saved checkpoint at epoch {epoch}")
|
| 631 |
-
|
| 632 |
-
# Final test evaluation
|
| 633 |
-
print("\nRunning final test evaluation...")
|
| 634 |
-
model.eval()
|
| 635 |
-
test_loss = 0.0
|
| 636 |
-
test_correct = 0
|
| 637 |
-
test_total = 0
|
| 638 |
-
|
| 639 |
-
with torch.no_grad():
|
| 640 |
-
progress_bar = tqdm(test_loader, desc="Testing", leave=False)
|
| 641 |
-
for batch in progress_bar:
|
| 642 |
-
positions = batch["position"]
|
| 643 |
-
moves = batch["move"]
|
| 644 |
-
is_white = batch["is_white"]
|
| 645 |
-
legal_moves_mask = batch["legal_moves_mask"]
|
| 646 |
-
castling_rights = batch["castling_rights"]
|
| 647 |
-
en_passant_file = batch["en_passant_file"]
|
| 648 |
-
halfmove_clock = batch["halfmove_clock"]
|
| 649 |
-
|
| 650 |
-
logits = model(positions, is_white, castling_rights, en_passant_file, halfmove_clock)
|
| 651 |
-
loss, llm_loss, legal_move_loss, legal_move_cross_entropy = compute_loss(
|
| 652 |
-
logits, moves, legal_moves_mask, args.beta, args.legal_move_loss_weight
|
| 653 |
-
)
|
| 654 |
-
|
| 655 |
-
batch_size = positions.size(0)
|
| 656 |
-
test_loss += loss.item() * batch_size
|
| 657 |
-
|
| 658 |
-
predictions = torch.argmax(logits, dim=-1)
|
| 659 |
-
test_correct += (predictions == moves).sum().item()
|
| 660 |
-
test_total += batch_size
|
| 661 |
-
|
| 662 |
-
test_loss /= len(test_set)
|
| 663 |
-
test_accuracy = test_correct / test_total if test_total > 0 else 0.0
|
| 664 |
-
|
| 665 |
-
accelerator.log(
|
| 666 |
-
{
|
| 667 |
-
"test/loss": test_loss,
|
| 668 |
-
"test/accuracy": test_accuracy,
|
| 669 |
-
},
|
| 670 |
-
step=global_step,
|
| 671 |
-
)
|
| 672 |
-
|
| 673 |
-
print(f"\nTest Results:")
|
| 674 |
-
print(f" Test Loss: {test_loss:.4f} | Test Accuracy: {test_accuracy:.4f}")
|
| 675 |
-
|
| 676 |
-
# End tracking
|
| 677 |
-
accelerator.end_training()
|
| 678 |
-
print(f"\nTraining complete! Logs saved to: {log_path}")
|
| 679 |
-
|
| 680 |
-
|
| 681 |
-
if __name__ == "__main__":
|
| 682 |
-
main()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
chesstransformer/utils/__init__.py
DELETED
|
@@ -1,3 +0,0 @@
|
|
| 1 |
-
from .chess_utils import generate_board_image, generate_board_ascii, generate_game_gif
|
| 2 |
-
|
| 3 |
-
__all__ = ["generate_board_image", "generate_board_ascii", "generate_game_gif"]
|
|
|
|
|
|
|
|
|
|
|
|
chesstransformer/utils/chess_utils.py
DELETED
|
@@ -1,123 +0,0 @@
|
|
| 1 |
-
import chess
|
| 2 |
-
from PIL import Image
|
| 3 |
-
from io import BytesIO
|
| 4 |
-
import cairosvg
|
| 5 |
-
|
| 6 |
-
|
| 7 |
-
def _create_board_from_moves(moves: list[str]) -> chess.Board:
|
| 8 |
-
"""
|
| 9 |
-
Create a chess board and apply a sequence of moves.
|
| 10 |
-
|
| 11 |
-
Args:
|
| 12 |
-
moves: List of moves in UCI format (e.g., ['e2e4', 'd7d5'])
|
| 13 |
-
|
| 14 |
-
Returns:
|
| 15 |
-
Chess board with moves applied
|
| 16 |
-
"""
|
| 17 |
-
board = chess.Board()
|
| 18 |
-
for move_str in moves:
|
| 19 |
-
try:
|
| 20 |
-
move = chess.Move.from_uci(move_str)
|
| 21 |
-
board.push(move)
|
| 22 |
-
except ValueError:
|
| 23 |
-
pass # Skip invalid moves
|
| 24 |
-
return board
|
| 25 |
-
|
| 26 |
-
|
| 27 |
-
def generate_board_image(moves: list[str], size: int = 400) -> Image.Image:
|
| 28 |
-
"""
|
| 29 |
-
Generate a PIL image of a chess board after applying a sequence of moves.
|
| 30 |
-
|
| 31 |
-
Args:
|
| 32 |
-
moves: List of moves in UCI format (e.g., ['e2e4', 'd7d5'])
|
| 33 |
-
size: Size of the output image in pixels (default: 400)
|
| 34 |
-
|
| 35 |
-
Returns:
|
| 36 |
-
PIL Image of the chess board
|
| 37 |
-
"""
|
| 38 |
-
import chess.svg
|
| 39 |
-
|
| 40 |
-
board = _create_board_from_moves(moves)
|
| 41 |
-
|
| 42 |
-
# Generate SVG using chess library
|
| 43 |
-
svg_data = chess.svg.board(board, size=size)
|
| 44 |
-
|
| 45 |
-
# Convert SVG to PNG using cairosvg
|
| 46 |
-
png_data = cairosvg.svg2png(bytestring=svg_data.encode("utf-8"))
|
| 47 |
-
|
| 48 |
-
# Load PNG into PIL Image
|
| 49 |
-
img = Image.open(BytesIO(png_data))
|
| 50 |
-
|
| 51 |
-
return img
|
| 52 |
-
|
| 53 |
-
|
| 54 |
-
def generate_board_ascii(moves: list[str]) -> str:
|
| 55 |
-
"""
|
| 56 |
-
Generate an ASCII representation of a chess board after applying a sequence of moves.
|
| 57 |
-
|
| 58 |
-
Args:
|
| 59 |
-
moves: List of moves in UCI format (e.g., ['e2e4', 'd7d5'])
|
| 60 |
-
|
| 61 |
-
Returns:
|
| 62 |
-
String representation of the chess board
|
| 63 |
-
"""
|
| 64 |
-
board = _create_board_from_moves(moves)
|
| 65 |
-
return str(board)
|
| 66 |
-
|
| 67 |
-
|
| 68 |
-
def generate_game_gif(moves: list[str], size: int = 400, duration: int = 500) -> bytes:
|
| 69 |
-
"""
|
| 70 |
-
Generate an animated GIF of a chess game from a sequence of moves.
|
| 71 |
-
|
| 72 |
-
Args:
|
| 73 |
-
moves: List of moves in UCI format (e.g., ['e2e4', 'd7d5'])
|
| 74 |
-
size: Size of each frame in pixels (default: 400)
|
| 75 |
-
duration: Duration of each frame in milliseconds (default: 500)
|
| 76 |
-
|
| 77 |
-
Returns:
|
| 78 |
-
Bytes of the animated GIF
|
| 79 |
-
"""
|
| 80 |
-
frames = []
|
| 81 |
-
|
| 82 |
-
# Initial position
|
| 83 |
-
img = generate_board_image([], size)
|
| 84 |
-
frames.append(img)
|
| 85 |
-
|
| 86 |
-
# Apply moves and capture frames
|
| 87 |
-
moves_so_far = []
|
| 88 |
-
for move_str in moves:
|
| 89 |
-
try:
|
| 90 |
-
move = chess.Move.from_uci(move_str)
|
| 91 |
-
moves_so_far.append(move_str)
|
| 92 |
-
img = generate_board_image(moves_so_far, size)
|
| 93 |
-
frames.append(img)
|
| 94 |
-
except ValueError:
|
| 95 |
-
pass # Skip invalid moves
|
| 96 |
-
|
| 97 |
-
# Save frames as GIF in memory
|
| 98 |
-
gif_bytes = BytesIO()
|
| 99 |
-
frames[0].save(
|
| 100 |
-
gif_bytes,
|
| 101 |
-
format="GIF",
|
| 102 |
-
save_all=True,
|
| 103 |
-
append_images=frames[1:],
|
| 104 |
-
duration=duration,
|
| 105 |
-
loop=0,
|
| 106 |
-
)
|
| 107 |
-
gif_bytes.seek(0)
|
| 108 |
-
|
| 109 |
-
return gif_bytes.getvalue()
|
| 110 |
-
|
| 111 |
-
|
| 112 |
-
if __name__ == "__main__":
|
| 113 |
-
moves = ["e2e4"]
|
| 114 |
-
img = generate_board_image(moves)
|
| 115 |
-
img.show()
|
| 116 |
-
|
| 117 |
-
ascii_board = generate_board_ascii(moves)
|
| 118 |
-
print(ascii_board)
|
| 119 |
-
|
| 120 |
-
coup_du_berget = ["e2e4", "e7e5", "f1c4", "b8c6", "d1h5", "g8f6", "h5f7"]
|
| 121 |
-
gif = generate_game_gif(coup_du_berget)
|
| 122 |
-
with open("coup_du_berget.gif", "wb") as f:
|
| 123 |
-
f.write(gif)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
chesstransformer/utils/evaluate_puzzles.py
DELETED
|
@@ -1,426 +0,0 @@
|
|
| 1 |
-
#!/usr/bin/env python3
|
| 2 |
-
"""
|
| 3 |
-
Evaluation script for chess puzzle solving with Position2Move model.
|
| 4 |
-
|
| 5 |
-
This script evaluates how well a trained model can solve chess puzzles by:
|
| 6 |
-
1. Loading a trained model checkpoint
|
| 7 |
-
2. Testing it on puzzles from the Lichess puzzle database
|
| 8 |
-
3. Measuring accuracy for first move, full solution, and by rating/theme
|
| 9 |
-
4. Generating detailed reports
|
| 10 |
-
|
| 11 |
-
Usage:
|
| 12 |
-
python src/chesstransformer/utils/evaluate_puzzles.py --model data/models/puzzle_training/run_001/best_model.pth
|
| 13 |
-
|
| 14 |
-
Options:
|
| 15 |
-
--model: Path to model checkpoint (required)
|
| 16 |
-
--puzzle-data: Path to puzzle .csv.zst file (default: data/lichess_db_puzzle.csv.zst)
|
| 17 |
-
--num-puzzles: Number of puzzles to evaluate (default: 1000)
|
| 18 |
-
--min-rating: Minimum puzzle rating (default: None)
|
| 19 |
-
--max-rating: Maximum puzzle rating (default: None)
|
| 20 |
-
--themes: Comma-separated puzzle themes to filter by (default: None)
|
| 21 |
-
--output: Output JSON file for results (default: results/puzzle_eval_{timestamp}.json)
|
| 22 |
-
"""
|
| 23 |
-
|
| 24 |
-
from pathlib import Path
|
| 25 |
-
from datetime import datetime
|
| 26 |
-
import argparse
|
| 27 |
-
import json
|
| 28 |
-
|
| 29 |
-
import torch
|
| 30 |
-
import chess
|
| 31 |
-
from tqdm.auto import tqdm
|
| 32 |
-
|
| 33 |
-
from chesstransformer.datasets.puzzle_dataset import LichessPuzzleFullSolutionDataset
|
| 34 |
-
from chesstransformer.models.transformer.position2move import Position2MoveModel
|
| 35 |
-
from chesstransformer.models.tokenizer.position_tokenizer import PostionTokenizer
|
| 36 |
-
from chesstransformer.models.tokenizer.move_tokenizer import MoveTokenizer
|
| 37 |
-
|
| 38 |
-
|
| 39 |
-
class PuzzleEvaluator:
|
| 40 |
-
"""Evaluates a Position2Move model on chess puzzles."""
|
| 41 |
-
|
| 42 |
-
def __init__(self, model, device="cpu"):
|
| 43 |
-
self.model = model
|
| 44 |
-
self.device = device
|
| 45 |
-
self.model.to(device)
|
| 46 |
-
self.model.eval()
|
| 47 |
-
|
| 48 |
-
self.position_tokenizer = PostionTokenizer()
|
| 49 |
-
self.move_tokenizer = MoveTokenizer()
|
| 50 |
-
|
| 51 |
-
def predict_move(self, board: chess.Board, top_k=5):
|
| 52 |
-
"""
|
| 53 |
-
Predict the best move for a given board position.
|
| 54 |
-
|
| 55 |
-
Args:
|
| 56 |
-
board: chess.Board object
|
| 57 |
-
top_k: Return top-k predictions
|
| 58 |
-
|
| 59 |
-
Returns:
|
| 60 |
-
List of (move_uci, probability, is_legal) tuples
|
| 61 |
-
"""
|
| 62 |
-
# Encode position
|
| 63 |
-
position_tokens = self.position_tokenizer.encode(board)
|
| 64 |
-
position_tensor = torch.tensor(position_tokens, dtype=torch.long).unsqueeze(0).to(self.device)
|
| 65 |
-
is_white = torch.tensor([board.turn == chess.WHITE], dtype=torch.long).to(self.device)
|
| 66 |
-
|
| 67 |
-
# Get predictions
|
| 68 |
-
with torch.no_grad():
|
| 69 |
-
logits = self.model(position_tensor, is_white)
|
| 70 |
-
|
| 71 |
-
# Create legal moves mask
|
| 72 |
-
legal_moves = [m.uci() for m in board.legal_moves]
|
| 73 |
-
legal_move_indices = []
|
| 74 |
-
for move_uci in legal_moves:
|
| 75 |
-
try:
|
| 76 |
-
idx = self.move_tokenizer.encode(move_uci)
|
| 77 |
-
legal_move_indices.append(idx)
|
| 78 |
-
except ValueError:
|
| 79 |
-
continue
|
| 80 |
-
|
| 81 |
-
# Apply mask to logits (set illegal moves to -inf before softmax)
|
| 82 |
-
mask = torch.full_like(logits, float("-inf"))
|
| 83 |
-
if legal_move_indices:
|
| 84 |
-
mask[0, legal_move_indices] = 0
|
| 85 |
-
masked_logits = logits + mask
|
| 86 |
-
|
| 87 |
-
probs = torch.softmax(masked_logits, dim=-1)
|
| 88 |
-
|
| 89 |
-
# Get top-k predictions
|
| 90 |
-
top_probs, top_indices = torch.topk(probs[0], k=top_k)
|
| 91 |
-
|
| 92 |
-
predictions = []
|
| 93 |
-
|
| 94 |
-
for prob, idx in zip(top_probs.cpu().numpy(), top_indices.cpu().numpy()):
|
| 95 |
-
try:
|
| 96 |
-
move_uci = self.move_tokenizer.decode(int(idx))
|
| 97 |
-
is_legal = move_uci in legal_moves
|
| 98 |
-
predictions.append((move_uci, float(prob), is_legal))
|
| 99 |
-
except ValueError:
|
| 100 |
-
continue
|
| 101 |
-
|
| 102 |
-
return predictions
|
| 103 |
-
|
| 104 |
-
def evaluate_puzzle(self, puzzle_data, max_moves=10):
|
| 105 |
-
"""
|
| 106 |
-
Evaluate a single puzzle.
|
| 107 |
-
|
| 108 |
-
Args:
|
| 109 |
-
puzzle_data: Dictionary with puzzle information
|
| 110 |
-
max_moves: Maximum number of moves to try in the solution
|
| 111 |
-
|
| 112 |
-
Returns:
|
| 113 |
-
Dictionary with evaluation results
|
| 114 |
-
"""
|
| 115 |
-
board = chess.Board(puzzle_data["fen"])
|
| 116 |
-
solution_moves = puzzle_data["moves_uci"]
|
| 117 |
-
|
| 118 |
-
results = {
|
| 119 |
-
"puzzle_id": puzzle_data["puzzle_id"],
|
| 120 |
-
"rating": puzzle_data["rating"],
|
| 121 |
-
"themes": puzzle_data["themes"],
|
| 122 |
-
"solution_length": len(solution_moves),
|
| 123 |
-
"moves_tried": [],
|
| 124 |
-
"first_move_correct": False,
|
| 125 |
-
"fully_solved": False,
|
| 126 |
-
"moves_correct": 0,
|
| 127 |
-
"predictions": [],
|
| 128 |
-
}
|
| 129 |
-
|
| 130 |
-
# Try to solve the puzzle move by move
|
| 131 |
-
for move_idx in range(0, len(solution_moves), 2): # Only predict our moves (every other move)
|
| 132 |
-
if move_idx >= max_moves:
|
| 133 |
-
break
|
| 134 |
-
|
| 135 |
-
# Apply opponent's move first (if not the first move)
|
| 136 |
-
if move_idx > 0:
|
| 137 |
-
opponent_move = chess.Move.from_uci(solution_moves[move_idx - 1])
|
| 138 |
-
if opponent_move in board.legal_moves:
|
| 139 |
-
board.push(opponent_move)
|
| 140 |
-
else:
|
| 141 |
-
# Opponent move is illegal - puzzle data might be corrupted
|
| 142 |
-
results["error"] = "Illegal opponent move in solution"
|
| 143 |
-
break
|
| 144 |
-
|
| 145 |
-
# Predict our move
|
| 146 |
-
expected_move = solution_moves[move_idx]
|
| 147 |
-
predictions = self.predict_move(board, top_k=5)
|
| 148 |
-
|
| 149 |
-
if not predictions:
|
| 150 |
-
results["error"] = "Model produced no valid predictions"
|
| 151 |
-
break
|
| 152 |
-
|
| 153 |
-
predicted_move = predictions[0][0]
|
| 154 |
-
predicted_prob = predictions[0][1]
|
| 155 |
-
is_correct = predicted_move == expected_move
|
| 156 |
-
|
| 157 |
-
results["moves_tried"].append(
|
| 158 |
-
{
|
| 159 |
-
"move_number": move_idx // 2 + 1,
|
| 160 |
-
"expected": expected_move,
|
| 161 |
-
"predicted": predicted_move,
|
| 162 |
-
"probability": predicted_prob,
|
| 163 |
-
"correct": is_correct,
|
| 164 |
-
"top_5_predictions": [{"move": m, "prob": p, "legal": l} for m, p, l in predictions],
|
| 165 |
-
}
|
| 166 |
-
)
|
| 167 |
-
|
| 168 |
-
if is_correct:
|
| 169 |
-
results["moves_correct"] += 1
|
| 170 |
-
if move_idx == 0:
|
| 171 |
-
results["first_move_correct"] = True
|
| 172 |
-
|
| 173 |
-
# Apply our correct move
|
| 174 |
-
our_move = chess.Move.from_uci(predicted_move)
|
| 175 |
-
board.push(our_move)
|
| 176 |
-
else:
|
| 177 |
-
# Wrong move - puzzle failed
|
| 178 |
-
break
|
| 179 |
-
|
| 180 |
-
# Check if fully solved
|
| 181 |
-
if results["moves_correct"] == (len(solution_moves) + 1) // 2:
|
| 182 |
-
results["fully_solved"] = True
|
| 183 |
-
|
| 184 |
-
return results
|
| 185 |
-
|
| 186 |
-
def evaluate_dataset(self, dataset, num_puzzles=None):
|
| 187 |
-
"""
|
| 188 |
-
Evaluate the model on a puzzle dataset.
|
| 189 |
-
|
| 190 |
-
Args:
|
| 191 |
-
dataset: LichessPuzzleFullSolutionDataset
|
| 192 |
-
num_puzzles: Number of puzzles to evaluate (None = all)
|
| 193 |
-
|
| 194 |
-
Returns:
|
| 195 |
-
Dictionary with aggregate results
|
| 196 |
-
"""
|
| 197 |
-
num_puzzles = min(num_puzzles or len(dataset), len(dataset))
|
| 198 |
-
|
| 199 |
-
print(f"Evaluating {num_puzzles} puzzles...")
|
| 200 |
-
|
| 201 |
-
results = {
|
| 202 |
-
"num_puzzles": num_puzzles,
|
| 203 |
-
"first_move_accuracy": 0,
|
| 204 |
-
"full_solution_accuracy": 0,
|
| 205 |
-
"average_moves_correct": 0,
|
| 206 |
-
"by_rating": {},
|
| 207 |
-
"by_theme": {},
|
| 208 |
-
"puzzle_results": [],
|
| 209 |
-
}
|
| 210 |
-
|
| 211 |
-
first_move_correct = 0
|
| 212 |
-
fully_solved = 0
|
| 213 |
-
total_moves_correct = 0
|
| 214 |
-
total_moves = 0
|
| 215 |
-
|
| 216 |
-
# Track by rating buckets
|
| 217 |
-
rating_buckets = {
|
| 218 |
-
"0-1000": {"first": 0, "full": 0, "total": 0},
|
| 219 |
-
"1000-1500": {"first": 0, "full": 0, "total": 0},
|
| 220 |
-
"1500-2000": {"first": 0, "full": 0, "total": 0},
|
| 221 |
-
"2000-2500": {"first": 0, "full": 0, "total": 0},
|
| 222 |
-
"2500+": {"first": 0, "full": 0, "total": 0},
|
| 223 |
-
}
|
| 224 |
-
|
| 225 |
-
# Track by theme
|
| 226 |
-
theme_stats = {}
|
| 227 |
-
|
| 228 |
-
for i in tqdm(range(num_puzzles), desc="Evaluating puzzles"):
|
| 229 |
-
puzzle_data = dataset[i]
|
| 230 |
-
puzzle_result = self.evaluate_puzzle(puzzle_data)
|
| 231 |
-
|
| 232 |
-
# Aggregate statistics
|
| 233 |
-
if puzzle_result["first_move_correct"]:
|
| 234 |
-
first_move_correct += 1
|
| 235 |
-
if puzzle_result["fully_solved"]:
|
| 236 |
-
fully_solved += 1
|
| 237 |
-
|
| 238 |
-
total_moves_correct += puzzle_result["moves_correct"]
|
| 239 |
-
total_moves += puzzle_result["solution_length"] // 2 + 1
|
| 240 |
-
|
| 241 |
-
# Rating bucket
|
| 242 |
-
rating = puzzle_result["rating"]
|
| 243 |
-
if rating < 1000:
|
| 244 |
-
bucket = "0-1000"
|
| 245 |
-
elif rating < 1500:
|
| 246 |
-
bucket = "1000-1500"
|
| 247 |
-
elif rating < 2000:
|
| 248 |
-
bucket = "1500-2000"
|
| 249 |
-
elif rating < 2500:
|
| 250 |
-
bucket = "2000-2500"
|
| 251 |
-
else:
|
| 252 |
-
bucket = "2500+"
|
| 253 |
-
|
| 254 |
-
rating_buckets[bucket]["total"] += 1
|
| 255 |
-
if puzzle_result["first_move_correct"]:
|
| 256 |
-
rating_buckets[bucket]["first"] += 1
|
| 257 |
-
if puzzle_result["fully_solved"]:
|
| 258 |
-
rating_buckets[bucket]["full"] += 1
|
| 259 |
-
|
| 260 |
-
# Theme statistics
|
| 261 |
-
for theme in puzzle_result["themes"]:
|
| 262 |
-
if theme not in theme_stats:
|
| 263 |
-
theme_stats[theme] = {"first": 0, "full": 0, "total": 0}
|
| 264 |
-
theme_stats[theme]["total"] += 1
|
| 265 |
-
if puzzle_result["first_move_correct"]:
|
| 266 |
-
theme_stats[theme]["first"] += 1
|
| 267 |
-
if puzzle_result["fully_solved"]:
|
| 268 |
-
theme_stats[theme]["full"] += 1
|
| 269 |
-
|
| 270 |
-
results["puzzle_results"].append(puzzle_result)
|
| 271 |
-
|
| 272 |
-
# Calculate aggregate metrics
|
| 273 |
-
results["first_move_accuracy"] = 100.0 * first_move_correct / num_puzzles
|
| 274 |
-
results["full_solution_accuracy"] = 100.0 * fully_solved / num_puzzles
|
| 275 |
-
results["average_moves_correct"] = total_moves_correct / num_puzzles
|
| 276 |
-
results["move_accuracy"] = 100.0 * total_moves_correct / total_moves if total_moves > 0 else 0
|
| 277 |
-
|
| 278 |
-
# Rating bucket statistics
|
| 279 |
-
for bucket, stats in rating_buckets.items():
|
| 280 |
-
if stats["total"] > 0:
|
| 281 |
-
results["by_rating"][bucket] = {
|
| 282 |
-
"count": stats["total"],
|
| 283 |
-
"first_move_accuracy": 100.0 * stats["first"] / stats["total"],
|
| 284 |
-
"full_solution_accuracy": 100.0 * stats["full"] / stats["total"],
|
| 285 |
-
}
|
| 286 |
-
|
| 287 |
-
# Theme statistics (top 20 themes by frequency)
|
| 288 |
-
sorted_themes = sorted(theme_stats.items(), key=lambda x: x[1]["total"], reverse=True)[:20]
|
| 289 |
-
for theme, stats in sorted_themes:
|
| 290 |
-
results["by_theme"][theme] = {
|
| 291 |
-
"count": stats["total"],
|
| 292 |
-
"first_move_accuracy": 100.0 * stats["first"] / stats["total"],
|
| 293 |
-
"full_solution_accuracy": 100.0 * stats["full"] / stats["total"],
|
| 294 |
-
}
|
| 295 |
-
|
| 296 |
-
return results
|
| 297 |
-
|
| 298 |
-
|
| 299 |
-
def main():
|
| 300 |
-
parser = argparse.ArgumentParser(description="Evaluate Position2Move model on chess puzzles")
|
| 301 |
-
parser.add_argument("--model", type=str, required=True, help="Path to model checkpoint")
|
| 302 |
-
parser.add_argument(
|
| 303 |
-
"--puzzle-data", type=str, default="data/lichess_db_puzzle.csv.zst", help="Path to puzzle .csv.zst file"
|
| 304 |
-
)
|
| 305 |
-
parser.add_argument("--num-puzzles", type=int, default=1000, help="Number of puzzles to evaluate")
|
| 306 |
-
parser.add_argument("--min-rating", type=int, default=None, help="Minimum puzzle rating")
|
| 307 |
-
parser.add_argument("--max-rating", type=int, default=None, help="Maximum puzzle rating")
|
| 308 |
-
parser.add_argument("--themes", type=str, default=None, help="Comma-separated puzzle themes to filter by")
|
| 309 |
-
parser.add_argument("--output", type=str, default=None, help="Output JSON file for results")
|
| 310 |
-
|
| 311 |
-
args = parser.parse_args()
|
| 312 |
-
|
| 313 |
-
# Parse themes
|
| 314 |
-
themes = args.themes.split(",") if args.themes else None
|
| 315 |
-
|
| 316 |
-
# Setup device
|
| 317 |
-
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
|
| 318 |
-
print(f"Using device: {device}")
|
| 319 |
-
|
| 320 |
-
# Load model
|
| 321 |
-
print(f"\nLoading model from {args.model}...")
|
| 322 |
-
model_path = Path(args.model)
|
| 323 |
-
|
| 324 |
-
# Support both .pth and .safetensors formats
|
| 325 |
-
if model_path.suffix == ".safetensors":
|
| 326 |
-
# Load config from same directory
|
| 327 |
-
from safetensors import safe_open
|
| 328 |
-
import json
|
| 329 |
-
|
| 330 |
-
config_path = model_path.parent / "config.json"
|
| 331 |
-
with open(config_path, "r") as f:
|
| 332 |
-
config = json.load(f)
|
| 333 |
-
|
| 334 |
-
model = Position2MoveModel(**config)
|
| 335 |
-
|
| 336 |
-
# Load weights from safetensors
|
| 337 |
-
with safe_open(str(model_path), framework="pt", device=str(device)) as f:
|
| 338 |
-
state_dict = {k: f.get_tensor(k) for k in f.keys()}
|
| 339 |
-
|
| 340 |
-
# Handle compiled model prefix (_orig_mod.)
|
| 341 |
-
if any(k.startswith("_orig_mod.") for k in state_dict.keys()):
|
| 342 |
-
state_dict = {k.replace("_orig_mod.", ""): v for k, v in state_dict.items()}
|
| 343 |
-
|
| 344 |
-
model.load_state_dict(state_dict)
|
| 345 |
-
|
| 346 |
-
print(f"Model loaded from safetensors")
|
| 347 |
-
else:
|
| 348 |
-
# Load from .pth checkpoint
|
| 349 |
-
checkpoint = torch.load(args.model, map_location=device, weights_only=False)
|
| 350 |
-
config = checkpoint["config"]
|
| 351 |
-
|
| 352 |
-
model = Position2MoveModel(**config)
|
| 353 |
-
model.load_state_dict(checkpoint["model_state_dict"])
|
| 354 |
-
print(f"Model loaded (epoch {checkpoint.get('epoch', 'unknown')})")
|
| 355 |
-
|
| 356 |
-
# Load puzzle dataset
|
| 357 |
-
print(f"\nLoading puzzle dataset...")
|
| 358 |
-
dataset = LichessPuzzleFullSolutionDataset(
|
| 359 |
-
puzzle_path=args.puzzle_data,
|
| 360 |
-
min_rating=args.min_rating,
|
| 361 |
-
max_rating=args.max_rating,
|
| 362 |
-
themes=themes,
|
| 363 |
-
max_puzzles=args.num_puzzles,
|
| 364 |
-
)
|
| 365 |
-
|
| 366 |
-
# Evaluate
|
| 367 |
-
print("\n" + "=" * 70)
|
| 368 |
-
print("Starting evaluation")
|
| 369 |
-
print("=" * 70)
|
| 370 |
-
|
| 371 |
-
evaluator = PuzzleEvaluator(model, device=device)
|
| 372 |
-
results = evaluator.evaluate_dataset(dataset, num_puzzles=args.num_puzzles)
|
| 373 |
-
|
| 374 |
-
# Print summary
|
| 375 |
-
print("\n" + "=" * 70)
|
| 376 |
-
print("EVALUATION RESULTS")
|
| 377 |
-
print("=" * 70)
|
| 378 |
-
print(f"Total puzzles: {results['num_puzzles']}")
|
| 379 |
-
print(f"First move accuracy: {results['first_move_accuracy']:.2f}%")
|
| 380 |
-
print(f"Full solution accuracy: {results['full_solution_accuracy']:.2f}%")
|
| 381 |
-
print(f"Move accuracy: {results['move_accuracy']:.2f}%")
|
| 382 |
-
print(f"Average moves correct: {results['average_moves_correct']:.2f}")
|
| 383 |
-
|
| 384 |
-
print("\n" + "-" * 70)
|
| 385 |
-
print("By Rating:")
|
| 386 |
-
print("-" * 70)
|
| 387 |
-
for rating, stats in sorted(results["by_rating"].items()):
|
| 388 |
-
print(
|
| 389 |
-
f" {rating:>12}: {stats['count']:4} puzzles | "
|
| 390 |
-
f"First: {stats['first_move_accuracy']:5.2f}% | "
|
| 391 |
-
f"Full: {stats['full_solution_accuracy']:5.2f}%"
|
| 392 |
-
)
|
| 393 |
-
|
| 394 |
-
print("\n" + "-" * 70)
|
| 395 |
-
print("Top Themes:")
|
| 396 |
-
print("-" * 70)
|
| 397 |
-
for theme, stats in list(results["by_theme"].items())[:10]:
|
| 398 |
-
print(
|
| 399 |
-
f" {theme:>20}: {stats['count']:4} puzzles | "
|
| 400 |
-
f"First: {stats['first_move_accuracy']:5.2f}% | "
|
| 401 |
-
f"Full: {stats['full_solution_accuracy']:5.2f}%"
|
| 402 |
-
)
|
| 403 |
-
|
| 404 |
-
# Save results
|
| 405 |
-
if args.output:
|
| 406 |
-
output_path = args.output
|
| 407 |
-
else:
|
| 408 |
-
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
|
| 409 |
-
output_path = f"results/puzzle_eval_{timestamp}.json"
|
| 410 |
-
|
| 411 |
-
output_path = Path(output_path)
|
| 412 |
-
output_path.parent.mkdir(parents=True, exist_ok=True)
|
| 413 |
-
|
| 414 |
-
# Don't save individual puzzle results to keep file size manageable
|
| 415 |
-
results_summary = {k: v for k, v in results.items() if k != "puzzle_results"}
|
| 416 |
-
results_summary["num_puzzles_detailed"] = len(results["puzzle_results"])
|
| 417 |
-
|
| 418 |
-
with open(output_path, "w") as f:
|
| 419 |
-
json.dump(results_summary, f, indent=2)
|
| 420 |
-
|
| 421 |
-
print(f"\nResults saved to {output_path}")
|
| 422 |
-
print("=" * 70)
|
| 423 |
-
|
| 424 |
-
|
| 425 |
-
if __name__ == "__main__":
|
| 426 |
-
main()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
frontend/.dockerignore
ADDED
|
@@ -0,0 +1,36 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Dependencies
|
| 2 |
+
node_modules
|
| 3 |
+
.pnp
|
| 4 |
+
.pnp.js
|
| 5 |
+
|
| 6 |
+
# Build outputs
|
| 7 |
+
.next
|
| 8 |
+
out
|
| 9 |
+
build
|
| 10 |
+
|
| 11 |
+
# Testing
|
| 12 |
+
coverage
|
| 13 |
+
|
| 14 |
+
# Misc
|
| 15 |
+
.DS_Store
|
| 16 |
+
*.pem
|
| 17 |
+
|
| 18 |
+
# Debug
|
| 19 |
+
npm-debug.log*
|
| 20 |
+
yarn-debug.log*
|
| 21 |
+
yarn-error.log*
|
| 22 |
+
|
| 23 |
+
# Local env files
|
| 24 |
+
.env*.local
|
| 25 |
+
.env
|
| 26 |
+
|
| 27 |
+
# Vercel
|
| 28 |
+
.vercel
|
| 29 |
+
|
| 30 |
+
# TypeScript
|
| 31 |
+
*.tsbuildinfo
|
| 32 |
+
next-env.d.ts
|
| 33 |
+
|
| 34 |
+
# IDE
|
| 35 |
+
.idea
|
| 36 |
+
.vscode
|
frontend/.gitignore
ADDED
|
@@ -0,0 +1,41 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# See https://help.github.com/articles/ignoring-files/ for more about ignoring files.
|
| 2 |
+
|
| 3 |
+
# dependencies
|
| 4 |
+
/node_modules
|
| 5 |
+
/.pnp
|
| 6 |
+
.pnp.*
|
| 7 |
+
.yarn/*
|
| 8 |
+
!.yarn/patches
|
| 9 |
+
!.yarn/plugins
|
| 10 |
+
!.yarn/releases
|
| 11 |
+
!.yarn/versions
|
| 12 |
+
|
| 13 |
+
# testing
|
| 14 |
+
/coverage
|
| 15 |
+
|
| 16 |
+
# next.js
|
| 17 |
+
/.next/
|
| 18 |
+
/out/
|
| 19 |
+
|
| 20 |
+
# production
|
| 21 |
+
/build
|
| 22 |
+
|
| 23 |
+
# misc
|
| 24 |
+
.DS_Store
|
| 25 |
+
*.pem
|
| 26 |
+
|
| 27 |
+
# debug
|
| 28 |
+
npm-debug.log*
|
| 29 |
+
yarn-debug.log*
|
| 30 |
+
yarn-error.log*
|
| 31 |
+
.pnpm-debug.log*
|
| 32 |
+
|
| 33 |
+
# env files (can opt-in for committing if needed)
|
| 34 |
+
.env*
|
| 35 |
+
|
| 36 |
+
# vercel
|
| 37 |
+
.vercel
|
| 38 |
+
|
| 39 |
+
# typescript
|
| 40 |
+
*.tsbuildinfo
|
| 41 |
+
next-env.d.ts
|
frontend/Dockerfile
ADDED
|
@@ -0,0 +1,50 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Frontend Dockerfile for ChessTransformer Next.js app
|
| 2 |
+
FROM node:20-alpine AS base
|
| 3 |
+
|
| 4 |
+
# Install dependencies only when needed
|
| 5 |
+
FROM base AS deps
|
| 6 |
+
RUN apk add --no-cache libc6-compat
|
| 7 |
+
WORKDIR /app
|
| 8 |
+
|
| 9 |
+
# Copy package files
|
| 10 |
+
COPY package.json package-lock.json* ./
|
| 11 |
+
|
| 12 |
+
# Install dependencies
|
| 13 |
+
RUN npm ci
|
| 14 |
+
|
| 15 |
+
# Rebuild the source code only when needed
|
| 16 |
+
FROM base AS builder
|
| 17 |
+
WORKDIR /app
|
| 18 |
+
COPY --from=deps /app/node_modules ./node_modules
|
| 19 |
+
COPY . .
|
| 20 |
+
|
| 21 |
+
# Set the API URL for build time
|
| 22 |
+
ARG NEXT_PUBLIC_API_URL=http://localhost:5001
|
| 23 |
+
ENV NEXT_PUBLIC_API_URL=$NEXT_PUBLIC_API_URL
|
| 24 |
+
|
| 25 |
+
# Build the application
|
| 26 |
+
RUN npm run build
|
| 27 |
+
|
| 28 |
+
# Production image, copy all the files and run next
|
| 29 |
+
FROM base AS runner
|
| 30 |
+
WORKDIR /app
|
| 31 |
+
|
| 32 |
+
ENV NODE_ENV=production
|
| 33 |
+
|
| 34 |
+
# Create non-root user
|
| 35 |
+
RUN addgroup --system --gid 1001 nodejs
|
| 36 |
+
RUN adduser --system --uid 1001 nextjs
|
| 37 |
+
|
| 38 |
+
# Copy built application
|
| 39 |
+
COPY --from=builder /app/public ./public
|
| 40 |
+
COPY --from=builder --chown=nextjs:nodejs /app/.next/standalone ./
|
| 41 |
+
COPY --from=builder --chown=nextjs:nodejs /app/.next/static ./.next/static
|
| 42 |
+
|
| 43 |
+
USER nextjs
|
| 44 |
+
|
| 45 |
+
EXPOSE 3000
|
| 46 |
+
|
| 47 |
+
ENV PORT=3000
|
| 48 |
+
ENV HOSTNAME="0.0.0.0"
|
| 49 |
+
|
| 50 |
+
CMD ["node", "server.js"]
|