diff --git a/.dockerignore b/.dockerignore
new file mode 100644
index 0000000000000000000000000000000000000000..a45886c9bf8b5d047a53fb396b4504a285f362d4
--- /dev/null
+++ b/.dockerignore
@@ -0,0 +1,3 @@
+**/node_modules
+**/.next
+**/target
diff --git a/.gitattributes b/.gitattributes
index 580d310c2d211aca166288a207e972ced9bb0100..0bb75f73d568dc6f0e1e0733e94ab58027ddbfbf 100644
--- a/.gitattributes
+++ b/.gitattributes
@@ -1 +1 @@
-*.safetensors filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
diff --git a/Dockerfile b/Dockerfile
new file mode 100644
index 0000000000000000000000000000000000000000..6eb10aa4400d434a958b112e560878079ca3f104
--- /dev/null
+++ b/Dockerfile
@@ -0,0 +1,39 @@
+# Hugging Face Docker Space: one process, one port. ct-bot serves the JSON API
+# and the static Next.js build together on 7860.
+# Build context is the bundle prepare.sh assembles (rust/, frontend/, the model).
+
+# --- frontend: static export ---------------------------------------------------
+FROM node:20-slim AS web
+WORKDIR /web
+COPY frontend/package.json frontend/package-lock.json ./
+RUN npm ci
+COPY frontend/ ./
+# Empty API URL = same origin: the page calls /api/* on the server that served it.
+ENV NEXT_OUTPUT=export NEXT_PUBLIC_API_URL=""
+RUN npx next build
+
+# --- engine --------------------------------------------------------------------
+FROM rust:1.92-slim-trixie AS engine
+# ort's build script downloads the prebuilt ONNX Runtime over TLS; g++ provides
+# the libstdc++ the statically linked runtime needs at link time.
+RUN apt-get update && apt-get install -y --no-install-recommends \
+ pkg-config ca-certificates libssl-dev g++ && rm -rf /var/lib/apt/lists/*
+WORKDIR /build
+COPY rust/ ./rust/
+RUN cargo build --release --manifest-path rust/Cargo.toml -p ct-bot
+
+# --- runtime -------------------------------------------------------------------
+FROM debian:trixie-slim
+RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates \
+ && rm -rf /var/lib/apt/lists/* && useradd -m -u 1000 user
+WORKDIR /app
+COPY --from=engine /build/rust/target/release/ct-bot /usr/local/bin/ct-bot
+COPY --from=web /web/out ./web
+COPY model.int8.onnx ./model.int8.onnx
+USER user
+
+# MCTS_SIMS can be overridden from the Space settings (Variables) without a
+# rebuild. On the free 2-vCPU tier 800 sims is ~2 s a move; 2000 is ~5 s.
+ENV PORT=7860 MCTS_SIMS=800 ORT_THREADS=2 STATIC_DIR=/app/web RUST_LOG=info
+EXPOSE 7860
+CMD ["sh", "-c", "exec ct-bot serve --model /app/model.int8.onnx --threads \"$ORT_THREADS\""]
diff --git a/README.md b/README.md
index fd307d87e006a4348b729a3eaf545fcc9f0d6448..3cfb9d29273b630c2c7ba7a1e04db1a7b41bce94 100644
--- a/README.md
+++ b/README.md
@@ -1,11 +1,10 @@
---
title: ChessTransformer
emoji: ♟️
-colorFrom: indigo
+colorFrom: green
colorTo: gray
-sdk: gradio
-sdk_version: 5.49.0
-app_file: app.py
+sdk: docker
+app_port: 7860
pinned: false
license: mit
short_description: Play a chess transformer trained only on human games
@@ -14,15 +13,20 @@ short_description: Play a chess transformer trained only on human games
# ♟️ ChessTransformer
Play an **11.7M-parameter transformer trained only on human games** (no self-play,
-no reinforcement learning), playing via AlphaZero-style MCTS. At full strength it
-reaches **~2100 Elo** against Stockfish; this Space runs on CPU at a lower
-simulation count so moves come back in ~1–2s.
+no reinforcement learning), searching with AlphaZero-style MCTS. At full strength it
+reaches **~2100 Elo** against Stockfish.
-You play **White** — drag a piece and the bot replies automatically. Use the
-slider to trade strength for speed.
+- Pick a color and a clock, then play: drag or click to move, premove while it
+ thinks, right-click to draw arrows and mark squares.
+- **Watch it play itself**, or press **Autoplay** mid-game to hand it your side.
+- The **Engine search** tab shows the branches it explored for each move; hover one
+ to replay that line on the board.
+
+The engine is the Rust `ct-bot` (ONNX Runtime, int8, CPU). On this free 2-vCPU
+Space it searches 800 positions a move, about 2 seconds.
Code, weights, and the training pipeline: **https://github.com/tchauffi/ChessTransformer**
> This `README.md` is the Space config. The deployable Space is assembled by
-> `prepare.sh` (see `DEPLOY.md`) — it bundles `app.py`, this file, the slim
-> `requirements.txt`, the `chesstransformer` package source, and the model weights.
+> `prepare.sh` (see `DEPLOY.md`): this file, the Dockerfile, the `rust/` and
+> `frontend/` sources, and the int8 model.
diff --git a/app.py b/app.py
deleted file mode 100644
index 3a5df93d37aa9386f27b94d31c611a0403e1dd38..0000000000000000000000000000000000000000
--- a/app.py
+++ /dev/null
@@ -1,243 +0,0 @@
-"""ChessTransformer — Hugging Face Space (Gradio).
-
-Play the transformer chess engine in the browser. Runs on CPU: the model is
-only 11.7M params, so MCTS at a modest sim count returns a move in ~1-2s.
-
-Weights are resolved in this order:
- 1. ``MODEL_PATH`` env var (a checkpoint dir with model.safetensors + config)
- 2. a ``model/`` dir next to this file (how ``prepare.sh`` ships the Space)
- 3. the repo's ``data/models/pos2move_v2.1`` (for local dev runs)
- 4. download from GitHub (LFS media) as a last resort
-"""
-
-from __future__ import annotations
-
-import itertools
-import os
-import sys
-from pathlib import Path
-
-import chess
-import gradio as gr
-from gradio_chessboard import Chessboard
-
-# ── make `chesstransformer` importable ──────────────────────────────────────
-# On the Space the package is vendored next to this file (cwd is on sys.path).
-# For local dev runs from the repo, add the repo's src/ to the path.
-_repo_src = Path(__file__).resolve().parents[1] / "src"
-if (_repo_src / "chesstransformer").exists():
- sys.path.insert(0, str(_repo_src))
-
-from chesstransformer.bots import Pos2MoveV2MctsBot # noqa: E402
-
-START_FEN = chess.STARTING_FEN
-DEFAULT_SIMS = int(os.environ.get("MCTS_SIMS", "400"))
-
-# GitHub locations for the last-resort weight download.
-_GH = "tchauffi/ChessTransformer"
-_BRANCH = os.environ.get("MODEL_BRANCH", "main")
-_RAW = f"https://raw.githubusercontent.com/{_GH}/{_BRANCH}/data/models/pos2move_v2.1"
-_LFS = f"https://media.githubusercontent.com/media/{_GH}/{_BRANCH}/data/models/pos2move_v2.1"
-
-
-def _download_weights() -> str:
- import urllib.request
-
- dest = Path("/tmp/pos2move_v2.1")
- dest.mkdir(parents=True, exist_ok=True)
- for fname, url in [("model_config.json", f"{_RAW}/model_config.json"),
- ("model.safetensors", f"{_LFS}/model.safetensors")]:
- out = dest / fname
- if not out.exists():
- print(f"Downloading {fname} …")
- urllib.request.urlretrieve(url, out)
- return str(dest)
-
-
-def resolve_model_dir() -> str:
- env = os.environ.get("MODEL_PATH")
- if env and (Path(env) / "model.safetensors").exists():
- return env
- for cand in (
- Path(__file__).resolve().parent / "model",
- Path("model"),
- Path(__file__).resolve().parents[1] / "data" / "models" / "pos2move_v2.1",
- ):
- if (cand / "model.safetensors").exists():
- return str(cand)
- return _download_weights()
-
-
-# ── load the bot once (CPU, no compile) ─────────────────────────────────────
-MODEL_DIR = resolve_model_dir()
-bot = Pos2MoveV2MctsBot(
- model_dir=MODEL_DIR,
- num_simulations=DEFAULT_SIMS,
- device="cpu",
- compile=False,
- time_limit=0.0,
- # Sample the opening moves (∝ visit count) so games aren't identical every
- # time; switches to deterministic best-move play after the opening.
- move_temp=1.0,
- move_temp_plies=12,
-)
-
-
-def _human_color(human_color: str) -> bool:
- return chess.BLACK if human_color == "black" else chess.WHITE
-
-
-def _white_pov_value(board: chess.Board) -> float:
- """Value-head eval in [-1, 1] from White's perspective (+ = White better)."""
- if board.is_game_over():
- if board.is_checkmate():
- return -1.0 if board.turn == chess.WHITE else 1.0
- return 0.0 # stalemate / draw
- v = bot.get_value(board)
- if v is None:
- return 0.0
- return v if board.turn == chess.WHITE else -v
-
-
-def eval_bar_html(board: chess.Board, orientation: str) -> str:
- """A vertical eval bar (White share fills from the bottom of White's side).
- Flipped to match the board orientation so the viewer's side is at the bottom."""
- wv = _white_pov_value(board)
- white_pct = round(max(0.0, min(1.0, (wv + 1) / 2)) * 100, 1)
- white_div = f'
'
- black_div = f''
- # orientation "white": Black on top, White on bottom; flipped when "black".
- segments = black_div + white_div if orientation == "white" else white_div + black_div
- return (
- f'
'
- f'{segments}
'
- )
-
-
-def _result_text(board: chess.Board, human_color: str) -> str:
- outcome = board.outcome()
- if outcome is None:
- return ""
- if outcome.winner is None:
- return f"Draw ({outcome.termination.name.lower().replace('_', ' ')}). New game?"
- who = "You win! 🎉" if outcome.winner == _human_color(human_color) else "ChessTransformer wins."
- return f"Checkmate — {who} New game?"
-
-
-def _bot_move(board: chess.Board, sims: int):
- """Play the side to move with the bot; return its SAN and value."""
- bot.num_simulations = int(sims)
- uci, value = bot.predict(board)
- san = board.san(chess.Move.from_uci(uci))
- board.push_uci(uci)
- return san, value
-
-
-def on_move(fen: str, sims: int, human_color: str):
- """Human just moved; reply with the bot's move (whichever side is to move).
- The board orientation matches the human's color, so the eval bar does too.
-
- A generator: first yields a "thinking" status (so the player sees the bot is
- working during the ~1-3s CPU search), then yields the bot's reply."""
- board = chess.Board(fen)
- if board.is_game_over():
- yield board.fen(), _result_text(board, human_color), eval_bar_html(board, human_color)
- return
-
- # Immediate feedback while the search runs (board/eval unchanged for now).
- yield gr.update(), "⏳ ChessTransformer is thinking…", gr.update()
-
- san, value = _bot_move(board, sims)
-
- if board.is_game_over():
- status = f"Bot played **{san}**. {_result_text(board, human_color)}"
- else:
- eval_str = f" (eval {value:+.2f})" if value is not None else ""
- status = f"Bot played **{san}**{eval_str}. Your move."
- yield board.fen(), status, eval_bar_html(board, human_color)
-
-
-# Monotonic id so every New game yields a *distinct* setup dict — otherwise a
-# same-colour replay returns an identical dict and the gr.render block, seeing no
-# change, wouldn't rebuild (and reset) the board.
-_game_counter = itertools.count(1)
-
-
-def new_game(play_as: str, sims: int):
- """Start a fresh game. If the human plays Black, the bot (White) opens.
-
- Returns (setup, eval_bar). The setup dict drives a gr.render block — changing
- it rebuilds the board so its orientation (chessboard.js reads it only at
- construction) actually flips to the human's side."""
- human_color = "black" if play_as == "Black" else "white"
- board = chess.Board()
- if human_color == "black":
- san, _ = _bot_move(board, sims)
- status = f"New game — you are **Black**. Bot opened with **{san}**. Your move."
- else:
- status = "New game — you are **White**. Make your move."
- setup = {"fen": board.fen(), "orientation": human_color, "color": human_color,
- "status": status, "nonce": next(_game_counter)}
- return setup, eval_bar_html(board, human_color)
-
-
-with gr.Blocks(title="ChessTransformer", theme=gr.themes.Soft()) as demo:
- gr.Markdown(
- "# ♟️ ChessTransformer\n"
- "An **11.7M-parameter transformer trained only on human games** "
- "(no self-play, no RL), playing via AlphaZero-style MCTS. "
- "It reaches **~2100 Elo** vs Stockfish at full strength; this CPU demo "
- "runs at a lower sim count so moves come back quickly.\n\n"
- "**Pick your color and hit New game.** Drag a piece to move — the bot "
- "replies automatically. "
- "[GitHub repo →](https://github.com/tchauffi/ChessTransformer)"
- )
- setup = gr.State({"fen": START_FEN, "orientation": "white", "color": "white",
- "status": "You are **White**. Make your move.", "nonce": 0})
-
- with gr.Row():
- # Solid min_widths so the board container always has a size when the
- # gr.render block (re)mounts it — chessboard.js reloads the whole page if
- # it inits at 0 width, which is what caused the mobile flicker/reload loop.
- eval_col = gr.Column(scale=0, min_width=44)
- board_col = gr.Column(scale=3, min_width=340)
- ctrl_col = gr.Column(scale=1, min_width=200)
-
- with eval_col:
- eval_bar = gr.HTML(eval_bar_html(chess.Board(), "white"), visible=False)
-
- with ctrl_col:
- play_as = gr.Radio(["White", "Black"], value="White", label="Play as")
- sims = gr.Slider(32, 1200, value=DEFAULT_SIMS, step=8,
- label="Engine strength (MCTS sims/move)",
- info="Higher = stronger but slower on CPU")
- show_eval = gr.Checkbox(False, label="Show evaluation bar")
- new_btn = gr.Button("New game", variant="primary")
-
- # The board lives in a gr.render so switching color rebuilds it with the new
- # orientation (chessboard.js reads orientation only at construction). The
- # min_width above keeps the container sized so the remount can't init at 0.
- with board_col:
- @gr.render(inputs=setup)
- def render_board(cfg):
- board = Chessboard(value=cfg["fen"], game_mode=True, min_width=320,
- orientation=cfg["orientation"], label="Drag to move")
- status = gr.Markdown(cfg["status"])
- color = cfg["color"]
-
- def handle(fen, s): # generator → streams the "thinking" update first
- yield from on_move(fen, s, color)
-
- # show_progress="hidden": our "⏳ thinking…" status is the signal, so
- # Gradio's default overlay doesn't sit over the board.
- board.move(handle, inputs=[board, sims], outputs=[board, status, eval_bar],
- show_progress="hidden")
-
- show_eval.change(lambda show: gr.update(visible=show), inputs=show_eval, outputs=eval_bar)
- new_btn.click(new_game, inputs=[play_as, sims], outputs=[setup, eval_bar])
-
-
-if __name__ == "__main__":
- demo.launch()
diff --git a/chesstransformer/__init__.py b/chesstransformer/__init__.py
deleted file mode 100644
index 206d4bbf521b2669a51e26f8664f8bfcae6d47d1..0000000000000000000000000000000000000000
--- a/chesstransformer/__init__.py
+++ /dev/null
@@ -1 +0,0 @@
-"""This is the main package for the ChessTransformer project."""
diff --git a/chesstransformer/benchmark/__init__.py b/chesstransformer/benchmark/__init__.py
deleted file mode 100644
index cf5eab0057d7c46a1125073146523269aeac1c2f..0000000000000000000000000000000000000000
--- a/chesstransformer/benchmark/__init__.py
+++ /dev/null
@@ -1,3 +0,0 @@
-from .bot_benchmark import BotBenchmark
-
-__all__ = ["BotBenchmark"]
diff --git a/chesstransformer/benchmark/bot_benchmark.py b/chesstransformer/benchmark/bot_benchmark.py
deleted file mode 100644
index 6b447c5d04c691934e4592e276b452b6fb7051cc..0000000000000000000000000000000000000000
--- a/chesstransformer/benchmark/bot_benchmark.py
+++ /dev/null
@@ -1,399 +0,0 @@
-"""
-Bot benchmarking system for comparing chess bot performance.
-
-Usage:
- python -m chesstransformer.benchmark.bot_benchmark \\
- --bot1-type position2move --bot1-checkpoint path/to/checkpoint \\
- --bot2-type random \\
- --num-games 100 \\
- --output results.json
-"""
-
-import chess
-import time
-import json
-from pathlib import Path
-from typing import List, Tuple, Optional
-from dataclasses import dataclass, asdict
-from tqdm import tqdm
-
-from chesstransformer.bots import Position2MoveBot, LegacyPosition2MoveBot, RandomBot
-
-
-@dataclass
-class GameResult:
- """Single game result statistics."""
-
- game_id: int
- white_bot: str
- black_bot: str
- result: str # "1-0", "0-1", "1/2-1/2"
- num_moves: int
- white_time: float
- black_time: float
- termination: str # "checkmate", "stalemate", "insufficient_material", "max_moves"
-
- def to_dict(self):
- return asdict(self)
-
-
-@dataclass
-class BenchmarkStats:
- """Aggregate statistics for a bot."""
-
- bot_name: str
- total_games: int
- wins: int
- losses: int
- draws: int
- avg_moves_per_game: float
- avg_time_per_move: float
- total_time: float
- win_rate: float
- win_as_white: int
- win_as_black: int
-
- def to_dict(self):
- return asdict(self)
-
-
-class BotBenchmark:
- """Benchmark system for comparing chess bots."""
-
- def __init__(self, max_moves: int = 500, time_limit: Optional[float] = None):
- """
- Initialize benchmark.
-
- Args:
- max_moves: Maximum moves per game before declaring draw
- time_limit: Optional time limit per move in seconds
- """
- self.max_moves = max_moves
- self.time_limit = time_limit
- self.game_results: List[GameResult] = []
- self.bot_counter = 0 # Counter for unique bot IDs
-
- def create_bot(
- self, bot_type: str, checkpoint_path: Optional[str] = None, bot_name: Optional[str] = None, **kwargs
- ):
- """
- Factory method to create bots.
-
- Args:
- bot_type: Type of bot ("position2move", "legacy_position2move", "random")
- checkpoint_path: Path to model checkpoint (for Position2MoveBot)
- bot_name: Optional custom name for the bot (for tracking multiple instances)
- **kwargs: Additional bot-specific arguments
- """
- if bot_type.lower() == "position2move":
- if checkpoint_path:
- bot = Position2MoveBot(model_path=checkpoint_path)
- else:
- bot = Position2MoveBot()
- elif bot_type.lower() == "legacy_position2move":
- if checkpoint_path:
- bot = LegacyPosition2MoveBot(model_path=checkpoint_path)
- else:
- bot = LegacyPosition2MoveBot()
- elif bot_type.lower() == "random":
- bot = RandomBot()
- else:
- raise ValueError(f"Unknown bot type: {bot_type}")
-
- # Assign unique identifier
- if bot_name:
- bot._benchmark_id = bot_name
- else:
- self.bot_counter += 1
- bot._benchmark_id = f"{bot.__class__.__name__}_{self.bot_counter}"
-
- return bot
-
- def play_game(self, white_bot, black_bot, game_id: int, verbose: bool = False) -> GameResult:
- """
- Play a single game between two bots.
-
- Args:
- white_bot: Bot playing white
- black_bot: Bot playing black
- game_id: Unique game identifier
- verbose: Print move-by-move progress
-
- Returns:
- GameResult with statistics
- """
- board = chess.Board()
- white_time = 0.0
- black_time = 0.0
- move_count = 0
-
- while not board.is_game_over() and move_count < self.max_moves:
- current_bot = white_bot if board.turn == chess.WHITE else black_bot
-
- start_time = time.time()
-
- move, _ = current_bot.predict(board)
-
- elapsed = time.time() - start_time
-
- if board.turn == chess.WHITE:
- white_time += elapsed
- else:
- black_time += elapsed
-
- board.push(chess.Move.from_uci(move))
- move_count += 1
-
- if verbose and move_count % 10 == 0:
- print(f"Move {move_count}: {move}")
-
- # Determine result and termination reason
- if board.is_checkmate():
- result = "0-1" if board.turn == chess.WHITE else "1-0"
- termination = "checkmate"
- elif board.is_stalemate():
- result = "1/2-1/2"
- termination = "stalemate"
- elif board.is_insufficient_material():
- result = "1/2-1/2"
- termination = "insufficient_material"
- elif board.is_seventyfive_moves():
- result = "1/2-1/2"
- termination = "75_move_rule"
- elif board.is_fivefold_repetition():
- result = "1/2-1/2"
- termination = "repetition"
- elif move_count >= self.max_moves:
- result = "1/2-1/2"
- termination = "max_moves"
- else:
- # Should not reach here if game ended normally
- result = "1/2-1/2"
- termination = "unknown"
-
- return GameResult(
- game_id=game_id,
- white_bot=white_bot._benchmark_id,
- black_bot=black_bot._benchmark_id,
- result=result,
- num_moves=move_count,
- white_time=white_time,
- black_time=black_time,
- termination=termination,
- )
-
- def run_benchmark(
- self, bot1, bot2, num_games: int, alternate_colors: bool = True, verbose: bool = False
- ) -> Tuple[BenchmarkStats, BenchmarkStats]:
- """
- Run full benchmark between two bots.
-
- Args:
- bot1: First bot
- bot2: Second bot
- num_games: Number of games to play
- alternate_colors: Alternate colors each game
- verbose: Print game-by-game results
-
- Returns:
- Tuple of (bot1_stats, bot2_stats)
- """
- self.game_results = []
-
- pbar = tqdm(range(num_games), desc="Playing games")
- for game_id in pbar:
- if alternate_colors and game_id % 2 == 1:
- # Swap colors
- white_bot, black_bot = bot2, bot1
- else:
- white_bot, black_bot = bot1, bot2
-
- result = self.play_game(white_bot, black_bot, game_id, verbose=verbose)
- self.game_results.append(result)
-
- if verbose:
- print(f"Game {game_id + 1}: {result.white_bot} vs {result.black_bot} = {result.result}")
-
- # Update progress bar with current stats
- bot1_wins = sum(1 for r in self.game_results if self._bot_won(r, bot1._benchmark_id))
- bot2_wins = sum(1 for r in self.game_results if self._bot_won(r, bot2._benchmark_id))
- pbar.set_postfix({bot1._benchmark_id: bot1_wins, bot2._benchmark_id: bot2_wins})
-
- # Calculate statistics
- bot1_stats = self._calculate_stats(bot1._benchmark_id)
- bot2_stats = self._calculate_stats(bot2._benchmark_id)
-
- return bot1_stats, bot2_stats
-
- def _bot_won(self, result: GameResult, bot_name: str) -> bool:
- """Check if bot won the game."""
- if bot_name == result.white_bot and result.result == "1-0":
- return True
- if bot_name == result.black_bot and result.result == "0-1":
- return True
- return False
-
- def _calculate_stats(self, bot_name: str) -> BenchmarkStats:
- """Calculate aggregate statistics for a bot."""
- bot_results = []
- wins = 0
- losses = 0
- draws = 0
- win_as_white = 0
- win_as_black = 0
- total_time = 0.0
- total_moves = 0
-
- for result in self.game_results:
- is_white = result.white_bot == bot_name
- is_black = result.black_bot == bot_name
-
- if not (is_white or is_black):
- continue
-
- bot_results.append(result)
- total_moves += result.num_moves
-
- if is_white:
- total_time += result.white_time
- if result.result == "1-0":
- wins += 1
- win_as_white += 1
- elif result.result == "0-1":
- losses += 1
- else:
- draws += 1
- else:
- total_time += result.black_time
- if result.result == "0-1":
- wins += 1
- win_as_black += 1
- elif result.result == "1-0":
- losses += 1
- else:
- draws += 1
-
- total_games = len(bot_results)
- avg_moves = total_moves / total_games if total_games > 0 else 0
- avg_time_per_move = total_time / total_moves if total_moves > 0 else 0
- win_rate = wins / total_games if total_games > 0 else 0
-
- return BenchmarkStats(
- bot_name=bot_name,
- total_games=total_games,
- wins=wins,
- losses=losses,
- draws=draws,
- avg_moves_per_game=avg_moves,
- avg_time_per_move=avg_time_per_move,
- total_time=total_time,
- win_rate=win_rate,
- win_as_white=win_as_white,
- win_as_black=win_as_black,
- )
-
- def save_results(self, output_path: str):
- """Save benchmark results to JSON file."""
- output = {
- "config": {"max_moves": self.max_moves, "time_limit": self.time_limit, "num_games": len(self.game_results)},
- "games": [r.to_dict() for r in self.game_results],
- "stats": {
- bot_name: self._calculate_stats(bot_name).to_dict()
- for bot_name in set([r.white_bot for r in self.game_results] + [r.black_bot for r in self.game_results])
- },
- }
-
- output_file = Path(output_path)
- output_file.parent.mkdir(parents=True, exist_ok=True)
-
- with output_file.open("w", encoding="utf-8") as f:
- json.dump(output, f, indent=2)
-
- print(f"Results saved to: {output_path}")
-
- def print_summary(self, bot1_stats: BenchmarkStats, bot2_stats: BenchmarkStats):
- """Print formatted summary of benchmark results."""
- print("\n" + "=" * 80)
- print("BENCHMARK RESULTS")
- print("=" * 80)
-
- for stats in [bot1_stats, bot2_stats]:
- print(f"\n{stats.bot_name}:")
- print(f" Total Games: {stats.total_games}")
- print(f" Wins: {stats.wins} ({stats.win_rate:.1%})")
- print(f" - As White: {stats.win_as_white}")
- print(f" - As Black: {stats.win_as_black}")
- print(f" Losses: {stats.losses}")
- print(f" Draws: {stats.draws}")
- print(f" Avg Moves/Game: {stats.avg_moves_per_game:.1f}")
- print(f" Avg Time/Move: {stats.avg_time_per_move:.3f}s")
- print(f" Total Time: {stats.total_time:.1f}s")
-
- print("\n" + "=" * 80)
-
-
-def main():
- """CLI entry point for bot benchmarking."""
- import argparse
-
- parser = argparse.ArgumentParser(description="Benchmark chess bots")
- parser.add_argument(
- "--bot1-type",
- type=str,
- required=True,
- choices=["position2move", "legacy_position2move", "random"],
- help="Type of first bot",
- )
- parser.add_argument("--bot1-checkpoint", type=str, default=None, help="Checkpoint path for bot1 (if applicable)")
- parser.add_argument(
- "--bot1-name", type=str, default=None, help="Custom name for bot1 (useful when comparing same bot types)"
- )
- parser.add_argument(
- "--bot2-type",
- type=str,
- required=True,
- choices=["position2move", "legacy_position2move", "random"],
- help="Type of second bot",
- )
- parser.add_argument("--bot2-checkpoint", type=str, default=None, help="Checkpoint path for bot2 (if applicable)")
- parser.add_argument(
- "--bot2-name", type=str, default=None, help="Custom name for bot2 (useful when comparing same bot types)"
- )
- parser.add_argument("--num-games", type=int, default=100, help="Number of games to play")
- parser.add_argument("--max-moves", type=int, default=500, help="Maximum moves per game")
- parser.add_argument("--time-limit", type=float, default=None, help="Time limit per move in seconds")
- parser.add_argument("--output", type=str, default="benchmark_results.json", help="Output file for results")
- parser.add_argument("--no-alternate", action="store_true", help="Don't alternate colors between games")
- parser.add_argument("--verbose", action="store_true", help="Print detailed game information")
-
- args = parser.parse_args()
-
- # Create benchmark
- benchmark = BotBenchmark(max_moves=args.max_moves, time_limit=args.time_limit)
-
- # Create bots
- print(f"Creating {args.bot1_type} bot...")
- bot1 = benchmark.create_bot(args.bot1_type, checkpoint_path=args.bot1_checkpoint, bot_name=args.bot1_name)
-
- print(f"Creating {args.bot2_type} bot...")
- bot2 = benchmark.create_bot(args.bot2_type, checkpoint_path=args.bot2_checkpoint, bot_name=args.bot2_name)
-
- # Run benchmark
- print(f"\nRunning benchmark: {args.num_games} games")
- print(f"Max moves per game: {args.max_moves}")
- if args.time_limit:
- print(f"Time limit per move: {args.time_limit}s")
-
- bot1_stats, bot2_stats = benchmark.run_benchmark(
- bot1, bot2, num_games=args.num_games, alternate_colors=not args.no_alternate, verbose=args.verbose
- )
-
- # Print summary
- benchmark.print_summary(bot1_stats, bot2_stats)
-
- # Save results
- benchmark.save_results(args.output)
-
-
-if __name__ == "__main__":
- main()
diff --git a/chesstransformer/bots/__init__.py b/chesstransformer/bots/__init__.py
deleted file mode 100644
index 93d475e795d897bd7a43e6e647637e42ae86d57c..0000000000000000000000000000000000000000
--- a/chesstransformer/bots/__init__.py
+++ /dev/null
@@ -1,5 +0,0 @@
-from .pos2move_v2_bot import Pos2MoveV2Bot
-from .pos2move_v2_mcts_bot import Pos2MoveV2MctsBot
-from .random_bot import RandomBot
-
-__all__ = ["Pos2MoveV2Bot", "Pos2MoveV2MctsBot", "RandomBot"]
diff --git a/chesstransformer/bots/legacy_position2move_bot.py b/chesstransformer/bots/legacy_position2move_bot.py
deleted file mode 100644
index 941ec8b68305c437c4ce257b10671680744eab4c..0000000000000000000000000000000000000000
--- a/chesstransformer/bots/legacy_position2move_bot.py
+++ /dev/null
@@ -1,222 +0,0 @@
-"""Legacy Position2Move bot for V2 model compatibility.
-
-The V2 model was trained with an older architecture that uses:
-- MultiHeadAttention instead of GroupedQueryAttention
-- LayerNorm with gamma/beta instead of RMSNorm with weight
-- Standard GELU FeedForward instead of SwiGLU
-- No CLS token, castling, en passant, or halfmove embeddings
-"""
-
-from pathlib import Path
-import json
-import torch
-import torch.nn as nn
-import chess
-from safetensors import safe_open
-from chesstransformer.models.transformer.attention import MultiHeadAttention
-from chesstransformer.models.tokenizer import PostionTokenizer, MoveTokenizer
-
-
-class LayerNorm(nn.Module):
- """Legacy LayerNorm with gamma/beta naming."""
-
- def __init__(self, embed_dim):
- super().__init__()
- self.gamma = nn.Parameter(torch.ones(embed_dim))
- self.beta = nn.Parameter(torch.zeros(embed_dim))
- self.eps = 1e-6
-
- def forward(self, x):
- mean = x.mean(dim=-1, keepdim=True)
- var = x.var(dim=-1, unbiased=False, keepdim=True)
- return self.gamma * (x - mean) / torch.sqrt(var + self.eps) + self.beta
-
-
-class FeedForward(nn.Module):
- """Legacy FeedForward network."""
-
- def __init__(self, embed_dim):
- super().__init__()
- self.layers = nn.Sequential(
- nn.Linear(embed_dim, 4 * embed_dim),
- nn.GELU(approximate="tanh"),
- nn.Linear(4 * embed_dim, embed_dim),
- )
-
- def forward(self, x):
- return self.layers(x)
-
-
-class LegacyTransformerBlock(nn.Module):
- """Legacy transformer block for V2 model."""
-
- def __init__(
- self,
- embed_dim,
- num_heads,
- context_length,
- dropout,
- qkv_bias,
- mask_future=False,
- ):
- super().__init__()
- self.att = MultiHeadAttention(
- d_in=embed_dim,
- d_out=embed_dim,
- num_heads=num_heads,
- context_length=context_length,
- dropout=dropout,
- qkv_bias=qkv_bias,
- mask_future=mask_future,
- )
- self.feed_forward = FeedForward(embed_dim)
- self.norm1 = LayerNorm(embed_dim)
- self.norm2 = LayerNorm(embed_dim)
- self.dropout_shortcut = nn.Dropout(dropout)
-
- def forward(self, x):
- shortcut = x
- x = self.norm1(x)
- x = self.att(x)
- x = self.dropout_shortcut(x)
- x = x + shortcut
-
- shortcut = x
- x = self.norm2(x)
- x = self.feed_forward(x)
- x = self.dropout_shortcut(x)
- x = x + shortcut
-
- return x
-
-
-class LegacyPosition2MoveModel(nn.Module):
- """Legacy Position2Move model architecture for V2 compatibility."""
-
- def __init__(
- self,
- vocab_size: int = 13,
- move_vocab_size: int = 1968,
- embed_dim: int = 512,
- nb_transformer_layers: int = 8,
- num_heads: int = 8,
- dropout: float = 0.1,
- kvq_bias: bool = False,
- mask_future: bool = False,
- ):
- super().__init__()
-
- # Board piece embeddings
- self.token_embedding = nn.Embedding(vocab_size, embed_dim)
- self.position_embedding = nn.Embedding(64, embed_dim) # 64 squares
- self.player_embedding = nn.Embedding(2, embed_dim) # 2 players
-
- self.dropout = nn.Dropout(dropout)
- self.transformer_blocks = nn.Sequential(
- *[
- LegacyTransformerBlock(
- embed_dim=embed_dim,
- num_heads=num_heads,
- context_length=64,
- dropout=dropout,
- qkv_bias=kvq_bias,
- mask_future=mask_future,
- )
- for _ in range(nb_transformer_layers)
- ]
- )
-
- self.norm = LayerNorm(embed_dim)
- self.lm_head = nn.Linear(embed_dim, move_vocab_size, bias=False)
-
- def forward(self, x, is_white):
- batch_size, seq_len = x.shape
-
- # Get embeddings
- token_emb = self.token_embedding(x)
- pos_indices = torch.arange(seq_len, device=x.device)
- pos_emb = self.position_embedding(pos_indices)
- player_emb = self.player_embedding(is_white.long())
-
- # Combine embeddings
- x = token_emb + pos_emb.unsqueeze(0) + player_emb.unsqueeze(1)
- x = self.dropout(x)
-
- # Transformer blocks
- x = self.transformer_blocks(x)
- x = self.norm(x)
-
- # Mean pooling
- x = x.mean(dim=1)
-
- # Output
- logits = self.lm_head(x)
- return logits
-
-
-data_folder = Path(__file__).parents[1] / "data"
-
-
-class LegacyPosition2MoveBot:
- """Bot using the legacy V2 model architecture."""
-
- def __init__(
- self,
- model_path: str = str((data_folder / "models/position2moveV2.0/model.safetensors").resolve()),
- device: str = "cpu",
- ):
- self.device = device
- self.position_tokenizer = PostionTokenizer()
- self.move_tokenizer = MoveTokenizer()
-
- config_path = Path(model_path).parent / "config.json"
-
- with open(config_path, "r") as f:
- self.config = json.load(f)
-
- # Load legacy model
- self.model = LegacyPosition2MoveModel(**self.config).to(device)
-
- with safe_open(model_path, framework="pt", device=device) as f:
- state_dict = {k: f.get_tensor(k) for k in f.keys()}
- self.model.load_state_dict(state_dict)
-
- self.model.eval()
-
- @torch.no_grad()
- def predict(self, board: chess.Board):
- tokens_ids = self.position_tokenizer.encode(board)
- torch_input = torch.tensor(tokens_ids).unsqueeze(0).long().to(self.device)
-
- is_white = board.turn
- is_white = torch.tensor([is_white]).bool().to(self.device)
-
- logits = self.model(torch_input, is_white)
-
- legal_moves = [m.uci() for m in board.legal_moves]
-
- mask = torch.full((logits.size(-1),), float("-inf")).to(self.device)
- for move in legal_moves:
- move_id = self.move_tokenizer.encode(move)
- mask[move_id] = 0.0
-
- masked_logits = logits[0] + mask
- probs = torch.softmax(masked_logits, dim=-1)
-
- # Sample from distribution with temperature
- predicted_move_id = torch.multinomial(probs * 0.2, num_samples=1).item()
- proba = probs[predicted_move_id].item()
-
- predicted_move = self.move_tokenizer.decode(predicted_move_id)
- return predicted_move, proba
-
-
-if __name__ == "__main__":
- bot = LegacyPosition2MoveBot()
-
- board = chess.Board()
- print("Initial board:")
- print(board)
-
- move, proba = bot.predict(board)
- print(f"Bot suggests move: {move} with probability {proba:.4f}")
diff --git a/chesstransformer/bots/pos2move_v2_bot.py b/chesstransformer/bots/pos2move_v2_bot.py
deleted file mode 100644
index d40be50a21f52cc8fbc6c4847d30a4ac344c1689..0000000000000000000000000000000000000000
--- a/chesstransformer/bots/pos2move_v2_bot.py
+++ /dev/null
@@ -1,499 +0,0 @@
-"""Bot for Pos2MoveV2 with alpha-beta search + BF16."""
-
-from __future__ import annotations
-
-import json
-import time as _time
-from pathlib import Path
-
-import chess
-import chess.polyglot
-import numpy as np
-import torch
-from safetensors import safe_open
-
-from chesstransformer.models.tokenizer import PostionTokenizer
-from chesstransformer.models.tokenizer.alphazero_move_encoder import move_to_action_plane
-from chesstransformer.models.transformer.pos2move_v2 import Pos2MoveV2
-
-_DEFAULT_RUN = Path(__file__).parents[3] / "data" / "models" / "pos2move_v2.1"
-
-# Fixed batch-size buckets. Batched evals are zero-padded up to the next bucket
-# so the (optionally compiled) model only ever sees a handful of static shapes.
-_BATCH_BUCKETS = (4, 8, 16, 32)
-
-
-class _SearchTimeout(Exception):
- pass
-
-
-class Pos2MoveV2Bot:
- def __init__(
- self,
- model_dir: str | Path = _DEFAULT_RUN,
- depth: int = 3,
- top_p: float = 0.90,
- time_limit: float = 30.0,
- tt_size: int = 500_000,
- device: str = "cuda" if torch.cuda.is_available() else "cpu",
- use_ema: bool = False,
- compile: bool = True,
- quiescence_depth: int = 4,
- ):
- self.depth = depth
- self.top_p = top_p
- self.time_limit = time_limit
- self.quiescence_depth = quiescence_depth
- self.nodes_searched = 0
- self.tt_hits = 0
- self.completed_depth = 0
- self.device = device
- self.position_tokenizer = PostionTokenizer()
-
- model_dir = Path(model_dir)
-
- config_path = model_dir / "model_config.json"
- if not config_path.exists():
- config_path = model_dir.parent.parent / "model_config.json"
- if not config_path.exists():
- raise FileNotFoundError(f"No model_config.json found for {model_dir}")
-
- with open(config_path) as f:
- config = json.load(f)
-
- model = Pos2MoveV2(**config)
-
- safetensors_path = model_dir / "model.safetensors"
- if not safetensors_path.exists():
- raise FileNotFoundError(f"No model.safetensors in {model_dir}")
- with safe_open(str(safetensors_path), framework="pt", device="cpu") as f:
- state_dict = {k: f.get_tensor(k) for k in f.keys()}
-
- if any(k.startswith("_orig_mod.") for k in state_dict):
- state_dict = {k.replace("_orig_mod.", ""): v for k, v in state_dict.items()}
-
- model.load_state_dict(state_dict)
-
- if use_ema:
- ema_path = model_dir / "ema_state.pt"
- if not ema_path.exists():
- raise FileNotFoundError(f"No ema_state.pt in {model_dir}")
- ema_state = torch.load(str(ema_path), map_location="cpu", weights_only=True)
- ema_state = {k.replace("_orig_mod.", ""): v for k, v in ema_state.items()}
- missing, unexpected = model.load_state_dict(ema_state, strict=False)
- if unexpected:
- print(f"WARNING: EMA contains unexpected keys: {unexpected}")
- print(f"EMA weights loaded ({len(ema_state)} tensors; {len(missing)} buffers kept).")
-
- model.eval()
- self.model = model.bfloat16().to(device)
-
- self._buf_board = torch.zeros(1, 64, dtype=torch.long, device=device)
- self._buf_player = torch.zeros(1, dtype=torch.long, device=device)
- self._buf_castling = torch.zeros(1, dtype=torch.long, device=device)
- self._buf_ep = torch.zeros(1, dtype=torch.long, device=device)
- self._batch_bufs: dict[int, tuple] = {}
-
- self._tt_size = tt_size
- self._tt: dict[int, tuple[np.ndarray, float]] = {}
- self._deadline: float = 0.0
-
- # CUDA graphs collapse the Python/kernel-launch overhead that dominates
- # this tiny model at small batch sizes. Bucketed batches keep the number
- # of captured graphs small. Falls back to eager if compilation fails.
- self._compiled = False
- if compile and device == "cuda":
- self._eager_model = self.model
- try:
- self.model = torch.compile(self.model, mode="reduce-overhead")
- self._compiled = True
- except Exception as e: # pragma: no cover - environment dependent
- print(f"WARNING: torch.compile unavailable ({e}); using eager.")
- self.model = self._eager_model
-
- self._warmup()
-
- print(
- f"Pos2MoveV2Bot loaded (device={device}, depth={depth}, "
- f"top_p={top_p:.0%}, params={sum(p.numel() for p in self.model.parameters()) / 1e6:.1f}M)"
- )
-
- @torch.no_grad()
- def _warmup(self):
- try:
- # Batch-1 path (single-position _evaluate).
- for _ in range(3):
- self.model(self._buf_board, self._buf_player, self._buf_castling, self._buf_ep)
- # Each bucket shape used by _batch_run, so compilation / CUDA-graph
- # capture happens up front rather than mid-search.
- for bucket in _BATCH_BUCKETS:
- b, p, c, e = self._batch_buffers(bucket)
- for _ in range(3):
- self.model(b, p, c, e)
- except Exception as e: # pragma: no cover - environment dependent
- if self._compiled:
- print(f"WARNING: compiled warmup failed ({e}); falling back to eager.")
- self.model = self._eager_model
- self._compiled = False
- self.model(self._buf_board, self._buf_player, self._buf_castling, self._buf_ep)
- else:
- raise
- if self.device == "cuda":
- torch.cuda.synchronize()
-
- def _adaptive_time_limit(self, board: chess.Board) -> float:
- if self.time_limit <= 0:
- return 0.0
- ply = len(board.move_stack)
- move_number = ply // 2 + 1
- if move_number <= 10:
- scale = 0.10
- elif move_number <= 20:
- scale = 0.40
- elif move_number <= 40:
- scale = 1.0
- else:
- scale = 0.60
- return self.time_limit * scale
-
- def predict(self, board: chess.Board) -> tuple[str, float | None]:
- self.nodes_searched = 0
- self.tt_hits = 0
- self.completed_depth = 0
-
- effective_limit = self._adaptive_time_limit(board)
- self._deadline = _time.monotonic() + effective_limit if effective_limit > 0 else float("inf")
-
- best_move = None
- best_value = 0.0
-
- for d in range(1, self.depth + 1):
- stack_len = len(board.move_stack)
- try:
- move, value = self._search_root(board, d)
- best_move = move
- best_value = value
- self.completed_depth = d
- except _SearchTimeout:
- while len(board.move_stack) > stack_len:
- board.pop()
- break
-
- if best_move is None:
- best_move = next(iter(board.legal_moves))
-
- return best_move.uci(), best_value
-
- def get_value(self, board: chess.Board) -> float:
- _, value = self._evaluate(board)
- return value
-
- def _check_time(self):
- if _time.monotonic() >= self._deadline:
- raise _SearchTimeout()
-
- @torch.no_grad()
- def _evaluate(self, board: chess.Board) -> tuple[np.ndarray, float]:
- key = chess.polyglot.zobrist_hash(board)
- cached = self._tt.get(key)
- if cached is not None:
- self.tt_hits += 1
- return cached
-
- position = self.position_tokenizer.encode(board)
- self._buf_board[0] = torch.tensor(position, dtype=torch.long, device=self.device)
- self._buf_player[0] = int(board.turn)
-
- castling = 0
- if board.has_kingside_castling_rights(chess.WHITE):
- castling |= 1
- if board.has_queenside_castling_rights(chess.WHITE):
- castling |= 2
- if board.has_kingside_castling_rights(chess.BLACK):
- castling |= 4
- if board.has_queenside_castling_rights(chess.BLACK):
- castling |= 8
- self._buf_castling[0] = castling
-
- ep_file = chess.square_file(board.ep_square) if board.has_legal_en_passant() else 8
- self._buf_ep[0] = ep_file
-
- move_logits, value_t = self.model(
- self._buf_board, self._buf_player, self._buf_castling, self._buf_ep
- )
-
- logits = move_logits[0].float().cpu().numpy()
- value = float(value_t.item())
-
- if self._tt_size > 0 and len(self._tt) >= self._tt_size:
- del self._tt[next(iter(self._tt))]
- self._tt[key] = (logits, value)
-
- return logits, value
-
- def _batch_buffers(self, bucket: int):
- """Persistent zero-padded device buffers per bucket size.
-
- Using a small fixed set of batch shapes keeps torch.compile / CUDA
- graphs from recompiling (and re-capturing) on every distinct batch size.
- """
- bufs = self._batch_bufs.get(bucket)
- if bufs is None:
- bufs = (
- torch.zeros(bucket, 64, dtype=torch.long, device=self.device),
- torch.zeros(bucket, dtype=torch.long, device=self.device),
- torch.zeros(bucket, dtype=torch.long, device=self.device),
- torch.zeros(bucket, dtype=torch.long, device=self.device),
- )
- self._batch_bufs[bucket] = bufs
- return bufs
-
- @staticmethod
- def _bucket_for(n: int) -> int:
- for b in _BATCH_BUCKETS:
- if n <= b:
- return b
- return _BATCH_BUCKETS[-1]
-
- @torch.no_grad()
- def _batch_run(self, items: list[tuple[np.ndarray, int, int, int, int]]):
- n = len(items)
- bucket = self._bucket_for(n)
- board_arr, player_arr, castling_arr, ep_arr = self._batch_buffers(bucket)
- # Zero the padding rows (token/index 0 is a valid embedding index).
- board_arr.zero_()
- player_arr.zero_()
- castling_arr.zero_()
- ep_arr.zero_()
-
- for i, (pos, player, castling, ep, _) in enumerate(items):
- board_arr[i] = torch.tensor(pos, dtype=torch.long)
- player_arr[i] = player
- castling_arr[i] = castling
- ep_arr[i] = ep
-
- move_logits, value_t = self.model(board_arr, player_arr, castling_arr, ep_arr)
- # Slice off padding before the host copy.
- logits_np = move_logits[:n].float().cpu().numpy()
- values_np = value_t[:n].float().cpu().numpy()
-
- for i, (_, _, _, _, key) in enumerate(items):
- if self._tt_size > 0 and len(self._tt) >= self._tt_size:
- del self._tt[next(iter(self._tt))]
- self._tt[key] = (logits_np[i], float(values_np[i].item()))
-
- def _encode_position(self, board: chess.Board) -> tuple[np.ndarray, int, int, int]:
- position = self.position_tokenizer.encode(board)
- player = int(board.turn)
- castling = 0
- if board.has_kingside_castling_rights(chess.WHITE):
- castling |= 1
- if board.has_queenside_castling_rights(chess.WHITE):
- castling |= 2
- if board.has_kingside_castling_rights(chess.BLACK):
- castling |= 4
- if board.has_queenside_castling_rights(chess.BLACK):
- castling |= 8
- ep_file = chess.square_file(board.ep_square) if board.has_legal_en_passant() else 8
- return position, player, castling, ep_file
-
- def _prefetch_children(self, board: chess.Board, moves: list[chess.Move]):
- uncached = []
- for move in moves:
- board.push(move)
- key = chess.polyglot.zobrist_hash(board)
- # Cheap terminal proxy: positions with no legal moves (checkmate /
- # stalemate) are scored directly by _alpha_beta, so skip a wasted
- # model eval on them. Claimable-draw positions (threefold / 50-move)
- # still have legal moves; caching their eval is harmless because
- # _alpha_beta's authoritative is_game_over check overrides it.
- if key not in self._tt and any(board.legal_moves):
- pos, player, castling, ep_file = self._encode_position(board)
- uncached.append((pos, player, castling, ep_file, key))
- board.pop()
- if len(uncached) >= 2:
- self._batch_run(uncached)
-
- def _top_p_for_depth(self, depth: int, max_depth: int) -> float:
- plies_from_root = max_depth - depth
- if plies_from_root == 0:
- return self.top_p
- elif plies_from_root <= 2:
- return self.top_p * 0.7
- else:
- return self.top_p * 0.5
-
- def _search_root(self, board: chess.Board, depth: int) -> tuple[chess.Move, float]:
- ordered_moves = self._order_moves(board, self.top_p)
- if not ordered_moves:
- return chess.Move.null(), 0.0
- self._prefetch_children(board, [m for m, _ in ordered_moves])
- best_move = ordered_moves[0][0]
- alpha = float("-inf")
- for move, _ in ordered_moves:
- self._check_time()
- board.push(move)
- value = -self._alpha_beta(board, depth - 1, -float("inf"), -alpha, depth)
- board.pop()
- if value > alpha:
- alpha = value
- best_move = move
- return best_move, alpha
-
- def _terminal_value(self, board: chess.Board) -> float | None:
- """Negamax terminal value (side-to-move POV), or None if not terminal.
-
- Uses cheap checks instead of ``is_game_over(claim_draw=True)`` (whose
- ``can_claim_threefold_repetition`` dominated the profile): no-legal-move
- positions are mate/stalemate, and draws are detected via material,
- 3-fold repetition and the 50-move clock — without the expensive
- "a move *could* force a repetition" probe.
- """
- if not any(board.legal_moves):
- return -1.0 if board.is_check() else 0.0
- if (
- board.is_insufficient_material()
- or board.is_repetition(3)
- or board.halfmove_clock >= 100
- ):
- return 0.0
- return None
-
- @torch.no_grad()
- def _quiescence(self, board: chess.Board, alpha: float, beta: float, qdepth: int) -> float:
- """Resolve tactical noise before trusting the value head.
-
- Stand-pat on the value head, then search only forcing moves (captures
- and promotions) so the leaf isn't evaluated in the middle of an
- unresolved exchange (the horizon effect).
- """
- self.nodes_searched += 1
- self._check_time()
-
- term = self._terminal_value(board)
- if term is not None:
- return term
-
- logits, stand_pat = self._evaluate(board)
- if stand_pat >= beta:
- return beta
- if stand_pat > alpha:
- alpha = stand_pat
- if qdepth <= 0:
- return alpha
-
- noisy = []
- for move in board.legal_moves:
- if board.is_capture(move) or move.promotion:
- plane = move_to_action_plane(move.from_square, move.to_square, move.promotion)
- noisy.append((float(logits[move.from_square, plane]), move))
- if not noisy:
- return alpha
- noisy.sort(key=lambda x: -x[0])
-
- for _, move in noisy:
- board.push(move)
- score = -self._quiescence(board, -beta, -alpha, qdepth - 1)
- board.pop()
- if score >= beta:
- return beta
- if score > alpha:
- alpha = score
- return alpha
-
- def _alpha_beta(self, board: chess.Board, depth: int, alpha: float, beta: float, max_depth: int) -> float:
- self.nodes_searched += 1
- self._check_time()
-
- term = self._terminal_value(board)
- if term is not None:
- return term
-
- if depth <= 0:
- if self.quiescence_depth > 0:
- return self._quiescence(board, alpha, beta, self.quiescence_depth)
- _, value = self._evaluate(board)
- return value
-
- p = self._top_p_for_depth(depth, max_depth)
- ordered_moves = self._order_moves(board, p)
- self._prefetch_children(board, [m for m, _ in ordered_moves])
-
- for move, _ in ordered_moves:
- board.push(move)
- score = -self._alpha_beta(board, depth - 1, -beta, -alpha, max_depth)
- board.pop()
- if score >= beta:
- return beta
- if score > alpha:
- alpha = score
- return alpha
-
- def _order_moves(self, board: chess.Board, p: float, min_moves: int = 2, max_moves: int = 30):
- logits, _ = self._evaluate(board)
- moves = []
- raw_scores = []
- for move in board.legal_moves:
- plane = move_to_action_plane(move.from_square, move.to_square, move.promotion)
- raw_scores.append(float(logits[move.from_square, plane]))
- moves.append(move)
- if not moves:
- return []
-
- scores = np.array(raw_scores, dtype=np.float64)
- scores -= scores.max()
- exp_scores = np.exp(scores)
- probs = exp_scores / exp_scores.sum()
-
- order = np.argsort(-probs)
- cumsum = 0.0
- selected = []
- for idx in order:
- selected.append((moves[idx], float(probs[idx])))
- cumsum += probs[idx]
- if cumsum >= p and len(selected) >= min_moves:
- break
- if len(selected) >= max_moves:
- break
-
- while len(selected) < min_moves and len(selected) < len(moves):
- selected.append((moves[order[len(selected)]], float(probs[order[len(selected)]])))
-
- return selected
-
-
-if __name__ == "__main__":
- import argparse
- import time
-
- parser = argparse.ArgumentParser()
- parser.add_argument("model_dir", nargs="?", default=str(_DEFAULT_RUN))
- parser.add_argument("depth", nargs="?", type=int, default=3)
- parser.add_argument("top_p", nargs="?", type=float, default=0.90)
- parser.add_argument("--ema", action="store_true")
- args = parser.parse_args()
-
- bot = Pos2MoveV2Bot(
- model_dir=args.model_dir, depth=args.depth, top_p=args.top_p,
- time_limit=30.0, use_ema=args.ema,
- )
-
- board = chess.Board()
- print(f"\nStarting position:\n{board}\n")
-
- for i in range(10):
- t0 = time.time()
- move_uci, value = bot.predict(board)
- elapsed = time.time() - t0
- print(
- f"{'White' if board.turn else 'Black'} plays {move_uci} "
- f"(depth={bot.completed_depth}/{args.depth}, value={value:+.3f}, "
- f"nodes={bot.nodes_searched}, tt_hits={bot.tt_hits}, time={elapsed:.2f}s)"
- )
- board.push_uci(move_uci)
- print(board)
- print()
-
- if board.is_game_over():
- print(f"Game over: {board.result()}")
- break
diff --git a/chesstransformer/bots/pos2move_v2_mcts_bot.py b/chesstransformer/bots/pos2move_v2_mcts_bot.py
deleted file mode 100644
index caa54576b7326aa142fe747f6696f89040cdfab9..0000000000000000000000000000000000000000
--- a/chesstransformer/bots/pos2move_v2_mcts_bot.py
+++ /dev/null
@@ -1,318 +0,0 @@
-"""AlphaZero-style PUCT MCTS engine for Pos2MoveV2.
-
-Subclasses :class:`Pos2MoveV2Bot` to reuse its (torch.compiled, transposition-
-cached) ``_evaluate`` — the policy head supplies move priors and the value head
-scores leaves. Move selection is by visit count (the standard AlphaZero choice).
-
-Node representation is **edge-centric**: a node stores its children's visit
-counts / values / priors as numpy arrays, and child ``_Node`` objects are created
-lazily only when a child is actually descended into. This avoids allocating a
-node per legal move at every expansion (which dominated latency) and lets PUCT
-selection run vectorized over the arrays. Leaves are collected in batches with
-virtual loss so the network sees one batched forward per wave.
-"""
-
-from __future__ import annotations
-
-import math
-
-import chess
-import numpy as np
-
-from chesstransformer.bots.pos2move_v2_bot import _BATCH_BUCKETS, _DEFAULT_RUN, Pos2MoveV2Bot
-from chesstransformer.models.tokenizer.alphazero_move_encoder import move_to_action_plane
-
-
-class _Node:
- """A search node. Child visit stats live in arrays here (edge-centric); child
- ``_Node`` objects are created lazily on first descent. A node is *expanded*
- once ``moves`` is set."""
-
- __slots__ = ("moves", "priors", "child_N", "child_W", "children")
-
- def __init__(self):
- self.moves: list[chess.Move] | None = None
- self.priors: np.ndarray | None = None
- self.child_N: np.ndarray | None = None # int64, per child
- self.child_W: np.ndarray | None = None # float64, child-POV total value
- self.children: list[_Node | None] | None = None
-
- @property
- def expanded(self) -> bool:
- return self.moves is not None
-
-
-class Pos2MoveV2MctsBot(Pos2MoveV2Bot):
- def __init__(
- self,
- *args,
- num_simulations: int = 800,
- c_puct: float = 1.0,
- prior_temp: float = 1.0,
- fpu: float | None = 0.2,
- sim_batch: int = 16,
- tree_reuse: bool = True,
- move_temp: float = 0.0,
- move_temp_plies: int = 0,
- **kwargs,
- ):
- super().__init__(*args, **kwargs)
- self.num_simulations = num_simulations
- self.c_puct = c_puct
- # Move selection. By default the move is argmax(visit count) — fully
- # deterministic (what the gauntlet / API rely on). For variety (e.g. so
- # the opening isn't identical every game), set move_temp > 0 to *sample*
- # the move ∝ visit_count**(1/move_temp) for the first move_temp_plies
- # half-moves, then fall back to argmax. move_temp=1 samples proportional
- # to visits; higher is more random.
- self.move_temp = move_temp
- self.move_temp_plies = move_temp_plies
- # Tree reuse: keep the searched subtree under the moves actually played
- # and re-root it next move, so prior visits carry over (effectively
- # deeper search for free). Requires the caller to keep board.move_stack
- # across calls; falls back to a fresh tree otherwise (e.g. boards rebuilt
- # from FEN with no history).
- self.tree_reuse = tree_reuse
- self._reuse_root: _Node | None = None
- self._reuse_moves: list | None = None
- # prior_temp > 1 flattens the policy priors so good-but-low-prior moves
- # still get explored (the value head can then promote them). fpu (first-
- # play-urgency): value unvisited children as parent_Q - fpu; None keeps
- # the legacy behavior (unvisited contribute 0 to selection).
- self.prior_temp = prior_temp
- self.fpu = fpu
- # Leaves collected per network call. >1 amortizes the GPU->CPU sync that
- # otherwise dominates (one stall per simulation). 1 = legacy single-leaf.
- # Capped at the largest batch bucket so _batch_run's preallocated buffers
- # (and the captured CUDA graphs) are never overrun.
- self.sim_batch = max(1, min(sim_batch, _BATCH_BUCKETS[-1]))
-
- # ── expansion ────────────────────────────────────────────────────────
- def _compute_priors(self, moves: list[chess.Move], logits: np.ndarray) -> np.ndarray:
- """Softmax policy priors over legal moves, with temperature flattening."""
- scores = np.array(
- [
- logits[m.from_square, move_to_action_plane(m.from_square, m.to_square, m.promotion)]
- for m in moves
- ],
- dtype=np.float64,
- )
- scores /= self.prior_temp
- scores -= scores.max()
- exp = np.exp(scores)
- return exp / exp.sum()
-
- def _expand_node(self, node: _Node, moves: list[chess.Move], priors: np.ndarray):
- self.nodes_searched += 1
- n = len(moves)
- node.moves = moves
- node.priors = priors
- node.child_N = np.zeros(n, dtype=np.int64)
- node.child_W = np.zeros(n, dtype=np.float64)
- node.children = [None] * n
-
- def _expand_leaf(self, node: _Node, board: chess.Board) -> float:
- """Evaluate ``board`` (policy + value) and expand ``node``; return value."""
- logits, value = self._evaluate(board)
- moves = list(board.legal_moves)
- self._expand_node(node, moves, self._compute_priors(moves, logits))
- return value
-
- # ── selection / descent / backup ─────────────────────────────────────
- def _select_idx(self, node: _Node) -> int:
- """Vectorized PUCT: pick the child index maximizing -Q + U."""
- N = node.child_N
- total = int(N.sum())
- sqrt_total = math.sqrt(1.0 + total) # +1 ≈ the node's own eval visit
- # value-to-parent = -child_Q (child stores value from its own POV).
- q_parent = np.zeros_like(node.child_W)
- np.divide(-node.child_W, N, out=q_parent, where=N > 0)
- if self.fpu is not None:
- parent_q = (-node.child_W.sum() / total) if total > 0 else 0.0
- q_parent[N == 0] = parent_q - self.fpu
- u = (self.c_puct * sqrt_total) * node.priors / (1.0 + N)
- return int(np.argmax(q_parent + u))
-
- def _descend(self, node: _Node, board: chess.Board, apply_vl: bool):
- """Walk to an unexpanded leaf, pushing moves. Returns (leaf_node, path),
- where path is the list of (node, child_idx) edges traversed."""
- path: list[tuple[_Node, int]] = []
- while node.expanded:
- idx = self._select_idx(node)
- if apply_vl:
- # Virtual loss: a virtual *win* in the child's POV looks like a
- # loss to this node under -Q selection, so concurrent descents in
- # the batch avoid the path.
- node.child_N[idx] += 1
- node.child_W[idx] += 1.0
- path.append((node, idx))
- board.push(node.moves[idx])
- child = node.children[idx]
- if child is None:
- child = _Node()
- node.children[idx] = child
- node = child
- return node, path
-
- @staticmethod
- def _backup(path: list[tuple[_Node, int]], value: float):
- """Negamax backup over edges (no virtual loss)."""
- v = value
- for node, idx in reversed(path):
- node.child_N[idx] += 1
- node.child_W[idx] += v
- v = -v
-
- @staticmethod
- def _backup_vl(path: list[tuple[_Node, int]], value: float):
- """Backup that undoes the per-edge virtual loss (+1 added during descent)
- and applies the real, sign-alternating value. N already counted."""
- v = value
- for node, idx in reversed(path):
- node.child_W[idx] += v - 1.0
- v = -v
-
- # ── search drivers ───────────────────────────────────────────────────
- def _simulate(self, root: _Node, board: chess.Board):
- node, path = self._descend(root, board, apply_vl=False)
- pushed = len(path)
- term = self._terminal_value(board)
- value = term if term is not None else self._expand_leaf(node, board)
- self._backup(path, value)
- for _ in range(pushed):
- board.pop()
-
- def _run_batch(self, root: _Node, board: chess.Board, batch_n: int):
- """Collect up to ``batch_n`` leaves (virtual loss during descent),
- evaluate the unique uncached non-terminal ones in one batched forward,
- then expand and back them up."""
- leaves = [] # (path, leaf_node, key, moves, cached eval or None)
- items_by_key = {} # key -> (pos, player, castling, ep, key)
-
- for _ in range(batch_n):
- node, path = self._descend(root, board, apply_vl=True)
- term = self._terminal_value(board)
- if term is not None:
- self._backup_vl(path, term)
- else:
- key = chess.polyglot.zobrist_hash(board)
- moves = list(board.legal_moves)
- # Reuse cached policy+value (transpositions / reused subtree /
- # prior moves); only batch positions we haven't evaluated yet.
- # Capture the cached entry *now*: once the TT is at capacity,
- # _batch_run's inserts evict oldest entries, so a hit found here
- # may no longer be in the TT after the batch runs.
- cached = self._tt.get(key)
- if cached is not None:
- self.tt_hits += 1
- elif key not in items_by_key:
- pos, player, castling, ep = self._encode_position(board)
- items_by_key[key] = (pos, player, castling, ep, key)
- leaves.append((path, node, key, moves, cached))
- for _ in range(len(path)):
- board.pop()
-
- if items_by_key:
- self._batch_run(list(items_by_key.values())) # populates self._tt
-
- for path, node, key, moves, cached in leaves:
- logits, value = cached if cached is not None else self._tt[key]
- if not node.expanded:
- self._expand_node(node, moves, self._compute_priors(moves, logits))
- self._backup_vl(path, value)
-
- # ── tree reuse ───────────────────────────────────────────────────────
- def _take_reuse_root(self, board: chess.Board) -> _Node | None:
- """Re-root the retained tree to ``board`` if it follows the previous
- search position by a few plies. Consumes the stored tree."""
- root = self._reuse_root
- self._reuse_root = None
- if root is None or self._reuse_moves is None:
- return None
- ms = board.move_stack
- rp = len(self._reuse_moves)
- gap = len(ms) - rp
- if gap < 1 or gap > 4 or list(ms[:rp]) != self._reuse_moves:
- return None
- node = root
- for mv in ms[rp:]:
- if not node.expanded:
- return None
- try:
- idx = node.moves.index(mv)
- except ValueError:
- return None
- child = node.children[idx]
- if child is None or not child.expanded:
- return None
- node = child
- return node if node.expanded else None
-
- def predict(self, board: chess.Board) -> tuple[str, float | None]:
- self.nodes_searched = 0
- self.tt_hits = 0
- self.completed_depth = self.num_simulations
-
- if not any(board.legal_moves):
- return chess.Move.null().uci(), 0.0
-
- root = self._take_reuse_root(board) if self.tree_reuse else None
- if root is None:
- root = _Node()
- self._expand_leaf(root, board)
-
- if self.sim_batch <= 1:
- for _ in range(self.num_simulations):
- self._simulate(root, board)
- else:
- done = 0
- while done < self.num_simulations:
- n = min(self.sim_batch, self.num_simulations - done)
- self._run_batch(root, board, n)
- done += n
-
- ply = 2 * (board.fullmove_number - 1) + (0 if board.turn == chess.WHITE else 1)
- if self.move_temp > 0 and ply < self.move_temp_plies and root.child_N.sum() > 0:
- # Sample ∝ visit_count**(1/temp) over the opening plies for variety.
- weights = root.child_N.astype(np.float64) ** (1.0 / self.move_temp)
- total = weights.sum()
- best_idx = int(np.random.choice(len(weights), p=weights / total)) if total > 0 \
- else int(np.argmax(root.child_N))
- else:
- best_idx = int(np.argmax(root.child_N))
- best_move = root.moves[best_idx]
- n = int(root.child_N[best_idx])
- # Root value from side-to-move POV (negate child's POV).
- best_value = float(-root.child_W[best_idx] / n) if n > 0 else 0.0
-
- if self.tree_reuse:
- self._reuse_root = root
- self._reuse_moves = list(board.move_stack)
-
- return best_move.uci(), best_value
-
-
-if __name__ == "__main__":
- import argparse
- import time
-
- parser = argparse.ArgumentParser()
- parser.add_argument("model_dir", nargs="?", default=str(_DEFAULT_RUN))
- parser.add_argument("--sims", type=int, default=800)
- parser.add_argument("--c-puct", type=float, default=1.0)
- parser.add_argument("--ema", action="store_true")
- args = parser.parse_args()
-
- bot = Pos2MoveV2MctsBot(
- model_dir=args.model_dir, num_simulations=args.sims, c_puct=args.c_puct,
- time_limit=0.0, use_ema=args.ema,
- )
- board = chess.Board()
- for _ in range(8):
- t0 = time.time()
- mv, val = bot.predict(board)
- print(f"{'W' if board.turn else 'B'} {mv} val={val:+.3f} "
- f"sims={bot.num_simulations} nodes={bot.nodes_searched} time={time.time()-t0:.2f}s")
- board.push_uci(mv)
- if board.is_game_over():
- break
diff --git a/chesstransformer/bots/position2move_bot.py b/chesstransformer/bots/position2move_bot.py
deleted file mode 100644
index ebd8b29efdc5e7b94fd342b55cbdec11a963551f..0000000000000000000000000000000000000000
--- a/chesstransformer/bots/position2move_bot.py
+++ /dev/null
@@ -1,88 +0,0 @@
-from pathlib import Path
-import json
-import torch
-import chess
-from safetensors import safe_open
-from chesstransformer.models.transformer.position2move import Position2MoveModel
-from chesstransformer.models.tokenizer import PostionTokenizer, MoveTokenizer
-
-data_foler = Path(__file__).parents[1] / "data"
-
-
-class Position2MoveBot:
- def __init__(
- self,
- model_path: str = str((data_foler / "models/position2moveV2.1/best_model/model.safetensors").resolve()),
- device: str = "cpu",
- ):
- self.device = device
- self.position_tokenizer = PostionTokenizer()
- self.move_tokenizer = MoveTokenizer()
-
- config_path = Path(model_path).parent / "config.json"
-
- with open(config_path, "r") as f:
- self.config = json.load(f)
-
- # Load model
- self.model = Position2MoveModel(**self.config).to(device)
-
- with safe_open(model_path, framework="pt", device=device) as f:
- state_dict = {k: f.get_tensor(k) for k in f.keys()}
-
- # Handle compiled model prefix (_orig_mod.)
- if any(k.startswith("_orig_mod.") for k in state_dict.keys()):
- state_dict = {k.replace("_orig_mod.", ""): v for k, v in state_dict.items()}
-
- self.model.load_state_dict(state_dict)
-
- self.model.eval()
-
- @torch.no_grad()
- def predict(self, board: chess.Board):
- tokens_ids = self.position_tokenizer.encode(board)
- torch_input = torch.tensor(tokens_ids).unsqueeze(0).long().to(self.device) # Add batch dimension
-
- is_white = board.turn
- is_white = torch.tensor([is_white]).bool().to(self.device) # Add batch dimension
-
- logits = self.model(torch_input, is_white)
-
- legal_moves = [m.uci() for m in board.legal_moves]
-
- mask = torch.full((logits.size(-1),), float("-inf")).to(self.device)
- for move in legal_moves:
- move_id = self.move_tokenizer.encode(move)
- mask[move_id] = 0.0
-
- masked_logits = logits[0] + mask # Assuming batch size of 1
- probs = torch.softmax(masked_logits, dim=-1)
- # draw the move from the distribution
- predicted_move_id = torch.multinomial(probs * 0.2, num_samples=1).item()
- proba = probs[predicted_move_id].item()
-
- predicted_move = self.move_tokenizer.decode(predicted_move_id)
- return predicted_move, proba
-
-
-if __name__ == "__main__":
- bot = Position2MoveBot()
-
- board = chess.Board()
-
- print("Initial board:")
- print(board)
-
- move, proba = bot.predict(board)
- print(f"Bot suggests move: {move} with probability {proba:.4f}")
-
- board.push_uci(move)
- print("Board after bot move:")
- print(board)
-
- move, proba = bot.predict(board)
- print(f"Bot suggests move: {move} with probability {proba:.4f}")
-
- board.push_uci(move)
- print("Board after bot move:")
- print(board)
diff --git a/chesstransformer/bots/random_bot.py b/chesstransformer/bots/random_bot.py
deleted file mode 100644
index 4075696d23cdbfcfbc6b512051c6a06985f0a498..0000000000000000000000000000000000000000
--- a/chesstransformer/bots/random_bot.py
+++ /dev/null
@@ -1,33 +0,0 @@
-import chess
-import random
-
-
-class RandomBot:
- def __init__(self):
- pass
-
- def predict(self, board: chess.Board) -> chess.Move:
- legal_moves = list(board.legal_moves)
- if not legal_moves:
- raise ValueError("No legal moves available.")
- # pick a random move
- move = random.choice(legal_moves)
- return move.uci(), None
-
-
-if __name__ == "__main__":
- bot = RandomBot()
- board = chess.Board()
- print("Initial board:")
- print(board)
- move = bot.predict(board)
- print(f"RandomBot suggests move: {move}")
- board.push(move)
- print("Board after move:")
- print(board)
-
- move = bot.predict(board)
- print(f"RandomBot suggests move: {move}")
- board.push(move)
- print("Board after move:")
- print(board)
diff --git a/chesstransformer/datasets/__init__.py b/chesstransformer/datasets/__init__.py
deleted file mode 100644
index 6b1c05cc18b81d6d340f8199987293ed40c12b2f..0000000000000000000000000000000000000000
--- a/chesstransformer/datasets/__init__.py
+++ /dev/null
@@ -1,10 +0,0 @@
-from .lichess import LichessDataset
-from .lichess_simple_uci import LichessSimpleUciDataset
-from .puzzle_dataset import LichessPuzzleDataset, LichessPuzzleFullSolutionDataset
-
-__all__ = [
- "LichessDataset",
- "LichessSimpleUciDataset",
- "LichessPuzzleDataset",
- "LichessPuzzleFullSolutionDataset",
-]
diff --git a/chesstransformer/datasets/dataset_h5_convertor.py b/chesstransformer/datasets/dataset_h5_convertor.py
deleted file mode 100644
index 36759a65bbe91aa5e26bf7fb4ad6c4964a2f0adb..0000000000000000000000000000000000000000
--- a/chesstransformer/datasets/dataset_h5_convertor.py
+++ /dev/null
@@ -1,404 +0,0 @@
-import h5py
-import chess.pgn
-import zstandard as zstd
-import numpy as np
-from tqdm import tqdm
-from multiprocessing import Pool, cpu_count
-from chesstransformer.models.tokenizer.move_tokenizer import MoveTokenizer
-import io
-from pathlib import Path
-
-# Per-worker globals initialised once per process
-_worker_tokenizer = None
-_worker_exclude_tcs = []
-
-
-def _worker_init(exclude_tcs):
- global _worker_tokenizer, _worker_exclude_tcs
- _worker_tokenizer = MoveTokenizer()
- _worker_exclude_tcs = exclude_tcs
-
-
-def _process_raw_game(raw_pgn: str):
- """Full pipeline in each worker: text-filter → parse → tokenize.
-
- Moving parse into workers parallelises the most expensive step.
- The fast text-level filter avoids even calling chess.pgn.read_game
- for excluded time controls.
- """
- global _worker_tokenizer, _worker_exclude_tcs
-
- # Fast text-level filter before expensive PGN parsing
- if _worker_exclude_tcs:
- idx = raw_pgn.find('[Event "')
- if idx >= 0:
- end = raw_pgn.find('"]', idx + 8)
- if end >= 0:
- event_lower = raw_pgn[idx + 8 : end].lower()
- if any(tc in event_lower for tc in _worker_exclude_tcs):
- return None
-
- game = chess.pgn.read_game(io.StringIO(raw_pgn))
- if game is None:
- return None
-
- if game.headers.get("Variant", "Standard") != "Standard":
- return None
-
- result_str = game.headers.get("Result", "*")
- if result_str == "1/2-1/2":
- result = 0
- elif result_str == "1-0":
- result = 1
- elif result_str == "0-1":
- result = 2
- else:
- return None
-
- try:
- white_elo = int(game.headers.get("WhiteElo", "1500"))
- except ValueError:
- white_elo = 1500
- try:
- black_elo = int(game.headers.get("BlackElo", "1500"))
- except ValueError:
- black_elo = 1500
-
- tokens = []
- for move in game.mainline_moves():
- try:
- tokens.append(_worker_tokenizer.encode(move.uci()))
- except ValueError:
- return None
-
- if len(tokens) < 10:
- return None
-
- return np.array(tokens, dtype=np.int16), white_elo, black_elo, result, len(tokens)
-
-
-def _iter_raw_games(stream):
- """Yield individual PGN game strings from a text stream.
-
- Splits on '[Event' lines, which always start each game in standard PGN.
- Line-by-line reading keeps memory usage constant regardless of file size.
- """
- buf = []
- for line in stream:
- if line.startswith("[Event ") and buf:
- yield "".join(buf)
- buf = [line]
- else:
- buf.append(line)
- if buf:
- text = "".join(buf).strip()
- if text:
- yield text
-
-
-class PGNtoHDF5Converter:
- def __init__(
- self,
- output_path: str,
- chunk_size: int = 10000,
- num_workers: int = None,
- exclude_time_controls: list = None,
- ):
- """
- Convert PGN files to HDF5 format storing complete games as UCI move sequences.
-
- Args:
- output_path: Path to save the HDF5 file
- chunk_size: Number of games to accumulate before writing (default: 10000)
- exclude_time_controls: List of time control categories to skip, matched against
- the Lichess Event header (e.g. ["Bullet", "Blitz", "UltraBullet"])
- """
- self.output_path = output_path
- self.chunk_size = chunk_size
- default_workers = (cpu_count() or 1) - 1
- self.num_workers = max(1, num_workers if num_workers is not None else max(1, default_workers))
- self.exclude_time_controls = [tc.lower() for tc in (exclude_time_controls or [])]
-
- def convert(self, input_path: str, max_games: int = None):
- """Convert PGN files to HDF5. Input can be a file or folder."""
- path = Path(input_path)
- if not path.exists():
- raise FileNotFoundError(f"Path not found: {input_path}")
- if path.is_file():
- self._convert_single_file(path, max_games)
- elif path.is_dir():
- self._convert_folder(path, max_games)
- else:
- raise ValueError(f"Invalid input path: {input_path}")
-
- def _convert_folder(self, folder_path: Path, max_games: int = None):
- pgn_files = sorted(list(folder_path.glob("*.pgn")) + list(folder_path.glob("*.pgn.zst")))
- if not pgn_files:
- raise ValueError(f"No PGN files found in {folder_path}")
-
- done = self._completed_files()
- pending = [f for f in pgn_files if f.name not in done]
-
- print(f"Found {len(pgn_files)} PGN file(s) in {folder_path}")
- if done:
- print(f"Resuming: {len(done)} already converted, {len(pending)} remaining")
- for f in pending:
- print(f" - {f.name}")
- print()
-
- if not pending:
- print("Nothing to do — every file is already in the output.")
- return 0
-
- total_games = 0
- started = bool(done)
- for pgn_file in pending:
- remaining = None if max_games is None else max(0, max_games - total_games)
- if remaining == 0:
- break
- print(f"\nProcessing: {pgn_file.name}")
- total_games += self._convert_single_file(pgn_file, remaining, append=started)
- started = True
- if max_games and total_games >= max_games:
- print(f"\nReached maximum game limit ({max_games})")
- break
-
- print(f"\n{'=' * 60}")
- print(f"All files processed! Total games: {total_games}")
- print(f"Output: {self.output_path}")
-
- def _convert_single_file(self, pgn_path: Path, max_games: int = None, append: bool = False):
- if pgn_path.suffix == ".zst" or pgn_path.name.endswith(".pgn.zst"):
- return self._convert_compressed_file(str(pgn_path), max_games, append)
- else:
- return self._convert_uncompressed_file(str(pgn_path), max_games, append)
-
- def convert_pgn_zst(self, pgn_path: str, max_games: int = None):
- """Deprecated: Use convert() instead."""
- return self._convert_compressed_file(pgn_path, max_games, append=False)
-
- # ------------------------------------------------------------------
- # Internal helpers
- # ------------------------------------------------------------------
-
- _DATASET_NAMES = ("moves", "white_elo", "black_elo", "result", "num_moves")
-
- def _completed_files(self) -> list[str]:
- """Names of PGN files already fully written into the output, for resume."""
- path = Path(self.output_path)
- if not path.exists():
- return []
- try:
- with h5py.File(path, "r") as f:
- return [str(n) for n in f.attrs.get("converted_files", [])]
- except OSError:
- return []
-
- def _mark_converted(self, hdf5_file, name: str):
- """Commit a finished PGN file: record its name and the row watermark."""
- done = [str(n) for n in hdf5_file.attrs.get("converted_files", [])]
- if name not in done:
- done.append(name)
- hdf5_file.attrs["converted_files"] = np.array(done, dtype=h5py.string_dtype())
- hdf5_file.attrs["committed_rows"] = int(min(hdf5_file[n].shape[0] for n in self._DATASET_NAMES))
-
- def _rewind_to_watermark(self, hdf5_file):
- """Discard everything written after the last fully converted PGN file.
-
- Length agreement is not enough to detect an interrupted run: `_write_batch`
- resizes all five datasets before writing any of them, so a crash mid-batch
- leaves them equal-length with a junk tail. Worse, the rows a dead run *did*
- write would be duplicated when its PGN file — never added to `converted_files`
- — gets reconverted. Rewinding to the committed watermark drops both.
- """
- lengths = {n: hdf5_file[n].shape[0] for n in self._DATASET_NAMES}
- mark = int(hdf5_file.attrs.get("committed_rows", -1))
- if mark < 0: # written before watermarks existed; fall back to length agreement
- mark = min(lengths.values())
- if any(length != mark for length in lengths.values()):
- print(f"Rewinding interrupted output: {lengths} -> {mark} rows")
- for name in self._DATASET_NAMES:
- hdf5_file[name].resize((mark,))
-
- def _create_datasets(self, hdf5_file):
- """Create all HDF5 datasets (called only when not appending)."""
- dt = h5py.vlen_dtype(np.dtype("int16"))
- for name, dtype in [
- ("moves", dt),
- ("white_elo", "int16"),
- ("black_elo", "int16"),
- ("result", "int8"),
- ("num_moves", "int16"),
- ]:
- hdf5_file.create_dataset(
- name,
- shape=(0,),
- maxshape=(None,),
- dtype=dtype,
- compression="gzip",
- compression_opts=4,
- chunks=(self.chunk_size,),
- )
-
- def _write_batch(self, hdf5_file, games, white_elos, black_elos, results, num_moves):
- old = hdf5_file["moves"].shape[0]
- new = old + len(games)
- for ds in ("moves", "white_elo", "black_elo", "result", "num_moves"):
- hdf5_file[ds].resize((new,))
- # One slice assignment, not one write per game: each per-game write re-read and
- # re-compressed the whole gzip chunk, costing ~1 ms/game in the main process and
- # starving the workers.
- moves_arr = np.empty(len(games), dtype=object)
- moves_arr[:] = games
- try:
- hdf5_file["moves"][old:new] = moves_arr
- except TypeError:
- # h5py re-normalises the object array internally, and when every game in the
- # batch happens to have the same length (always so for a single game) that
- # collapses it to 2-D and the vlen write is refused. Only degenerate batches
- # hit this — typically the 1-game remainder of a file's final flush — so the
- # slow path costs nothing in practice.
- for i, g in enumerate(games):
- hdf5_file["moves"][old + i] = g
- hdf5_file["white_elo"][old:new] = np.array(white_elos, dtype=np.int16)
- hdf5_file["black_elo"][old:new] = np.array(black_elos, dtype=np.int16)
- hdf5_file["result"][old:new] = np.array(results, dtype=np.int8)
- hdf5_file["num_moves"][old:new] = np.array(num_moves, dtype=np.int16)
-
- def _run_conversion(self, stream, hdf5_file, max_games) -> tuple[int, int]:
- """Core loop: stream raw PGN text → workers parse+tokenize → write HDF5.
-
- Returns (game_count, total_positions).
- """
- game_count = 0
- total_positions = 0
- games_buf, we_buf, be_buf, res_buf, nm_buf = [], [], [], [], []
-
- pbar = tqdm(desc="Processing games", unit="games")
-
- def flush():
- if games_buf:
- self._write_batch(hdf5_file, games_buf, we_buf, be_buf, res_buf, nm_buf)
- games_buf.clear(); we_buf.clear(); be_buf.clear()
- res_buf.clear(); nm_buf.clear()
-
- def accumulate(result):
- nonlocal game_count, total_positions
- token_array, we, be, res, nm = result
- games_buf.append(token_array)
- we_buf.append(we); be_buf.append(be)
- res_buf.append(res); nm_buf.append(nm)
- total_positions += nm
- game_count += 1
- pbar.update(1)
- if len(games_buf) >= self.chunk_size:
- flush()
-
- use_parallel = self.num_workers > 1
- raw_iter = _iter_raw_games(stream)
-
- if use_parallel:
- pool = Pool(
- processes=self.num_workers,
- initializer=_worker_init,
- initargs=(self.exclude_time_controls,),
- )
- early_stop = False
- try:
- # imap_unordered streams results as soon as each worker finishes —
- # no need to wait for a full chunk_size batch before seeing progress.
- for result in pool.imap_unordered(_process_raw_game, raw_iter, chunksize=200):
- if result is None:
- continue
- accumulate(result)
- if max_games and game_count >= max_games:
- early_stop = True
- break
- finally:
- if early_stop:
- pool.terminate()
- else:
- pool.close()
- pool.join()
- else:
- # Sequential fallback (single worker)
- _worker_init(self.exclude_time_controls)
- for raw_pgn in raw_iter:
- result = _process_raw_game(raw_pgn)
- if result is None:
- continue
- accumulate(result)
- if max_games and game_count >= max_games:
- break
-
- flush()
- pbar.close()
- return game_count, total_positions
-
- def _convert_compressed_file(self, pgn_path: str, max_games: int = None, append: bool = False):
- with open(pgn_path, "rb") as f:
- dctx = zstd.ZstdDecompressor()
- with dctx.stream_reader(f) as reader:
- stream = io.TextIOWrapper(reader, encoding="utf-8")
- mode = "a" if append else "w"
- with h5py.File(self.output_path, mode) as hdf5_file:
- if append and "moves" in hdf5_file:
- self._rewind_to_watermark(hdf5_file)
- else:
- self._create_datasets(hdf5_file)
- game_count, total_positions = self._run_conversion(stream, hdf5_file, max_games)
- if not max_games or game_count < max_games:
- self._mark_converted(hdf5_file, Path(pgn_path).name)
-
- self._print_summary(game_count, total_positions)
- return game_count
-
- def _convert_uncompressed_file(self, pgn_path: str, max_games: int = None, append: bool = False):
- with open(pgn_path, "r", encoding="utf-8") as pgn_file:
- mode = "a" if append else "w"
- with h5py.File(self.output_path, mode) as hdf5_file:
- if append and "moves" in hdf5_file:
- self._rewind_to_watermark(hdf5_file)
- else:
- self._create_datasets(hdf5_file)
- game_count, total_positions = self._run_conversion(pgn_file, hdf5_file, max_games)
- if not max_games or game_count < max_games:
- self._mark_converted(hdf5_file, Path(pgn_path).name)
-
- self._print_summary(game_count, total_positions)
- return game_count
-
- def _print_summary(self, game_count, total_positions):
- print("File conversion complete!")
- print(f"Games from this file: {game_count}")
- print(f"Positions from this file: {total_positions}")
- avg = (total_positions / game_count) if game_count else 0.0
- print(f"Average moves per game: {avg:.1f}")
-
-
-# Usage script
-if __name__ == "__main__":
- import argparse
-
- parser = argparse.ArgumentParser(description="Convert PGN/PGN.zst file(s) or folder to HDF5")
- parser.add_argument("--input", type=str, required=True, help="Input .pgn/.pgn.zst file or folder")
- parser.add_argument("--output", type=str, required=True, help="Output .h5 file")
- parser.add_argument("--max-games", type=int, default=None, help="Maximum games to process")
- parser.add_argument("--chunk-size", type=int, default=10000, help="Batch size for writing")
- parser.add_argument("--num-workers", type=int, default=None, help="Worker processes (default: cpu count - 1)")
- parser.add_argument(
- "--exclude-time-controls",
- nargs="+",
- default=[],
- metavar="TC",
- help="Time control categories to skip (e.g. Bullet Blitz UltraBullet)",
- )
-
- args = parser.parse_args()
- converter = PGNtoHDF5Converter(
- args.output,
- chunk_size=args.chunk_size,
- num_workers=args.num_workers,
- exclude_time_controls=args.exclude_time_controls,
- )
- converter.convert(args.input, max_games=args.max_games)
diff --git a/chesstransformer/datasets/flat_shard_dataset.py b/chesstransformer/datasets/flat_shard_dataset.py
deleted file mode 100644
index be373fe73889af294018cb1b2fc6cd14c4a3d66d..0000000000000000000000000000000000000000
--- a/chesstransformer/datasets/flat_shard_dataset.py
+++ /dev/null
@@ -1,231 +0,0 @@
-"""Memmap-backed dataset over the flat shards written by ``scripts/build_shards.py``.
-
-Implements ``doc/dataset_shards.md`` §4. Every expensive thing ``HDF5ChessDataset`` does
-per sample — gzip decompression, python-chess replay, legal-move enumeration, tokenizing —
-has already been paid at build time. What is left here is a numpy gather out of page cache.
-
-Two decisions carry most of the speed:
-
-**Batch-level fetch.** At 202 bytes per sample the per-item Python call overhead dominates
-the actual work, so ``__getitem__`` accepts a *list* of indices and does one vectorised
-gather per batch. Drive it with :func:`make_dataloader`, which wires up a ``BatchSampler``
-and ``batch_size=None`` so the DataLoader hands the whole index list through untouched.
-
-**The legal mask is not built on CPU.** Returning ``(64, 73)`` bools would put 2.4 MB per
-batch of 512 back on the H2D wire — the exact cost the shard format exists to remove. The
-loader ships the ``(B, 64)`` uint16 index list (64 KB) and the training loop calls
-:func:`scatter_legal_planes` on device.
-
-**Distributed use.** This is a map-style dataset, so rank splitting is
-``DistributedSampler``'s job, not the dataset's — see :func:`make_dataloader`. Map-style +
-``DistributedSampler`` is preferred over an ``IterableDataset`` that owns shard *k* when
-``k % (world_size * num_workers) == rank * num_workers + worker`` because the latter can
-only shuffle within a worker's own shards: sample correlation then survives into the batch,
-and the assignment silently unbalances whenever ``world_size * num_workers`` does not
-divide the shard count. The shards here are a build-parallelism and train/val/test-split
-unit; they deliberately do not constrain who reads what.
-"""
-
-import json
-import os
-from pathlib import Path
-
-import numpy as np
-import torch
-from torch.utils.data import BatchSampler, DataLoader, RandomSampler, SequentialSampler
-
-from chesstransformer.datasets.shard_format import (
- FORMAT_VERSION,
- NUM_ACTION_PLANES,
- NUM_SQUARES,
- PAD_ACTION,
- dtype_from_meta,
-)
-
-#: Fields handed to the trainer, in the dtype the model wants them in. Kept narrow on the
-#: wire (uint8/int16) and widened on device — an int64 ``position`` would be 8x the copy
-#: for an embedding lookup that accepts int32 anyway.
-_SCALAR_FIELDS = ("from_square", "action_plane", "castling_rights", "en_passant_file", "is_white")
-
-
-class FlatShardDataset(torch.utils.data.Dataset):
- """One split of a shard directory, as one addressable array of samples."""
-
- def __init__(self, shard_dir: str | Path, split: str = "train"):
- self.shard_dir = Path(shard_dir)
- self.split = split
-
- with (self.shard_dir / "meta.json").open() as fh:
- self.meta = json.load(fh)
-
- if self.meta["format_version"] != FORMAT_VERSION:
- raise ValueError(
- f"{self.shard_dir} was built with format_version "
- f"{self.meta['format_version']}, this code reads {FORMAT_VERSION}. Rebuild."
- )
-
- if split not in self.meta["splits"]:
- raise KeyError(f"unknown split {split!r}; have {list(self.meta['splits'])}")
-
- self.dtype = dtype_from_meta(self.meta["record_dtype"])
- self.pad_action = int(self.meta["pad_action"])
-
- wanted = set(self.meta["splits"][split])
- self.shard_meta = [m for m in self.meta["shards"] if m["shard_id"] in wanted]
- if not self.shard_meta:
- raise ValueError(f"split {split!r} of {self.shard_dir} is empty")
-
- # Cumulative offsets: a global index resolves to (shard, row) with one searchsorted.
- counts = np.array([m["num_samples"] for m in self.shard_meta], dtype=np.int64)
- self.offsets = np.concatenate([[0], np.cumsum(counts)])
- self._len = int(self.offsets[-1])
-
- # Opened lazily, per process. np.memmap survives fork harmlessly, but building the
- # handles in __init__ would map every shard in the parent for no reason.
- self._shards = None
- self._shards_pid = None
-
- # The whole point of memmap is that the page cache holds one copy that all workers
- # share. np.load(...) without mmap_mode would give each of N workers its own.
- self.game_ranges = [(m["game_start"], m["game_end"]) for m in self.shard_meta]
-
- def __len__(self):
- return self._len
-
- @property
- def shards(self):
- pid = os.getpid()
- if self._shards is None or self._shards_pid != pid:
- self._shards = [
- np.load(self.shard_dir / m["path"], mmap_mode="r") for m in self.shard_meta
- ]
- self._shards_pid = pid
- return self._shards
-
- def __getstate__(self):
- state = self.__dict__.copy()
- state["_shards"] = None
- state["_shards_pid"] = None
- return state
-
- def _gather(self, idx: np.ndarray) -> np.ndarray:
- """Fetch records for a batch of global indices, preserving their order."""
- shard_of = np.searchsorted(self.offsets, idx, side="right") - 1
- rows = idx - self.offsets[shard_of]
-
- shards = self.shards
- out = np.empty(len(idx), dtype=self.dtype)
- for sid in np.unique(shard_of):
- where = np.nonzero(shard_of == sid)[0]
- r = rows[where]
- # Read each shard in ascending row order: sequential page touches, even though
- # the batch itself is shuffled.
- order = np.argsort(r, kind="stable")
- out[where[order]] = shards[sid][r[order]]
- return out
-
- def __getitem__(self, index):
- """Accepts a single index or — the fast path — a list/array of them."""
- if isinstance(index, (int, np.integer)):
- batch = self._gather(np.array([index], dtype=np.int64))
- else:
- batch = self._gather(np.asarray(index, dtype=np.int64))
-
- # Everything stays at its on-disk width. A batch of 512 is then 103 KB rather than
- # the 5.81 MB the HDF5 path collates, and the widening to int64/bool happens on
- # device in the trainer's unpack_batch, where it is free.
- #
- # int16 rather than uint16 for the two u2 fields because torch has no uint16; both
- # are safely in range (legal_idx <= 4672, move_number max observed 601).
- out = {
- "position": torch.from_numpy(np.ascontiguousarray(batch["position"])),
- "legal_idx": torch.from_numpy(np.ascontiguousarray(batch["legal_idx"]).astype(np.int16)),
- "legal_cnt": torch.from_numpy(np.ascontiguousarray(batch["legal_cnt"])),
- "result": torch.from_numpy(np.ascontiguousarray(batch["result"])),
- "move_number": torch.from_numpy(
- np.ascontiguousarray(batch["move_number"]).astype(np.int16)
- ),
- }
- for name in _SCALAR_FIELDS:
- out[name] = torch.from_numpy(np.ascontiguousarray(batch[name]))
- return out
-
- def game_id_set(self) -> set:
- """Game indices this split covers. Used by the split-disjointness assertion."""
- ids = set()
- for g0, g1 in self.game_ranges:
- ids.update(range(g0, g1))
- return ids
-
-
-def scatter_legal_planes(legal_idx: torch.Tensor, pad_action: int = PAD_ACTION) -> torch.Tensor:
- """Rebuild the ``(B, 64, 73)`` bool legal mask from the packed index list, on device.
-
- ``legal_idx`` is ``(B, C)`` with unused slots set to ``pad_action`` (== 64*73), one past
- the end of the action space. Scattering into a tensor with one extra column parks that
- padding in a scratch slot which is then sliced off — if padding were 0 instead, every
- sample would come back with square a1 / plane 0 marked legal.
- """
- b = legal_idx.size(0)
- flat = torch.zeros(b, pad_action + 1, dtype=torch.bool, device=legal_idx.device)
- flat.scatter_(1, legal_idx.long(), True)
- return flat[:, :pad_action].view(b, NUM_SQUARES, NUM_ACTION_PLANES)
-
-
-def worker_init_fn(worker_id: int):
- """Give every worker (and every rank) its own numpy RNG stream.
-
- PyTorch reseeds ``random`` and ``torch`` per worker but *not* ``numpy``, so all workers
- inherit the parent's numpy state and draw the identical sequence — and under DDP so
- would every rank. Nothing in this dataset's hot path uses ``np.random`` any more, but
- ``HDF5ChessDataset._sample_move_idx`` does, and the fix belongs wherever a worker starts.
- """
- rank = int(os.environ.get("RANK", 0))
- np.random.seed((torch.initial_seed() + 977 * rank) % (2**32))
-
-
-def make_dataloader(
- dataset: FlatShardDataset,
- batch_size: int,
- shuffle: bool,
- num_workers: int = 4,
- drop_last: bool = False,
- pin_memory: bool = True,
- prefetch_factor: int = 4,
- seed: int = 0,
- distributed: bool = False,
-) -> DataLoader:
- """Build the batch-fetch DataLoader this dataset is designed for.
-
- ``batch_size=None`` disables the DataLoader's own batching so the ``BatchSampler``'s
- index *list* reaches ``__getitem__`` intact and the gather happens once per batch
- rather than once per sample. Without this the dataset still works and runs roughly an
- order of magnitude slower.
-
- ``distributed=True`` wraps a ``DistributedSampler`` so each rank sees a disjoint
- 1/world_size of the split. Remember to call ``loader.sampler.sampler.set_epoch(epoch)``
- — the BatchSampler nests it one level down.
- """
- if distributed:
- from torch.utils.data import DistributedSampler
-
- inner = DistributedSampler(dataset, shuffle=shuffle, seed=seed, drop_last=drop_last)
- elif shuffle:
- generator = torch.Generator()
- generator.manual_seed(seed)
- inner = RandomSampler(dataset, generator=generator)
- else:
- inner = SequentialSampler(dataset)
-
- sampler = BatchSampler(inner, batch_size=batch_size, drop_last=drop_last)
-
- return DataLoader(
- dataset,
- batch_size=None,
- sampler=sampler,
- num_workers=num_workers,
- pin_memory=pin_memory,
- persistent_workers=num_workers > 0,
- prefetch_factor=prefetch_factor if num_workers > 0 else None,
- worker_init_fn=worker_init_fn,
- )
diff --git a/chesstransformer/datasets/h5_lichess_dataset.py b/chesstransformer/datasets/h5_lichess_dataset.py
deleted file mode 100644
index 817442c6d2431d307684e39058b2bc75acf0a0c1..0000000000000000000000000000000000000000
--- a/chesstransformer/datasets/h5_lichess_dataset.py
+++ /dev/null
@@ -1,295 +0,0 @@
-import os
-
-import h5py
-import torch
-from torch.utils.data import Dataset
-import numpy as np
-import chess
-from chesstransformer.models.tokenizer.position_tokenizer import PostionTokenizer
-from chesstransformer.models.tokenizer.move_tokenizer import MoveTokenizer
-from chesstransformer.models.tokenizer.alphazero_move_encoder import (
- move_to_action_plane,
- NUM_ACTION_PLANES,
-)
-
-
-class HDF5ChessDataset(Dataset):
- def __init__(
- self,
- hdf5_path: str,
- cache_size: int = 500,
- min_elo: int = None,
- max_elo: int = None,
- sample_weighting: str = "middlegame",
- skip_opening_plies: int = 0,
- rdcc_nbytes: int = 32 << 20,
- ):
- """
- Efficient dataset that loads games as UCI sequences and samples random positions.
-
- Each dataset index corresponds to one game. When a sample is requested, it:
- 1. Uses the index to select a game
- 2. Samples a position within that game (uniform or phase-weighted)
- 3. Replays the game to that position to get the board state
- 4. Returns the encoded board state + the next move
-
- Args:
- hdf5_path: Path to HDF5 file containing games
- cache_size: Number of games to cache in memory
- min_elo: Minimum average ELO for filtering games (optional)
- max_elo: Maximum average ELO for filtering games (optional)
- sample_weighting: "uniform" (legacy) or "middlegame" (triangular peak around move 25)
- skip_opening_plies: Force sampling to start at this ply (0 = no restriction)
- rdcc_nbytes: HDF5 chunk cache per dataset. ``moves`` is gzipped in chunks of
- 10,000 games, so a random read decompresses a whole chunk; h5py's 1 MB
- default cannot hold one and every read pays full decompression. 32 MB
- takes a random ``moves[g]`` from 0.22 ms to 0.12 ms.
- """
- self.hdf5_path = hdf5_path
- self.cache_size = cache_size
- self.game_cache = {} # Cache for game move sequences
- self.position_tokenizer = PostionTokenizer()
- self.move_tokenizer = MoveTokenizer()
- self.sample_weighting = sample_weighting
- self.skip_opening_plies = skip_opening_plies
- self.rdcc_nbytes = rdcc_nbytes
-
- # Long-lived read handle, opened lazily by _h5(). See the note there for why
- # this must never be opened in __init__.
- self._file = None
- self._file_pid = None
-
- # Load metadata and filter games by ELO
- with h5py.File(hdf5_path, "r") as f:
- self.num_games = f["moves"].shape[0]
- num_moves = f["num_moves"][:]
- white_elos = f["white_elo"][:]
- black_elos = f["black_elo"][:]
- results = f["result"][:]
-
- # Filter games by ELO if specified
- valid_games = np.ones(self.num_games, dtype=bool)
- if min_elo is not None or max_elo is not None:
- avg_elos = (white_elos + black_elos) / 2
- if min_elo is not None:
- valid_games &= avg_elos >= min_elo
- if max_elo is not None:
- valid_games &= avg_elos <= max_elo
-
- # Store valid game indices and their move counts
- self.valid_game_indices = np.where(valid_games)[0]
- self.num_moves_per_game = num_moves[valid_games]
-
- # Per-game metadata stays in RAM: 1.53M games is 1.5 MB of int8 plus
- # 3 MB of int16, and reading these three scalars back out of HDF5 per
- # sample cost more than the game decode itself. Kept *unfiltered*, so
- # they are indexed by the actual game index, not the dataset index.
- self.white_elos = white_elos
- self.black_elos = black_elos
- self.results = results
-
- total_positions = int(self.num_moves_per_game.sum())
- print(f"Loaded dataset: {len(self.valid_game_indices)} games with {total_positions} total positions")
- if min_elo or max_elo:
- print(f"Filtered by ELO: {min_elo or 0} - {max_elo or 'inf'}")
-
- def __len__(self):
- """Return the number of games in the dataset."""
- return len(self.valid_game_indices)
-
- def _sample_move_idx(self, max_move_idx: int) -> int:
- """Sample a position index within a game.
-
- - "uniform": legacy behavior, np.random.randint(0, max_move_idx)
- - "middlegame": triangular weight peaking around ply 25,
- downweights opening (rare unique positions) and very late
- endgame (sparse signal). Falls back to uniform for short games.
-
- Optionally enforces ``skip_opening_plies`` as a hard floor.
- """
- lo = min(self.skip_opening_plies, max_move_idx - 1) if max_move_idx > 1 else 0
- hi = max_move_idx # exclusive upper bound
-
- if self.sample_weighting == "uniform" or hi - lo <= 8:
- return int(np.random.randint(lo, hi))
-
- # Triangular weights: ramp up over plies 0->20, flat 20-40, taper 40->80
- plies = np.arange(lo, hi)
- ramp_up = np.minimum(plies / 20.0, 1.0)
- taper = np.minimum(np.maximum((80.0 - plies) / 40.0, 0.1), 1.0)
- weights = ramp_up * taper
- weights = weights / weights.sum()
- return int(np.random.choice(plies, p=weights))
-
- def _h5(self) -> h5py.File:
- """Return this *process's* HDF5 handle, opening it on first use.
-
- Deliberately not opened in ``__init__``. DataLoader workers are forked, and
- HDF5 is not fork-safe: a handle inherited across ``fork()`` is shared library
- state that two processes then mutate independently. It does not raise — it
- returns corrupted data or deadlocks, which is far worse. Opening on first
- ``__getitem__`` means every worker gets its own handle after the fork.
-
- The PID check covers the case where the parent process reads a sample (e.g.
- ``num_workers=0``, or a shape probe) before spawning workers, which would
- otherwise leave an open handle for the children to inherit.
- """
- pid = os.getpid()
- if self._file is None or self._file_pid != pid:
- self._file = h5py.File(
- self.hdf5_path,
- "r",
- rdcc_nbytes=self.rdcc_nbytes,
- rdcc_nslots=10007, # prime, comfortably > chunks held in the cache
- )
- self._file_pid = pid
- return self._file
-
- def __getstate__(self):
- """Drop the handle when pickled (``spawn`` start method); h5py.File is not
- picklable and each process must open its own anyway."""
- state = self.__dict__.copy()
- state["_file"] = None
- state["_file_pid"] = None
- return state
-
- def _get_game_moves(self, game_idx: int) -> np.ndarray:
- """Load and cache a game's move sequence.
-
- Only ``moves`` comes from HDF5; the per-game scalars live in RAM (see __init__).
- """
- if game_idx in self.game_cache:
- return self.game_cache[game_idx]
-
- moves = self._h5()["moves"][game_idx]
-
- # Update cache (simple LRU-like behavior)
- if len(self.game_cache) >= self.cache_size:
- # Remove oldest item
- self.game_cache.pop(next(iter(self.game_cache)))
-
- self.game_cache[game_idx] = moves
- return moves
-
- def __getitem__(self, idx):
- """
- Returns a position-move pair by:
- 1. Using idx to select a game
- 2. Uniformly sampling a random position within that game
- 3. Replaying the game to get the board state
- 4. Returning encoded board + next move
- """
- # Map dataset index to actual game index
- actual_game_idx = self.valid_game_indices[idx]
-
- # Load game moves; the scalars are RAM lookups (indexed by actual game index)
- game_moves = self._get_game_moves(actual_game_idx)
- white_elo = int(self.white_elos[actual_game_idx])
- black_elo = int(self.black_elos[actual_game_idx])
- result = int(self.results[actual_game_idx])
- num_moves = self.num_moves_per_game[idx]
-
- # Uniformly sample a position within the game (exclude last move - no next move)
- max_move_idx = num_moves - 1
- if max_move_idx <= 0:
- # Edge case: game with only one move, use position 0
- move_idx = 0
- else:
- move_idx = self._sample_move_idx(int(max_move_idx))
-
- # Replay game to position
- board = chess.Board()
- for i in range(move_idx):
- # Decode move token to UCI string
- move_token = int(game_moves[i])
- move_uci = self._decode_move_token(move_token)
- try:
- move = chess.Move.from_uci(move_uci)
- board.push(move)
- except (ValueError, AssertionError):
- # If move is invalid, skip to next position
- # This shouldn't happen with properly encoded data
- pass
-
- # Encode current position
- position = self.position_tokenizer.encode(board)
- position_tensor = torch.tensor(position, dtype=torch.long)
-
- legal_moves_planes = torch.zeros(64, NUM_ACTION_PLANES, dtype=torch.bool)
- for move in board.legal_moves:
- plane = move_to_action_plane(move.from_square, move.to_square, move.promotion)
- legal_moves_planes[move.from_square, plane] = True
-
- # Get next move
- next_move_token = torch.tensor(int(game_moves[move_idx]), dtype=torch.long)
-
- # Parse next move for grid format (from/to squares + promotion)
- next_move_uci = self._decode_move_token(int(game_moves[move_idx]))
- next_move = chess.Move.from_uci(next_move_uci)
- from_square = next_move.from_square
- to_square = next_move.to_square
-
- # AlphaZero action plane for this move
- action_plane = move_to_action_plane(from_square, to_square, next_move.promotion)
-
- # Get turn indicator
- is_white = board.turn # True for white, False for black
-
- # Get castling rights as a single integer (0-15)
- # Bit 0: white kingside, Bit 1: white queenside, Bit 2: black kingside, Bit 3: black queenside
- castling_rights = 0
- if board.has_kingside_castling_rights(chess.WHITE):
- castling_rights |= 1
- if board.has_queenside_castling_rights(chess.WHITE):
- castling_rights |= 2
- if board.has_kingside_castling_rights(chess.BLACK):
- castling_rights |= 4
- if board.has_queenside_castling_rights(chess.BLACK):
- castling_rights |= 8
-
- # Get en passant file (0-7 for a-h, 8 for none)
- if board.has_legal_en_passant():
- en_passant_file = chess.square_file(board.ep_square)
- else:
- en_passant_file = 8 # No en passant
-
- return {
- "position": position_tensor,
- "move": next_move_token,
- "is_white": is_white,
- "castling_rights": castling_rights,
- "en_passant_file": en_passant_file,
- "move_number": move_idx,
- "white_elo": white_elo,
- "black_elo": black_elo,
- "result": result,
- "from_square": from_square,
- "action_plane": action_plane,
- "legal_moves_planes": legal_moves_planes,
- }
-
- def _decode_move_token(self, token: int) -> str:
- """Decode a move token back to UCI string."""
- return self.move_tokenizer.decode(token)
-
-
-if __name__ == "__main__":
- # Example usage
- import sys
-
- if len(sys.argv) < 2:
- print("Usage: python h5_lichess_dataset.py ")
- sys.exit(1)
-
- dataset = HDF5ChessDataset(sys.argv[1], cache_size=100)
- print(f"Dataset length: {len(dataset)}")
-
- # Test a few samples
- for i in range(min(5, len(dataset))):
- sample = dataset[i]
- print(f"\nSample {i}:")
- print(f" Position shape: {sample['position'].shape}")
- print(f" Move token: {sample['move'].item()}")
- print(f" Turn: {'White' if sample['is_white'] else 'Black'}")
- print(f" ELOs: {sample['white_elo']} vs {sample['black_elo']}")
- print(f" Move number: {sample['move_number']} over {dataset.num_moves_per_game[i]} moves")
diff --git a/chesstransformer/datasets/lichess.py b/chesstransformer/datasets/lichess.py
deleted file mode 100644
index 38c861492879c2d0956af75bd6c661acc967a544..0000000000000000000000000000000000000000
--- a/chesstransformer/datasets/lichess.py
+++ /dev/null
@@ -1,223 +0,0 @@
-from torch.utils.data import Dataset
-import zstandard as zstd
-import chess.pgn
-import io
-import torch
-
-
-def extract_games_optimized(filepath, num_games=1000000, batch_size=1000):
- """
- Optimized game extraction with progress tracking.
- Process in batches and show estimated time remaining.
-
- Args:
- filepath: Path to the compressed PGN file
- num_games: Number of games to extract (default: 1,000,000)
- batch_size: Number of games to process in each batch (default: 1,000)
-
- Returns:
- List of extracted games
- """
- games = []
- dctx = zstd.ZstdDecompressor()
- with open(filepath, "rb") as f:
- with dctx.stream_reader(f) as reader:
- text_stream = io.TextIOWrapper(reader, encoding="utf-8")
- pgn = chess.pgn.read_game(text_stream)
- game_count = 0
- batch_count = 0
-
- while pgn is not None and game_count < num_games:
- # check if the game ended by a win/loss/draw
- if pgn.headers.get("Result") not in ["1-0", "0-1", "1/2-1/2"]:
- pgn = chess.pgn.read_game(text_stream)
- continue
-
- # Skip Chess960 games - only use classical chess
- variant = pgn.headers.get("Variant", "Standard")
- if variant != "Standard":
- pgn = chess.pgn.read_game(text_stream)
- continue
-
- games.append(pgn)
- game_count += 1
- batch_count += 1
-
- if batch_count >= batch_size:
- batch_count = 0
- print(f"Extracted {game_count}/{num_games} games...")
-
- pgn = chess.pgn.read_game(text_stream)
-
- print(f"Finished extracting {len(games)} games.")
- return games
-
-
-def encode_game_moves(game, notation="uci", seprator=" "):
- """
- Encode the moves of a single game using the provided tokenizer.
-
- Args:
- game: A chess.pgn.Game object
- notation: Notation format for moves ("uci" or "san", default: "uci")
- seprator: String to separate moves (default: " ")
-
- Returns:
- List of moves as a single formatted string
- """
- if notation == "uci":
- move_sequence = seprator.join([move.uci() for move in game.mainline_moves()])
- elif notation == "san":
- board = game.board()
- move_sequence = seprator.join(
- [
- board.san(move)
- for move in game.mainline_moves()
- if not board.push(move) # Push the move to update the board state
- ]
- )
- else:
- raise ValueError("Unsupported notation format. Use 'uci' or 'san'.")
-
- ending_condition = seprator + {
- "1-0": "<1-0>",
- "0-1": "<0-1>",
- "1/2-1/2": "<1/2-1/2>",
- }.get(game.headers.get("Result", ""), "")
- if ending_condition:
- move_sequence += ending_condition
-
- return move_sequence
-
-
-class LichessDataset(Dataset):
- def __init__(
- self,
- dataset_path,
- tokenizer,
- notation="uci",
- context_length=512,
- nb_games=100000,
- ):
- """
- Lichess dataset for training chess move prediction models.
-
- Args:
- dataset_path: Path to the lichess dataset file (PGN format)
- tokenizer: Tokenizer object to convert moves to token IDs
- notation: Notation format for moves ("uci" or "san", default: "uci")
- context_length: Maximum sequence length for model input (default: 512)
- """
- self.tokenizer = tokenizer
- self.context_length = context_length
- self.notation = notation
-
- print(f"Loading dataset from {dataset_path}...")
- self.games = extract_games_optimized(dataset_path, num_games=nb_games)
- print(f"Total games loaded: {len(self.games)}")
-
- self.encoded_games = []
- # encode and stack games together
- for idx, game in enumerate(self.games):
- encoded_moves = encode_game_moves(game, notation=self.notation)
- token_ids = self.tokenizer.encode(encoded_moves).ids
-
- self.encoded_games.append(token_ids)
-
- # flatten the list of token ids
- self.encoded_games = [token_id for game in self.encoded_games for token_id in game]
-
- def __len__(self):
- return len(self.encoded_games) // self.context_length
-
- def __getitem__(self, idx):
- """
- Get a sequence of token IDs for training.
-
- Args:
- idx: Index of the game to retrieve
- Returns:
- input, target: Tuple of input and target token ID lists
- """
- start_idx = idx * self.context_length
- end_idx = start_idx + self.context_length + 1 # +1 for target shift
-
- if end_idx > len(self.encoded_games):
- raise IndexError("Index out of range for the dataset.")
-
- input_ids = self.encoded_games[start_idx : end_idx - 1]
- target_ids = self.encoded_games[start_idx + 1 : end_idx]
-
- input_tensor = torch.tensor(input_ids, dtype=torch.long)
- target_tensor = torch.tensor(target_ids, dtype=torch.long)
-
- return input_tensor, target_tensor
-
-
-if __name__ == "__main__":
- from pathlib import Path
- import argparse
- from chesstransformer.models.tokenizer import create_bpe_tokenizer
-
- def arg_parser():
- parser = argparse.ArgumentParser(description="Test LichessDataset")
- parser.add_argument(
- "--dataset_path",
- type=str,
- default="data/Lichess Rated Games 2017.pgn.zst",
- help="Path to the lichess dataset file (PGN format)",
- )
- parser.add_argument(
- "--num_games",
- type=int,
- default=10000,
- help="Number of games to load for testing (default: 1000)",
- )
- parser.add_argument(
- "--vocab_path",
- type=str,
- default="src/chesstransformer/data/tokenizer_models/bpe_tokenizer_vocab2000.json",
- help="Path to the trained tokenizer file",
- )
- parser.add_argument(
- "--context_length",
- type=int,
- default=512,
- help="Context length for sequences (default: 512)",
- )
- parser.add_argument(
- "--notation",
- type=str,
- choices=["uci", "san"],
- default="uci",
- help="Notation format for moves (default: uci)",
- )
- args = parser.parse_args()
- return args
-
- args = arg_parser()
- dataset_path = Path(args.dataset_path)
- num_games = args.num_games
- vocab_path = Path(args.vocab_path)
- context_length = args.context_length
- notation = args.notation
-
- import tokenizers
-
- tokenizer = tokenizers.Tokenizer.from_file(str(vocab_path))
- print(f"Loading Lichess dataset from {dataset_path}...")
- dataset = LichessDataset(
- dataset_path=dataset_path,
- tokenizer=tokenizer,
- notation=notation,
- context_length=context_length,
- nb_games=num_games,
- )
-
- print(f"Dataset loaded with {len(dataset)} games.")
- input_ids, target_ids = dataset[0]
- print(f"Sample input IDs: {input_ids}")
- print(f"Sample target IDs: {target_ids}")
- print(f"Encoded and stacked {len(dataset.encoded_games)} token IDs.")
-
- print(f"Decoded sample moves: {tokenizer.decode(input_ids)}")
diff --git a/chesstransformer/datasets/lichess_simple_uci.py b/chesstransformer/datasets/lichess_simple_uci.py
deleted file mode 100644
index 64a345732694247ed629a4fed2d340c7728926b3..0000000000000000000000000000000000000000
--- a/chesstransformer/datasets/lichess_simple_uci.py
+++ /dev/null
@@ -1,136 +0,0 @@
-from pathlib import Path
-import json
-from torch.utils.data import Dataset
-import zstandard as zstd
-import chess.pgn
-import io
-import torch
-import chess
-
-
-root_path = Path(__file__).parents[1]
-print(f"Root path: {root_path}")
-
-
-class LichessSimpleUciDataset(Dataset):
- def __init__(
- self,
- dataset_path,
- vocab_path=root_path / "data/tokenizer_models/uci_vocab.json",
- context_length=512,
- num_samples=100_000,
- ):
- """
- Lichess dataset in the simple UCI format."""
-
- self.dataset_path = dataset_path
- self.num_samples = num_samples
- self.vocab_path = vocab_path
- self.context_length = context_length
- self._load_vocab()
- self.games = self._extract_games()
-
- def _extract_games(self, only_check_mate=True):
- games = []
- dctx = zstd.ZstdDecompressor()
- with open(self.dataset_path, "rb") as f:
- with dctx.stream_reader(f) as reader:
- text_stream = io.TextIOWrapper(reader, encoding="utf-8")
- game_count = 0
-
- while game_count < self.num_samples:
- pgn = chess.pgn.read_game(text_stream)
- if pgn is None:
- break
-
- # Check if the game ended by a win/loss/draw
- if pgn.headers.get("Result") not in ["1-0", "0-1", "1/2-1/2"]:
- continue
-
- # Skip Chess960 games - only use classical chess
- variant = pgn.headers.get("Variant", "Standard")
- if variant != "Standard":
- continue
-
- games.append(pgn)
- game_count += 1
-
- if game_count % 10000 == 0:
- print(f"Extracted {game_count}/{self.num_samples} games...", end="\r")
-
- print(f"\nFinished extracting {len(games)} games.")
- return games
-
- def _load_vocab(self):
- with open(self.vocab_path, "r") as f:
- self.vocab = json.load(f)
- self.token_to_id = {token: idx for idx, token in enumerate(self.vocab)}
-
- def __len__(self):
- return len(self.games)
-
- def __getitem__(self, idx):
- game = self.games[idx]
- token_ids = self.encode_game(game)
- legals_mask = self.get_legal_moves_mask(game)
- target_ids = token_ids[1:] + [-100] # Shifted targets with -100 for padding
- return {
- "input_ids": torch.tensor(token_ids, dtype=torch.long),
- "target_ids": torch.tensor(target_ids, dtype=torch.long),
- "legal_moves_mask": legals_mask,
- }
-
- def encode_game(self, game):
- move_list = [move.uci() for move in game.mainline_moves()]
- move_list = [""] + move_list + [""]
- token_ids = self.encode_moves(move_list)
- return token_ids
-
- def encode_moves(self, moves):
- token_ids = []
- for move in moves:
- if move in self.token_to_id:
- token_ids.append(self.token_to_id[move])
- else:
- token_ids.append(self.token_to_id[""])
-
- return token_ids
-
- def get_legal_moves_mask(self, game):
- """
- For each move in the game, generate a mask of legal moves at that position.
- Returns a list of masks, where each mask is a tensor of shape (vocab_size,)
- with 1.0 for legal moves and 0.0 for illegal moves.
- """
- board = game.board()
- masks = []
-
- for move in game.mainline_moves():
- # Get all legal moves at current position
- legal_moves = [m.uci() for m in board.legal_moves]
-
- # Create mask for all tokens in vocabulary
- mask = torch.zeros(len(self.vocab), dtype=torch.float32)
-
- # Set 1.0 for legal moves
- for legal_move in legal_moves:
- if legal_move in self.token_to_id:
- mask[self.token_to_id[legal_move]] = 1.0
-
- masks.append(mask)
-
- # Apply the move to the board for next iteration
- board.push(move)
-
- # Add mask for and tokens
- start_end_mask = torch.zeros(len(self.vocab), dtype=torch.float32)
- start_end_mask[self.token_to_id[""]] = 1.0
- start_end_mask[self.token_to_id[""]] = 1.0
- masks = [start_end_mask] + masks + [start_end_mask]
-
- return torch.stack(masks)
-
- def decode_game(self, token_ids):
- id_to_token = {idx: token for token, idx in self.token_to_id.items()}
- moves = [id_to_token.get(tid, "") for tid in token_ids if tid in id_to_token]
- return " ".join(moves)
diff --git a/chesstransformer/datasets/puzzle_dataset.py b/chesstransformer/datasets/puzzle_dataset.py
deleted file mode 100644
index 386e06ce8e3417c409f1fdcdbb546779b214c4da..0000000000000000000000000000000000000000
--- a/chesstransformer/datasets/puzzle_dataset.py
+++ /dev/null
@@ -1,259 +0,0 @@
-import csv
-import zstandard as zstd
-import torch
-from torch.utils.data import Dataset
-import chess
-from typing import Optional
-from chesstransformer.models.tokenizer.position_tokenizer import PostionTokenizer
-from chesstransformer.models.tokenizer.move_tokenizer import MoveTokenizer
-
-
-class LichessPuzzleDataset(Dataset):
- """Dataset for Lichess chess puzzles.
-
- Each puzzle consists of:
- - A FEN position (the position where the puzzle starts)
- - A sequence of moves (the solution to the puzzle)
- - Rating and themes
-
- The dataset returns position-move pairs where:
- - position: The current board state encoded as 64 tokens
- - move: The next move in the puzzle solution
- - is_white: Whether white is to move
- """
-
- def __init__(
- self,
- puzzle_path: str,
- min_rating: int = None,
- max_rating: int = None,
- themes: list[str] = None,
- max_puzzles: int = None,
- ):
- """
- Args:
- puzzle_path: Path to the .csv.zst puzzle file
- min_rating: Minimum puzzle rating to include
- max_rating: Maximum puzzle rating to include
- themes: List of themes to filter by (e.g., ['mate', 'mateIn2'])
- max_puzzles: Maximum number of puzzles to load (for testing)
- """
- self.position_tokenizer = PostionTokenizer()
- self.move_tokenizer = MoveTokenizer()
- self.puzzles = []
-
- print(f"Loading puzzles from {puzzle_path}...")
-
- # Load and filter puzzles
- with zstd.open(puzzle_path, "rt", encoding="utf-8") as f:
- reader = csv.DictReader(f)
- for i, row in enumerate(reader):
- if max_puzzles and i >= max_puzzles:
- break
-
- # Parse puzzle data
- puzzle_rating = int(row["Rating"])
- puzzle_themes = row["Themes"].split()
-
- # Apply filters
- if min_rating and puzzle_rating < min_rating:
- continue
- if max_rating and puzzle_rating > max_rating:
- continue
- if themes and not any(theme in puzzle_themes for theme in themes):
- continue
-
- # Parse moves (space-separated UCI moves)
- moves = row["Moves"].split()
- if len(moves) < 2: # Need at least opponent move + our response
- continue
-
- self.puzzles.append(
- {
- "puzzle_id": row["PuzzleId"],
- "fen": row["FEN"],
- "moves": moves,
- "rating": puzzle_rating,
- "themes": puzzle_themes,
- "game_url": row["GameUrl"],
- }
- )
-
- if (i + 1) % 10000 == 0:
- print(f"Loaded {i + 1} puzzles...")
-
- print(f"Total puzzles loaded: {len(self.puzzles)}")
-
- # Calculate total position-move pairs
- self.total_pairs = sum(len(p["moves"]) - 1 for p in self.puzzles)
- print(f"Total position-move pairs: {self.total_pairs}")
-
- def __len__(self):
- """Return total number of position-move pairs across all puzzles."""
- return self.total_pairs
-
- def __getitem__(self, idx):
- """
- Returns a position-move pair from the puzzle dataset.
-
- The dataset is indexed as a flat sequence of all position-move pairs
- across all puzzles. This means we need to:
- 1. Find which puzzle this index belongs to
- 2. Find which move within that puzzle
- 3. Replay the game to get the board state
- 4. Return the encoded position + next move
- """
- # Find the puzzle and move index
- cumulative = 0
- for puzzle in self.puzzles:
- puzzle_pairs = len(puzzle["moves"]) - 1
- if idx < cumulative + puzzle_pairs:
- # Found the right puzzle
- move_idx = idx - cumulative
- break
- cumulative += puzzle_pairs
- else:
- raise IndexError(f"Index {idx} out of range")
-
- # Set up the board from FEN
- board = chess.Board(puzzle["fen"])
-
- # Apply moves up to (but not including) the target move
- for i in range(move_idx + 1): # +1 because first move is opponent's
- move = chess.Move.from_uci(puzzle["moves"][i])
- board.push(move)
-
- # Get the target move
- target_move = puzzle["moves"][move_idx + 1]
-
- # Encode the position
- position_tokens = self.position_tokenizer.encode(board)
-
- # Encode the move
- try:
- move_token = self.move_tokenizer.encode(target_move)
- except ValueError:
- # If move not in vocabulary, skip this sample
- # This shouldn't happen often with proper vocab
- print(f"Warning: Move {target_move} not in vocabulary for puzzle {puzzle['puzzle_id']}")
- move_token = 0 # Use a default
-
- # Get legal moves mask for loss calculation
- legal_moves = list(board.legal_moves)
- legal_moves_mask = torch.zeros(self.move_tokenizer.vocab_size, dtype=torch.float32)
- for move in legal_moves:
- try:
- move_id = self.move_tokenizer.encode(move.uci())
- legal_moves_mask[move_id] = 1.0
- except ValueError:
- continue
-
- return {
- "position": torch.tensor(position_tokens, dtype=torch.long),
- "move": torch.tensor(move_token, dtype=torch.long),
- "is_white": board.turn == chess.WHITE,
- "puzzle_id": puzzle["puzzle_id"],
- "puzzle_rating": puzzle["rating"],
- "legal_moves_mask": legal_moves_mask,
- "themes": puzzle["themes"],
- }
-
- def get_puzzle_by_id(self, puzzle_id: str):
- """Get a complete puzzle by its ID."""
- for puzzle in self.puzzles:
- if puzzle["puzzle_id"] == puzzle_id:
- return puzzle
- return None
-
-
-class LichessPuzzleFullSolutionDataset(Dataset):
- """Alternative puzzle dataset that returns complete puzzles (not individual moves).
-
- Useful for evaluation where you want to test solving the entire puzzle.
- """
-
- def __init__(
- self,
- puzzle_path: str,
- min_rating: int = None,
- max_rating: int = None,
- themes: list[str] = None,
- max_puzzles: int = None,
- ):
- """
- Args:
- puzzle_path: Path to the .csv.zst puzzle file
- min_rating: Minimum puzzle rating to include
- max_rating: Maximum puzzle rating to include
- themes: List of themes to filter by
- max_puzzles: Maximum number of puzzles to load
- """
- self.position_tokenizer = PostionTokenizer()
- self.move_tokenizer = MoveTokenizer()
- self.puzzles = []
-
- print(f"Loading puzzles from {puzzle_path}...")
-
- with zstd.open(puzzle_path, "rt", encoding="utf-8") as f:
- reader = csv.DictReader(f)
- for i, row in enumerate(reader):
- if max_puzzles and i >= max_puzzles:
- break
-
- puzzle_rating = int(row["Rating"])
- puzzle_themes = row["Themes"].split()
-
- if min_rating and puzzle_rating < min_rating:
- continue
- if max_rating and puzzle_rating > max_rating:
- continue
- if themes and not any(theme in puzzle_themes for theme in themes):
- continue
-
- moves = row["Moves"].split()
- if len(moves) < 2:
- continue
-
- self.puzzles.append(
- {
- "puzzle_id": row["PuzzleId"],
- "fen": row["FEN"],
- "moves": moves,
- "rating": puzzle_rating,
- "themes": puzzle_themes,
- "game_url": row["GameUrl"],
- }
- )
-
- print(f"Total puzzles loaded: {len(self.puzzles)}")
-
- def __len__(self):
- return len(self.puzzles)
-
- def __getitem__(self, idx):
- """Returns a complete puzzle with starting position and full solution."""
- puzzle = self.puzzles[idx]
-
- # Starting position
- board = chess.Board(puzzle["fen"])
- position_tokens = self.position_tokenizer.encode(board)
-
- # Encode solution moves
- solution_tokens = []
- for move_uci in puzzle["moves"]:
- try:
- move_token = self.move_tokenizer.encode(move_uci)
- solution_tokens.append(move_token)
- except ValueError:
- solution_tokens.append(0)
-
- return {
- "puzzle_id": puzzle["puzzle_id"],
- "position": torch.tensor(position_tokens, dtype=torch.long),
- "solution": torch.tensor(solution_tokens, dtype=torch.long),
- "fen": puzzle["fen"],
- "moves_uci": puzzle["moves"],
- "rating": puzzle["rating"],
- "themes": puzzle["themes"],
- "is_white": board.turn == chess.WHITE,
- }
diff --git a/chesstransformer/datasets/shard_format.py b/chesstransformer/datasets/shard_format.py
deleted file mode 100644
index 7934161b6f0f611472adc5a0f2581692f2aede57..0000000000000000000000000000000000000000
--- a/chesstransformer/datasets/shard_format.py
+++ /dev/null
@@ -1,129 +0,0 @@
-"""On-disk record format for the flat, memory-mappable training shards.
-
-Shared by ``scripts/build_shards.py`` (writer), ``FlatShardDataset`` (reader) and
-``scripts/verify_shards.py`` (equivalence harness), so the layout is defined exactly
-once. See ``doc/dataset_shards.md`` §3 for the design rationale and the measurements
-behind the choices below.
-
-One sample is one fixed-width row. Only the nine fields ``pos2move_v2_trainer.py``
-actually reads are stored; everything else ``HDF5ChessDataset`` returns is dropped.
-
-The legal-move mask is the whole design problem: ``(64, 73)`` bool is 4,672 bytes and
-would dominate the record 20:1. It is stored instead as a fixed-width list of at most
-``LEGAL_CAP`` flat ``from_square * 73 + plane`` indices (129 bytes), which is what makes
-one memmap row per sample — and therefore trivial rank/worker slicing — possible. The
-mask is rebuilt on GPU by :func:`~chesstransformer.datasets.flat_shard_dataset.scatter_legal_planes`.
-
-Fixed-width was chosen over CSR (~72 B/sample, no cap) deliberately: CSR is ~28 % smaller
-and is what Megatron's ``IndexedDataset`` does, but it needs a custom ragged collate, and
-the 1.4 GB it saves is on a dataset that already fits in page cache. The cap is not a
-correctness risk because the *target* move's index is always written first (see
-``LEGAL_CAP`` below), so clipping can only ever drop alternative legal moves, never the
-label. Observed legal-move counts over 400 real positions: mean 33.9, p99.9 = 53, max 53.
-"""
-
-import numpy as np
-
-NUM_ACTION_PLANES = 73
-NUM_SQUARES = 64
-
-#: Number of legal-move indices stored per sample. Sits well above the observed max of
-#: 53. The builder counts overflows and prints the rate; if it ever climbs above ~0.01 %
-#: raise this to 96 and rebuild.
-LEGAL_CAP = 64
-
-#: Flat action index space is ``[0, 4672)``. Unused slots in ``legal_idx`` are filled with
-#: ``PAD_ACTION`` (== 4672), one past the end, so a GPU ``scatter_`` can write padding into
-#: a scratch column that is then sliced off. Writing padding as 0 instead would silently
-#: mark square a1 / plane 0 legal on every single sample.
-PAD_ACTION = NUM_SQUARES * NUM_ACTION_PLANES
-
-#: Packed record layout. Offsets are explicit rather than auto-computed so that both
-#: ``uint16`` fields land on even offsets — an unaligned memmap view of ``legal_idx``
-#: would force numpy into a slow per-element copy on every batch gather.
-#:
-#: ========== ====== =========================================================
-#: offset dtype field
-#: ========== ====== =========================================================
-#: 0-63 u1×64 position board tokens, vocab 0-12
-#: 64-191 u2×64 legal_idx flat legal actions, target first, PAD-filled
-#: 192 u1 legal_cnt number of valid entries in legal_idx
-#: 193 u1 from_square target move source square, 0-63
-#: 194 u1 action_plane target move AlphaZero plane, 0-72
-#: 195 u1 castling_rights 4-bit mask, 0-15
-#: 196 u1 en_passant_file 0-7 = file a-h, 8 = none
-#: 197 u1 is_white side to move
-#: 198 i1 result 0 = draw, 1 = white win, 2 = black win (as in the HDF5)
-#: 199 u1 (pad) keeps move_number 2-byte aligned
-#: 200-201 u2 move_number ply index, max observed 601
-#: ========== ====== =========================================================
-RECORD_DTYPE = np.dtype(
- {
- "names": [
- "position",
- "legal_idx",
- "legal_cnt",
- "from_square",
- "action_plane",
- "castling_rights",
- "en_passant_file",
- "is_white",
- "result",
- "move_number",
- ],
- "formats": [
- (np.uint8, NUM_SQUARES),
- (np.uint16, LEGAL_CAP),
- np.uint8,
- np.uint8,
- np.uint8,
- np.uint8,
- np.uint8,
- np.uint8,
- np.int8,
- np.uint16,
- ],
- "offsets": [0, 64, 192, 193, 194, 195, 196, 197, 198, 200],
- "itemsize": 202,
- }
-)
-
-RECORD_BYTES = RECORD_DTYPE.itemsize
-
-#: Bumped whenever the layout above changes in a way that makes old shards unreadable.
-#: ``FlatShardDataset`` refuses to load a directory built by a different version.
-FORMAT_VERSION = 1
-
-
-def dtype_to_meta() -> dict:
- """Serialise :data:`RECORD_DTYPE` into ``meta.json``.
-
- Stored so a shard directory is self-describing: a reader can reconstruct the exact
- layout it was written with instead of trusting that this module never changed.
- """
- return {
- "names": list(RECORD_DTYPE.names),
- "formats": [
- (
- [RECORD_DTYPE[n].subdtype[0].str, RECORD_DTYPE[n].subdtype[1][0]]
- if RECORD_DTYPE[n].subdtype
- else RECORD_DTYPE[n].str
- )
- for n in RECORD_DTYPE.names
- ],
- "offsets": [RECORD_DTYPE.fields[n][1] for n in RECORD_DTYPE.names],
- "itemsize": RECORD_BYTES,
- }
-
-
-def dtype_from_meta(meta: dict) -> np.dtype:
- """Rebuild a record dtype from what ``meta.json`` recorded."""
- formats = [tuple(f) if isinstance(f, list) else f for f in meta["formats"]]
- return np.dtype(
- {
- "names": list(meta["names"]),
- "formats": formats,
- "offsets": list(meta["offsets"]),
- "itemsize": int(meta["itemsize"]),
- }
- )
diff --git a/chesstransformer/distributed_muon.py b/chesstransformer/distributed_muon.py
deleted file mode 100644
index 55ce071f3b13659a3cbd37e3b82cd3fda5354d5f..0000000000000000000000000000000000000000
--- a/chesstransformer/distributed_muon.py
+++ /dev/null
@@ -1,222 +0,0 @@
-"""Muon with Newton-Schulz distributed across ranks, plus its schedules.
-
-Under plain DDP every rank all-reduces to the *same* gradient and then every rank runs the
-*same* Newton-Schulz iteration on it. That is pure redundant compute: at world size N,
-N-1 copies of the NS work are thrown away, and every rank stores a full momentum buffer.
-
-The fix is to partition the work — but the partition axis is the whole point. NS is
-``X @ X.T`` plus a global spectral-norm division, so it needs the *complete* matrix.
-Measured on a 64x32 weight, orthogonalizing per-shard and concatenating gives a **52-58 %
-wrong update** at world size 2-4, and nothing raises: a row-slice of a 2D matrix is still a
-2D matrix. That is the ZeRO-2 trap, and it is why sharding *inside* a tensor is off limits
-here even though it is perfectly safe for elementwise optimizers like AdamW.
-
-So this partitions by **whole parameter**, following nanochat's ``MuonAdamW``
-(https://github.com/karpathy/nanochat, ``nanochat/optim.py``): params of a given shape are
-stacked, the K of them are divided into contiguous chunks across ranks, each rank runs NS
-only on the complete matrices it owns, and one ``all_gather`` per shape bucket republishes
-the results. Momentum state exists only for owned params, so it is sharded too.
-
-**Communication note, stated honestly.** nanochat does not use DDP: it ``reduce_scatter``s
-gradients inside the optimizer, so grad-sync and state-sharding share one collective and
-the total wire cost equals what DDP's all-reduce would have been. This class runs *on top
-of* DDP, which has already all-reduced the gradients, so the ``all_gather`` here is
-**additional** traffic — bytes DDP did not previously move. It buys NS compute (÷N) and
-momentum memory (÷N) at the cost of one param-sized gather per step. On a fast interconnect
-that is a clear win; on PCIe it may not be. ``scripts/bench_distributed_muon.py`` measures
-it rather than assuming.
-
-**Measured so far (gloo/CPU, preset 46m, 121 2D tensors, 16-core box):**
-
-=========== =========== ==================
-world size step time momentum state
-=========== =========== ==================
-1 243 ms 177.0 MiB/rank
-2 638 ms 89.0 MiB/rank
-4 491 ms 45.0 MiB/rank
-=========== =========== ==================
-
-The memory result is the designed one and is exact: state divides by world size. **The step
-time is currently a regression**, and the honest reading is that this configuration has not
-been shown to be a compute win. Three things confound the CPU numbers -- gloo is far slower
-relative to compute than NCCL, N ranks on one 16-core box contend for the same cores so the
-"NS compute / N" never materialises, and the benchmark is pure optimizer with no forward or
-backward, so the all_gather is 100 % of the measured time instead of a few percent of a real
-step. It still should not be reported as a win until multi-GPU says so.
-
-The structural fix is the fused variant, which is why nanochat does not use DDP at all: it
-``reduce_scatter``s gradients inside the optimizer, so grad-sync and state-sharding share one
-collective and the total wire cost equals DDP's all-reduce -- the ``all_gather`` stops being
-*extra*. Doing that here means excluding the Muon params from DDP's reducer
-(``DistributedDataParallel._set_params_and_buffers_to_ignore_for_model``) and doing the
-reduce-scatter in ``step()``. That is the next step, and it is what turns this from "memory
-win, compute unproven" into a pure win.
-"""
-
-import math
-
-import torch
-import torch.distributed as dist
-
-# Reuse torch's own kernels rather than reimplementing the algorithm: it makes this
-# optimizer numerically identical to torch.optim.Muon by construction, which is exactly
-# what tests/test_distributed_muon.py asserts. Private, so guarded.
-try:
- from torch.optim._muon import _adjust_lr, _zeropower_via_newtonschulz
-except ImportError as exc: # pragma: no cover
- raise ImportError(
- "DistributedMuon reuses torch.optim._muon internals; needs a torch with "
- "torch.optim.Muon (2.9+)."
- ) from exc
-
-#: Params smaller than this stay replicated: every rank computes them redundantly and no
-#: collective is issued. For Pos2MoveV2-large that is 9 of 13 shape buckets holding 0.2M of
-#: 156.4M params — distributing them would cost 9 extra collectives to save nothing.
-DEFAULT_MIN_DISTRIBUTED_NUMEL = 65_536
-
-
-def muon_momentum(step: int, total_steps: int, warmup: int = 400,
- warmdown_ratio: float = 0.2, lo: float = 0.85,
- hi: float = 0.97, end: float = 0.90) -> float:
- """Momentum schedule: warm up 0.85 -> 0.97, then down to 0.90 during LR warmdown.
-
- Muon's momentum controls how much history the orthogonalized direction carries. Early
- on the gradient is changing fast and a long memory averages over a moving target, so it
- starts low; late in training a slightly *shorter* memory helps the run settle, which is
- why it comes back down rather than staying pinned at the peak. Schedule follows
- nanochat's ``get_muon_momentum``.
- """
- if step < warmup:
- frac = step / max(1, warmup)
- return (1 - frac) * lo + frac * hi
- warmdown_iters = max(1, round(warmdown_ratio * total_steps))
- warmdown_start = total_steps - warmdown_iters
- if step >= warmdown_start:
- progress = min(1.0, (step - warmdown_start) / warmdown_iters)
- return hi * (1 - progress) + end * progress
- return hi
-
-
-def muon_weight_decay(step: int, total_steps: int, base_wd: float) -> float:
- """Cosine-decay weight decay to zero over the run, as nanochat's ``get_weight_decay``."""
- return base_wd * 0.5 * (1 + math.cos(math.pi * min(1.0, step / max(1, total_steps))))
-
-
-class DistributedMuon(torch.optim.Optimizer):
- """Drop-in for ``torch.optim.Muon`` that splits NS work and momentum across ranks.
-
- Assumes gradients are already synchronized (DDP has all-reduced them), so every rank
- starts from an identical gradient and only the *work* is divided. With no process group
- initialized it degenerates to exactly ``torch.optim.Muon``.
- """
-
- def __init__(self, params, lr=1e-3, weight_decay=0.1, momentum=0.95, nesterov=True,
- ns_coefficients=(3.4445, -4.7750, 2.0315), eps=1e-7, ns_steps=5,
- adjust_lr_fn=None, min_distributed_numel=DEFAULT_MIN_DISTRIBUTED_NUMEL):
- defaults = dict(lr=lr, weight_decay=weight_decay, momentum=momentum,
- nesterov=nesterov, ns_coefficients=ns_coefficients, eps=eps,
- ns_steps=ns_steps, adjust_lr_fn=adjust_lr_fn)
- super().__init__(params, defaults)
-
- self.world_size = dist.get_world_size() if dist.is_initialized() else 1
- self.rank = dist.get_rank() if dist.is_initialized() else 0
- self.min_distributed_numel = min_distributed_numel
- self._plan = [self._plan_group(g) for g in self.param_groups]
-
- owned = sum(len(b["owned"]) for p in self._plan for b in p["buckets"])
- total = sum(len(g["params"]) for g in self.param_groups)
- self.owned_param_count, self.total_param_count = owned, total
-
- def _plan_group(self, group):
- """Bucket a group's params by shape and assign contiguous chunks to ranks.
-
- Contiguous rather than strided so that ``all_gather_into_tensor`` — which lays rank
- i's contribution at output[i*chunk:(i+1)*chunk] — reassembles the bucket in order
- with no extra permutation.
- """
- by_shape: dict[tuple, list] = {}
- for p in group["params"]:
- if p.ndim != 2:
- raise ValueError(f"DistributedMuon handles 2D params only, got {p.ndim}D")
- by_shape.setdefault(tuple(p.shape), []).append(p)
-
- buckets = []
- for shape, plist in sorted(by_shape.items(), key=lambda kv: -kv[1][0].numel() * len(kv[1])):
- k = len(plist)
- distributed = (self.world_size > 1
- and plist[0].numel() >= self.min_distributed_numel)
- if not distributed:
- # Replicated: every rank updates every param, no collective.
- buckets.append(dict(shape=shape, params=plist, owned=plist,
- offset=0, chunk=k, distributed=False))
- continue
- chunk = math.ceil(k / self.world_size)
- start = self.rank * chunk
- owned = plist[start:start + chunk] # may be short on the last rank
- buckets.append(dict(shape=shape, params=plist, owned=owned,
- offset=start, chunk=chunk, distributed=True))
- return dict(buckets=buckets)
-
- @torch.no_grad()
- def step(self, closure=None):
- loss = None
- if closure is not None:
- with torch.enable_grad():
- loss = closure()
-
- for group, plan in zip(self.param_groups, self._plan):
- lr, wd = group["lr"], group["weight_decay"]
- momentum, nesterov = group["momentum"], group["nesterov"]
- for bucket in plan["buckets"]:
- for p in bucket["owned"]:
- if p.grad is None:
- continue
- self._update_one(p, lr, wd, momentum, nesterov, group)
- for bucket in plan["buckets"]:
- if bucket["distributed"]:
- self._publish(bucket)
- return loss
-
- def _update_one(self, p, lr, wd, momentum, nesterov, group):
- """Exactly torch.optim.Muon's ``_single_tensor_muon`` body, for one param."""
- grad = p.grad
- if grad.ndim != 2:
- raise ValueError("Param gradient must be a 2D matrix")
- state = self.state[p]
- if "momentum_buffer" not in state:
- state["momentum_buffer"] = torch.zeros_like(p)
- buf = state["momentum_buffer"]
- buf.lerp_(grad, 1 - momentum)
- update = grad.lerp(buf, momentum) if nesterov else buf
- update = _zeropower_via_newtonschulz(
- update, group["ns_coefficients"], group["ns_steps"], group["eps"])
- adjusted_lr = _adjust_lr(lr, group["adjust_lr_fn"], p.shape)
- p.mul_(1 - lr * wd)
- p.add_(update, alpha=-adjusted_lr)
-
- def _publish(self, bucket):
- """One all_gather per shape bucket to give every rank the updated params.
-
- Zero-padding when K does not divide world_size: the tail slots are gathered and
- discarded, which costs a little bandwidth and keeps the collective's shape uniform.
- """
- plist, chunk, shape = bucket["params"], bucket["chunk"], bucket["shape"]
- k, ws = len(plist), self.world_size
- ref = plist[0]
- local = torch.zeros(chunk, *shape, dtype=ref.dtype, device=ref.device)
- for i, p in enumerate(bucket["owned"]):
- local[i].copy_(p)
- out = torch.empty(chunk * ws, *shape, dtype=ref.dtype, device=ref.device)
- dist.all_gather_into_tensor(out, local)
- for i in range(k):
- plist[i].copy_(out[i])
-
- def sharding_summary(self) -> str:
- parts = []
- for plan in self._plan:
- for b in plan["buckets"]:
- tag = "dist" if b["distributed"] else "repl"
- parts.append(f"{tuple(b['shape'])}x{len(b['params'])}:{tag}")
- return (f"DistributedMuon rank {self.rank}/{self.world_size}: owns "
- f"{self.owned_param_count}/{self.total_param_count} params | "
- + " ".join(parts))
diff --git a/chesstransformer/introspection/__init__.py b/chesstransformer/introspection/__init__.py
deleted file mode 100644
index 6f9090bab576afaa3fb3ccd3686e504f5fec0ffd..0000000000000000000000000000000000000000
--- a/chesstransformer/introspection/__init__.py
+++ /dev/null
@@ -1,5 +0,0 @@
-"""Model introspection: attention capture, logit lens, occlusion saliency."""
-
-from chesstransformer.introspection.analyze import PositionAnalyzer, load_model
-
-__all__ = ["PositionAnalyzer", "load_model"]
diff --git a/chesstransformer/introspection/analyze.py b/chesstransformer/introspection/analyze.py
deleted file mode 100644
index 9624f9e05f4b78ba9a72a54e3d14d6dd1d4209f5..0000000000000000000000000000000000000000
--- a/chesstransformer/introspection/analyze.py
+++ /dev/null
@@ -1,450 +0,0 @@
-"""Per-position introspection of a Pos2MoveV2 network.
-
-Everything here is read-only with respect to the model: attention maps are
-recomputed from the captured layer inputs using the module's own projections,
-so the numbers match what SDPA saw during the real forward pass.
-
-Produced per position:
- * policy over legal moves (probability, raw logit, action plane)
- * value head output, plus the value/policy read off *every* layer
- (logit lens) so you can see at which depth the model makes up its mind
- * full attention tensor (L, H, 67, 67)
- * attention rollout (Abnar & Zuidema 2020) over the layer stack
- * occlusion saliency: remove one piece, measure the swing in value and in
- the probability of the move the model wanted to play
- * 1-ply child values, i.e. what the value head says about the position
- *after* each legal move — the fastest way to spot policy/value disagreement
-"""
-
-from __future__ import annotations
-
-import json
-import math
-import zlib
-from base64 import b64encode
-from dataclasses import dataclass
-from pathlib import Path
-
-import chess
-import numpy as np
-import torch
-from safetensors import safe_open
-
-from chesstransformer.models.tokenizer import PostionTokenizer
-from chesstransformer.models.tokenizer.alphazero_move_encoder import (
- NUM_ACTION_PLANES,
- move_to_action_plane,
-)
-from chesstransformer.models.transformer.pos2move_v2 import Pos2MoveV2
-
-BOARD_SQUARES = 64
-CONTEXT_SIZE = 67
-STATE_TOKEN_NAMES = ["castling", "en_passant", "player"]
-
-
-def load_model(model_dir: str | Path, device: str = "cpu", use_ema: bool = False):
- """Load a Pos2MoveV2 checkpoint in fp32 (no compile, no bf16).
-
- Introspection wants clean numbers more than it wants speed, and the model
- is small enough that fp32 eager is instant either way.
- """
- model_dir = Path(model_dir)
-
- config_path = model_dir / "model_config.json"
- if not config_path.exists():
- config_path = model_dir.parent.parent / "model_config.json"
- if not config_path.exists():
- raise FileNotFoundError(f"No model_config.json found for {model_dir}")
- with open(config_path) as f:
- config = json.load(f)
-
- model = Pos2MoveV2(**config)
-
- weights = model_dir / "model.safetensors"
- if not weights.exists():
- raise FileNotFoundError(f"No model.safetensors in {model_dir}")
- with safe_open(str(weights), framework="pt", device="cpu") as f:
- state_dict = {k: f.get_tensor(k) for k in f.keys()}
- state_dict = {k.replace("_orig_mod.", ""): v for k, v in state_dict.items()}
- model.load_state_dict(state_dict)
-
- if use_ema:
- ema_path = model_dir / "ema_state.pt"
- if not ema_path.exists():
- raise FileNotFoundError(f"No ema_state.pt in {model_dir}")
- ema_state = torch.load(str(ema_path), map_location="cpu", weights_only=True)
- ema_state = {k.replace("_orig_mod.", ""): v for k, v in ema_state.items()}
- model.load_state_dict(ema_state, strict=False)
-
- model.eval()
- return model.float().to(device), config
-
-
-def encode_board(board: chess.Board, tokenizer: PostionTokenizer):
- """Same encoding the bots use: 64 piece tokens + castling/ep/player."""
- tokens = tokenizer.encode(board)
- castling = 0
- if board.has_kingside_castling_rights(chess.WHITE):
- castling |= 1
- if board.has_queenside_castling_rights(chess.WHITE):
- castling |= 2
- if board.has_kingside_castling_rights(chess.BLACK):
- castling |= 4
- if board.has_queenside_castling_rights(chess.BLACK):
- castling |= 8
- ep_file = chess.square_file(board.ep_square) if board.has_legal_en_passant() else 8
- return tokens, int(board.turn), castling, ep_file
-
-
-@dataclass
-class _Capture:
- """Residual stream and attention input for one layer."""
-
- attn_input: torch.Tensor # normed input handed to the attention module
- output: torch.Tensor # residual stream leaving the layer
-
-
-class PositionAnalyzer:
- def __init__(
- self,
- model: Pos2MoveV2,
- device: str = "cpu",
- max_moves: int = 64,
- saliency: bool = True,
- ):
- self.model = model
- self.device = device
- self.tokenizer = PostionTokenizer()
- self.max_moves = max_moves
- self.saliency = saliency
- self.n_layers = len(model.transformer_layers)
- self.n_heads = model.transformer_layers[0].attn.num_heads
-
- # ── raw forward helpers ────────────────────────────────────────────
- @torch.no_grad()
- def _forward_batch(self, boards: list[chess.Board]):
- enc = [encode_board(b, self.tokenizer) for b in boards]
- tokens = torch.tensor([e[0] for e in enc], dtype=torch.long, device=self.device)
- player = torch.tensor([e[1] for e in enc], dtype=torch.long, device=self.device)
- castling = torch.tensor([e[2] for e in enc], dtype=torch.long, device=self.device)
- ep = torch.tensor([e[3] for e in enc], dtype=torch.long, device=self.device)
- return self.model(tokens, player, castling, ep)
-
- @torch.no_grad()
- def _forward_tokens(self, tokens, player, castling, ep):
- return self.model(tokens, player, castling, ep)
-
- @torch.no_grad()
- def _forward_with_capture(self, board: chess.Board):
- """One forward pass, keeping every layer's input and output."""
- captures: list[_Capture] = []
- pending: dict[int, torch.Tensor] = {}
- handles = []
-
- def make_pre_hook(idx):
- def hook(_module, args):
- pending[idx] = args[0].detach()
-
- return hook
-
- def make_post_hook(idx):
- def hook(_module, _args, output):
- captures.append(_Capture(attn_input=pending[idx], output=output.detach()))
-
- return hook
-
- for i, layer in enumerate(self.model.transformer_layers):
- handles.append(layer.attn.register_forward_pre_hook(make_pre_hook(i)))
- handles.append(layer.register_forward_hook(make_post_hook(i)))
-
- try:
- tokens, player, castling, ep = encode_board(board, self.tokenizer)
- out = self._forward_tokens(
- torch.tensor([tokens], dtype=torch.long, device=self.device),
- torch.tensor([player], dtype=torch.long, device=self.device),
- torch.tensor([castling], dtype=torch.long, device=self.device),
- torch.tensor([ep], dtype=torch.long, device=self.device),
- )
- finally:
- for h in handles:
- h.remove()
-
- return out, captures
-
- @torch.no_grad()
- def _attention_probs(self, layer_idx: int, attn_input: torch.Tensor) -> torch.Tensor:
- """Recompute softmax attention for one layer: (H, 67, 67).
-
- Mirrors ChessGroupedQueryAttention.forward up to the softmax — SDPA
- adds the chess-relation bias to the *scaled* scores, so we do too.
- """
- attn = self.model.transformer_layers[layer_idx].attn
- x = attn_input
- b, t, _ = x.shape
-
- q = attn.q_proj(x).view(b, t, attn.num_heads, attn.head_dim).transpose(1, 2)
- k = attn.k_proj(x).view(b, t, attn.num_kv_groups, attn.head_dim).transpose(1, 2)
- q = attn.q_norm(q)
- k = attn.k_norm(k)
- k = k.repeat_interleave(attn.group_size, dim=1)
-
- scores = (q @ k.transpose(-2, -1)) / math.sqrt(attn.head_dim)
- scores = scores + attn.bias_table[:, attn.rel_idx].unsqueeze(0)
- return torch.softmax(scores, dim=-1)[0]
-
- # ── analysis ───────────────────────────────────────────────────────
- def analyze(self, board: chess.Board, include_attention: bool = True) -> dict:
- (move_logits, value), captures = self._forward_with_capture(board)
- logits = move_logits[0].float() # (64, 73)
- value = float(value[0].item())
-
- result: dict = {
- "fen": board.fen(),
- "turn": "w" if board.turn else "b",
- "fullmove": board.fullmove_number,
- "check": board.is_check(),
- "pieces": [
- (board.piece_at(sq).symbol() if board.piece_at(sq) else None)
- for sq in range(64)
- ],
- "value": value,
- }
-
- legal = list(board.legal_moves)
- result["legal_count"] = len(legal)
- if not legal:
- result["policy"] = {"moves": [], "entropy": 0.0, "top1_prob": 0.0}
- result["terminal"] = board.result()
- return result
-
- result.update(self._policy_block(board, logits, legal))
- result.update(self._depth_block(captures, logits, board, legal))
-
- if include_attention:
- result["attention"] = self._attention_block(captures)
-
- if self.saliency:
- result["saliency"] = self._saliency_block(board, value, result)
-
- return result
-
- def _policy_block(self, board: chess.Board, logits: torch.Tensor, legal) -> dict:
- planes = [
- move_to_action_plane(m.from_square, m.to_square, m.promotion) for m in legal
- ]
- idx = torch.tensor(
- [m.from_square * NUM_ACTION_PLANES + p for m, p in zip(legal, planes)],
- dtype=torch.long,
- )
- flat = logits.reshape(-1).cpu()
- legal_logits = flat[idx]
- probs = torch.softmax(legal_logits, dim=0)
-
- # Illegal mass: how much of the raw 64x73 softmax the model spends on
- # moves that do not exist. A cheap sanity signal for a broken head.
- full_probs = torch.softmax(flat, dim=0)
- legal_mass = float(full_probs[idx].sum())
-
- order = torch.argsort(probs, descending=True)
- moves = []
- for rank, i in enumerate(order.tolist()):
- m = legal[i]
- moves.append(
- {
- "uci": m.uci(),
- "san": board.san(m),
- "from": m.from_square,
- "to": m.to_square,
- "plane": planes[i],
- "prob": float(probs[i]),
- "logit": float(legal_logits[i]),
- "rank": rank,
- "capture": board.is_capture(m),
- "check": board.gives_check(m),
- }
- )
-
- p = probs[probs > 0]
- entropy = float(-(p * p.log()).sum())
- return {
- "policy": {
- "moves": moves[: self.max_moves],
- "n_moves": len(moves),
- "entropy": entropy,
- "max_entropy": math.log(len(legal)),
- "top1_prob": moves[0]["prob"],
- "legal_mass": legal_mass,
- }
- }
-
- @torch.no_grad()
- def _depth_block(self, captures, final_logits, board, legal) -> dict:
- """Logit lens: decode value + policy from every layer's residual."""
- norm = self.model.final_norm
- value_by_layer = []
- top_by_layer = []
- write_norms = []
-
- planes = [
- move_to_action_plane(m.from_square, m.to_square, m.promotion) for m in legal
- ]
- idx = torch.tensor(
- [m.from_square * NUM_ACTION_PLANES + p for m, p in zip(legal, planes)],
- dtype=torch.long,
- device=self.device,
- )
- final_probs = torch.softmax(final_logits.reshape(-1)[idx.cpu()], dim=0)
- final_top = int(torch.argmax(final_probs))
-
- prev = None
- for cap in captures:
- x = norm(cap.output)
- state = x[:, -3:, :].mean(dim=1)
- value_by_layer.append(float(self.model.value_head(state)[0].item()))
-
- lens_logits = self.model.move_head(x[:, :64, :])[0].reshape(-1)[idx]
- lens_probs = torch.softmax(lens_logits.float(), dim=0)
- best = int(torch.argmax(lens_probs))
- top_by_layer.append(
- {
- "uci": legal[best].uci(),
- "san": board.san(legal[best]),
- "prob": float(lens_probs[best]),
- "final_move_prob": float(lens_probs[final_top]),
- "agrees": best == final_top,
- }
- )
-
- out = cap.output
- write_norms.append(
- float((out - prev).norm().item()) if prev is not None else float(out.norm().item())
- )
- prev = out
-
- return {
- "depth": {
- "value_by_layer": value_by_layer,
- "top_by_layer": top_by_layer,
- "residual_write_norm": write_norms,
- }
- }
-
- @torch.no_grad()
- def _attention_block(self, captures) -> dict:
- """Full (L, H, 67, 67) attention, u8-quantised per row, deflated."""
- maps = []
- for i, cap in enumerate(captures):
- maps.append(self._attention_probs(i, cap.attn_input).cpu())
- attn = torch.stack(maps) # (L, H, 67, 67)
-
- rollout = self._rollout(attn)
-
- # u8 per row, companded through a square root: attention rows are long
- # tails of small weights, and linear u8 would quantise those to 20-30%
- # relative error. sqrt spreads the error evenly across magnitudes.
- row_max = attn.amax(dim=-1, keepdim=True).clamp(min=1e-9)
- quant = ((attn / row_max).sqrt() * 255).round().clamp(0, 255).to(torch.uint8).numpy()
- payload = zlib.compress(quant.tobytes(), 6)
-
- return {
- "shape": list(attn.shape),
- "scale": row_max.squeeze(-1).flatten().numpy().astype(np.float32).tolist(),
- "data": b64encode(payload).decode("ascii"),
- "rollout": [round(v, 5) for v in rollout.tolist()],
- }
-
- @staticmethod
- def _rollout(attn: torch.Tensor) -> torch.Tensor:
- """Attention rollout: how much of each token the output actually sees.
-
- Head-averaged, residual-corrected (0.5*A + 0.5*I), multiplied through
- the stack, then read from the three game-state token rows — those are
- what the value head consumes.
- """
- a = attn.mean(dim=1) # (L, T, T)
- eye = torch.eye(a.shape[-1])
- joint = None
- for layer in a:
- m = 0.5 * layer + 0.5 * eye
- m = m / m.sum(dim=-1, keepdim=True)
- joint = m if joint is None else m @ joint
- return joint[-3:, :].mean(dim=0)
-
- @torch.no_grad()
- def _saliency_block(self, board: chess.Board, base_value: float, result: dict) -> dict:
- """Occlusion: delete one piece at a time, measure what moves.
-
- Kings are left in place (an occluded king is not a chess position the
- model has ever seen). Everything runs as one batch.
- """
- top = result["policy"]["moves"][0]
- top_idx = top["from"] * NUM_ACTION_PLANES + top["plane"]
-
- squares = [
- sq
- for sq in range(64)
- if board.piece_at(sq) is not None and board.piece_at(sq).piece_type != chess.KING
- ]
- value_delta = [0.0] * 64
- policy_delta = [0.0] * 64
- if not squares:
- return {"value_delta": value_delta, "policy_delta": policy_delta, "squares": []}
-
- tokens, player, castling, ep = encode_board(board, self.tokenizer)
- base = torch.tensor(tokens, dtype=torch.long).repeat(len(squares), 1)
- for row, sq in enumerate(squares):
- base[row, sq] = 0
- n = len(squares)
- move_logits, values = self._forward_tokens(
- base.to(self.device),
- torch.full((n,), player, dtype=torch.long, device=self.device),
- torch.full((n,), castling, dtype=torch.long, device=self.device),
- torch.full((n,), ep, dtype=torch.long, device=self.device),
- )
-
- # Probability of the top move under the *full* 64x73 softmax: occlusion
- # changes the legal move set, so a legal-masked comparison would not be
- # measuring the same quantity across rows.
- flat = move_logits.float().reshape(n, -1).cpu()
- probs = torch.softmax(flat, dim=1)[:, top_idx]
-
- ref_logits, _ = self._forward_batch([board])
- ref_prob = float(torch.softmax(ref_logits[0].float().reshape(-1), dim=0)[top_idx])
-
- for row, sq in enumerate(squares):
- value_delta[sq] = float(values[row].item()) - base_value
- policy_delta[sq] = float(probs[row]) - ref_prob
-
- return {
- "value_delta": [round(v, 5) for v in value_delta],
- "policy_delta": [round(v, 5) for v in policy_delta],
- "squares": squares,
- "top_move": top["uci"],
- "ref_prob": ref_prob,
- }
-
- @torch.no_grad()
- def child_values(self, board: chess.Board, moves: list[dict]) -> None:
- """Fill in the value of the position *after* each candidate move.
-
- Stored from the mover's point of view (the raw head is side-to-move
- POV, so it gets negated) — directly comparable to ``value``.
- """
- if not moves:
- return
- boards = []
- terminal = {}
- for i, m in enumerate(moves):
- b = board.copy(stack=False)
- b.push(chess.Move.from_uci(m["uci"]))
- if b.is_checkmate():
- terminal[i] = 1.0
- elif b.is_stalemate() or b.is_insufficient_material():
- terminal[i] = 0.0
- boards.append(b)
-
- _, values = self._forward_batch(boards)
- for i, m in enumerate(moves):
- m["child_value"] = (
- terminal[i] if i in terminal else -float(values[i].item())
- )
diff --git a/chesstransformer/introspection/viewer.html b/chesstransformer/introspection/viewer.html
deleted file mode 100644
index 6721dc955379954e8be105a9dc56ecb45d94abae..0000000000000000000000000000000000000000
--- a/chesstransformer/introspection/viewer.html
+++ /dev/null
@@ -1,860 +0,0 @@
-Pos2MoveV2 Glass Box
-
-
-
-
-
-
-
-
-
-
Model introspectionGlass Box
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
Legal moves — policy vs. value
-
-
-
-
-
Where the answer forms
-
-
-
-
-
-
-
- Attention
-
-
-
-
-
-
-
-
-
-
-
-
-
0max
-
Rows are queries (the token doing the looking), columns are keys. The last three rows/columns are the castling, en-passant and side-to-move tokens. Click a cell to select that square on the board.
-
-
-
All heads, layer click to select
-
-
Relation bias — head learned, position-independent
-
-
-
-
-
-
- How to read this / how it was made
-
-
-
Policy vs. value disagreement
-
Every legal move carries its policy probability and the child value — the value head's verdict on the position after that move, from the mover's side. A move the policy loves but the value dislikes (or vice versa) is where search and network fight each other.
-
-
-
Where the answer forms
-
The value and policy are decoded from every layer's residual stream through the final norm and the real heads (a logit lens). If the top move is already locked in by layer 6 and nothing moves after, the last ten layers are not doing policy work.
-
-
-
Occlusion saliency
-
Each non-king piece is deleted in turn and the position re-evaluated. The square shows the swing in value, or in the probability of the move the model wanted. Big swings mark the pieces the verdict actually rests on.
-
-
-
Attention and rollout
-
Attention is recomputed from each layer's captured input using the model's own projections and its chess-relation bias, then checked against the real forward pass. Rollout multiplies the head-averaged, residual-corrected maps through the stack and reads the game-state token rows.
-
-
-
-
-
-
-
-
-
-
diff --git a/chesstransformer/models/__init__.py b/chesstransformer/models/__init__.py
deleted file mode 100644
index 5f1c61c6771a75a9b8f3c160f8125e14d6894905..0000000000000000000000000000000000000000
--- a/chesstransformer/models/__init__.py
+++ /dev/null
@@ -1 +0,0 @@
-"""This module contains all the models."""
diff --git a/chesstransformer/models/tokenizer/__init__.py b/chesstransformer/models/tokenizer/__init__.py
deleted file mode 100644
index d8441ec6110d9a064a3db5a735a46057825307b6..0000000000000000000000000000000000000000
--- a/chesstransformer/models/tokenizer/__init__.py
+++ /dev/null
@@ -1,93 +0,0 @@
-from tokenizers import Tokenizer
-from tokenizers import models, pre_tokenizers, trainers, processors, normalizers
-from .move_tokenizer import MoveTokenizer
-from .position_tokenizer import PostionTokenizer
-
-
-def create_bpe_tokenizer(
- vocab_size: int = 2000,
-) -> tuple[Tokenizer, trainers.BpeTrainer]:
- """
- Create a BPE tokenizer for chess moves.
-
- The tokenizer uses Byte-Pair Encoding (BPE) with ByteLevel pre-tokenization
- to handle the unique structure of chess moves. The token is used to
- explicitly separate each move, ensuring clear move boundaries.
-
- Special tokens included:
- - : Padding token
- - : Start of sequence token
- - : End of sequence token
- - : Move separator token
- - <1-0>: White wins
- - <0-1>: Black wins
- - <1/2-1/2>: Draw
- - : Unknown token
-
- Args:
- vocab_size: Size of the vocabulary (default: 2000)
-
- Returns:
- A Tokenizer object and trainer
- """
-
- # Initialize a tokenizer with BPE model
- encoder = Tokenizer(models.BPE(unk_token=""))
- encoder.normalizer = normalizers.Sequence([]) # No normalization needed for chess moves
-
- # remove the white space pre-tokenizer
- encoder.pre_tokenizer = pre_tokenizers.Whitespace()
-
- # Setup trainer with special tokens, including space
- trainer = trainers.BpeTrainer(
- vocab_size=vocab_size,
- special_tokens=[
- "",
- "",
- "",
- "<1-0>",
- "<0-1>",
- "<1/2-1/2>",
- "",
- "",
- ],
- show_progress=True,
- min_frequency=2,
- initial_alphabet=[
- "a",
- "b",
- "c",
- "d",
- "e",
- "f",
- "g",
- "h",
- "1",
- "2",
- "3",
- "4",
- "5",
- "6",
- "7",
- "8",
- "q",
- "k",
- "r",
- "b",
- "n", # Piece notations
- ],
- )
-
- # Add post-processing to add special tokens
- encoder.post_processor = processors.TemplateProcessing(
- single=" $A ",
- special_tokens=[
- ("", 1),
- ("", 2),
- ],
- )
-
- return encoder, trainer
-
-
-__all__ = ["create_bpe_tokenizer", "MoveTokenizer", "PostionTokenizer"]
diff --git a/chesstransformer/models/tokenizer/alphazero_move_encoder.py b/chesstransformer/models/tokenizer/alphazero_move_encoder.py
deleted file mode 100644
index 69fda46f509655dbe6978ce8417a4dd3597de0cc..0000000000000000000000000000000000000000
--- a/chesstransformer/models/tokenizer/alphazero_move_encoder.py
+++ /dev/null
@@ -1,127 +0,0 @@
-"""AlphaZero-style action encoding for chess moves.
-
-Encodes moves as (from_square, action_plane) pairs using the 8×8×73
-representation from the AlphaZero paper (Silver et al., 2018).
-
-The 73 action planes decompose as:
- 0–55: Queen-type moves — 8 directions × 7 distances
- 56–63: Knight moves — 8 L-shaped jumps
- 64–72: Underpromotions — 3 directions × 3 piece types (knight/bishop/rook)
-
-Queen promotions are encoded as regular queen-type moves (distance-1 to the
-last rank). Underpromotions get their own dedicated planes.
-"""
-
-import chess
-import torch
-
-NUM_ACTION_PLANES = 73
-
-# Direction vectors (file_delta, rank_delta) for queen-type moves
-QUEEN_DIRECTIONS = [
- (0, 1), # N
- (1, 1), # NE
- (1, 0), # E
- (1, -1), # SE
- (0, -1), # S
- (-1, -1), # SW
- (-1, 0), # W
- (-1, 1), # NW
-]
-
-# Knight jump offsets (file_delta, rank_delta)
-KNIGHT_JUMPS = [
- (1, 2), (2, 1), (2, -1), (1, -2),
- (-1, -2), (-2, -1), (-2, 1), (-1, 2),
-]
-
-# Underpromotion piece types (queen promotion is a regular queen-type move)
-UNDERPROMO_PIECES = [chess.KNIGHT, chess.BISHOP, chess.ROOK]
-
-# Pre-compute knight jump lookup for O(1) detection
-_KNIGHT_JUMP_INDEX = {(df, dr): i for i, (df, dr) in enumerate(KNIGHT_JUMPS)}
-
-
-def move_to_action_plane(from_sq: int, to_sq: int, promotion: int | None = None) -> int:
- """Convert a chess move to its action plane index (0–72).
-
- Args:
- from_sq: Source square (0–63, python-chess ordering).
- to_sq: Destination square (0–63).
- promotion: Promotion piece type (chess.QUEEN, chess.KNIGHT, etc.) or None.
-
- Returns:
- Action plane index in [0, 72].
- """
- df = (to_sq % 8) - (from_sq % 8)
- dr = (to_sq // 8) - (from_sq // 8)
-
- # ── Underpromotion ───────────────────────────────────────────────────
- if promotion is not None and promotion != chess.QUEEN:
- # Direction: file delta ∈ {-1, 0, +1} → index 0, 1, 2
- dir_idx = df + 1
- piece_idx = UNDERPROMO_PIECES.index(promotion)
- return 64 + dir_idx * 3 + piece_idx
-
- # ── Knight move ──────────────────────────────────────────────────────
- knight_idx = _KNIGHT_JUMP_INDEX.get((df, dr))
- if knight_idx is not None:
- return 56 + knight_idx
-
- # ── Queen-type move (includes queen promotions) ──────────────────────
- distance = max(abs(df), abs(dr))
- dir_f = (df // distance) if df != 0 else 0
- dir_r = (dr // distance) if dr != 0 else 0
- dir_idx = QUEEN_DIRECTIONS.index((dir_f, dir_r))
- return dir_idx * 7 + (distance - 1)
-
-
-def action_to_move(from_sq: int, plane: int) -> tuple[int, int | None]:
- """Decode an action plane back to (to_square, promotion_piece | None).
-
- For queen-type moves landing on promotion ranks, the caller must check
- whether the source piece is a pawn and add ``chess.QUEEN`` promotion.
-
- Args:
- from_sq: Source square (0–63).
- plane: Action plane index (0–72).
-
- Returns:
- (to_sq, promotion_piece) — promotion_piece is set only for underpromotions.
- """
- from_file = from_sq % 8
- from_rank = from_sq // 8
-
- if plane < 56:
- # Queen-type move
- dir_idx = plane // 7
- distance = plane % 7 + 1
- df, dr = QUEEN_DIRECTIONS[dir_idx]
- to_file = from_file + df * distance
- to_rank = from_rank + dr * distance
- return to_rank * 8 + to_file, None
-
- if plane < 64:
- # Knight move
- df, dr = KNIGHT_JUMPS[plane - 56]
- to_file = from_file + df
- to_rank = from_rank + dr
- return to_rank * 8 + to_file, None
-
- # Underpromotion
- promo_idx = plane - 64
- dir_idx = promo_idx // 3
- piece_idx = promo_idx % 3
- df = dir_idx - 1 # 0→-1, 1→0, 2→+1
- to_file = from_file + df
- to_rank = 7 if from_rank == 6 else 0 # white promotes to rank 8, black to rank 1
- return to_rank * 8 + to_file, UNDERPROMO_PIECES[piece_idx]
-
-
-def build_legal_moves_planes(board: chess.Board) -> torch.Tensor:
- """Build a (64, 73) boolean mask of legal moves in action-plane format."""
- mask = torch.zeros(64, NUM_ACTION_PLANES, dtype=torch.bool)
- for move in board.legal_moves:
- plane = move_to_action_plane(move.from_square, move.to_square, move.promotion)
- mask[move.from_square, plane] = True
- return mask
diff --git a/chesstransformer/models/tokenizer/move_tokenizer.py b/chesstransformer/models/tokenizer/move_tokenizer.py
deleted file mode 100644
index 63907ff5b5057734920062c8065c8ddde7b8450a..0000000000000000000000000000000000000000
--- a/chesstransformer/models/tokenizer/move_tokenizer.py
+++ /dev/null
@@ -1,38 +0,0 @@
-import json
-from pathlib import Path
-
-DATA_DIR = Path(__file__).parents[2] / "data"
-
-
-class MoveTokenizer:
- def __init__(self, vocab_path=DATA_DIR / "tokenizer_models/chess_moves_vocab.json"):
- with open(vocab_path, "r") as f:
- self.vocab = json.load(f)
- self.inv_vocab = {v: k for k, v in self.vocab.items()}
-
- def encode(self, move: str) -> int:
- """Encode a chess move string into a token ID.
- Args:
- move: A string representing a chess move in UCI format (e.g., "e2e4", "g1f3").
- Returns:
- An integer token ID representing the move.
- """
- if move not in self.vocab:
- raise ValueError(f"Move '{move}' not in vocabulary.")
- return self.vocab[move]
-
- def decode(self, token_id: int) -> str:
- """Decode a token ID back into a chess move string.
- Args:
- token_id: An integer token ID representing the move.
- Returns:
- A string representing a chess move in UCI format.
- """
- if token_id not in self.inv_vocab:
- raise ValueError(f"Token ID '{token_id}' not in vocabulary.")
- return self.inv_vocab[token_id]
-
- @property
- def vocab_size(self) -> int:
- """Return the size of the vocabulary."""
- return len(self.vocab)
diff --git a/chesstransformer/models/tokenizer/position_tokenizer.py b/chesstransformer/models/tokenizer/position_tokenizer.py
deleted file mode 100644
index 7ef254ca372c2484b0b45bf52000a5b7a809b727..0000000000000000000000000000000000000000
--- a/chesstransformer/models/tokenizer/position_tokenizer.py
+++ /dev/null
@@ -1,88 +0,0 @@
-import chess
-
-_PIECE_TOKEN = {
- (chess.PAWN, chess.WHITE): 1, (chess.KNIGHT, chess.WHITE): 2,
- (chess.BISHOP, chess.WHITE): 3, (chess.ROOK, chess.WHITE): 4,
- (chess.QUEEN, chess.WHITE): 5, (chess.KING, chess.WHITE): 6,
- (chess.PAWN, chess.BLACK): 7, (chess.KNIGHT, chess.BLACK): 8,
- (chess.BISHOP, chess.BLACK): 9, (chess.ROOK, chess.BLACK): 10,
- (chess.QUEEN, chess.BLACK): 11, (chess.KING, chess.BLACK): 12,
-}
-
-
-class PostionTokenizer:
- """A simple tokenizer for chess board positions.
- Each square on the board is represented by a token ID based on the piece occupying it.
- Empty squares are represented by the token ID 0.
- The board is represented as an 8x8 grid, flattened into a list of 64 tokens.
- """
-
- def __init__(self):
- self.vocab = {
- "P": 1,
- "N": 2,
- "B": 3,
- "R": 4,
- "Q": 5,
- "K": 6,
- "p": 7,
- "n": 8,
- "b": 9,
- "r": 10,
- "q": 11,
- "k": 12,
- ".": 0,
- }
- self.inv_vocab = {v: k for k, v in self.vocab.items()}
-
- def encode(self, board: chess.Board) -> list[int]:
- """Encode a chess.Board object into a list of token IDs.
- The board is represented as an 8x8 grid, flattened into a list of 64 tokens.
-
- Args:
- board: A chess.Board object
-
- Returns:
- List of token IDs representing the board position
- """
- result = [0] * 64
- for sq, piece in board.piece_map().items():
- result[sq] = _PIECE_TOKEN[(piece.piece_type, piece.color)]
- return result
-
- def decode(self, token_ids: list[int]) -> chess.Board:
- """Decode a list of token IDs back into a chess board string.
- Args:
- token_ids: List of token IDs representing the board position
- Returns:
- A chess.Board object
- """
- chars = [self.inv_vocab[token_id] for token_id in token_ids]
- rows = ["".join(chars[i * 8 : (i + 1) * 8]) for i in range(8)][::-1] # Reverse to get the correct order
- board_str = "\n".join(rows)
- board = self._ascii2board(board_str)
- return board
-
- def _ascii2board(self, ascii_board: str) -> chess.Board:
- """Convert an ASCII board representation back to a chess.Board object.
- Args:
- ascii_board: String representation of the board (8 lines of 8 characters)
- Returns:
- A chess.Board object
- """
- board = chess.Board.empty()
- rows = ascii_board.split("\n")
- for rank in range(8):
- file = 0
- for char in rows[7 - rank]: # Reverse the order of ranks
- if char in self.vocab and char != ".":
- square = chess.square(file, rank)
- piece = chess.Piece.from_symbol(char)
- board.set_piece_at(square, piece)
- file += 1
- return board
-
- @property
- def vocab_size(self) -> int:
- """Return the size of the vocabulary."""
- return len(self.vocab)
diff --git a/chesstransformer/models/tokenizer/train_bpe_tokenizer.py b/chesstransformer/models/tokenizer/train_bpe_tokenizer.py
deleted file mode 100644
index 5212e8ddad45b66cf22bd812af6fc4f6335bd5fe..0000000000000000000000000000000000000000
--- a/chesstransformer/models/tokenizer/train_bpe_tokenizer.py
+++ /dev/null
@@ -1,94 +0,0 @@
-from pathlib import Path
-import argparse
-
-from chesstransformer.models.tokenizer import create_bpe_tokenizer
-
-
-def arg_parser():
- parser = argparse.ArgumentParser(description="Train a BPE tokenizer for chess moves.")
- parser.add_argument(
- "--vocab_size",
- type=int,
- default=2000,
- help="Size of the vocabulary (default: 2000)",
- )
- parser.add_argument(
- "--output_dir",
- type=str,
- help="Directory to save the trained tokenizer",
- required=True,
- )
- parser.add_argument(
- "--dataset_path",
- type=str,
- help="Path to the lichess dataset file (PGN format)",
- required=True,
- )
-
- parser.add_argument(
- "--num_samples",
- type=int,
- default=100000,
- help="Number of games to sample for training (default: 100000)",
- )
-
- parser.add_argument(
- "--notation",
- type=str,
- choices=["uci", "san"],
- default="uci",
- help="Notation format for moves (default: uci)",
- )
-
- args = parser.parse_args()
- return args
-
-
-def main():
- args = arg_parser()
- vocab_size = args.vocab_size
- output_dir = Path(args.output_dir)
- dataset_path = Path(args.dataset_path)
- num_samples = args.num_samples
- notation = args.notation
-
- output_dir.mkdir(parents=True, exist_ok=True)
-
- print(f"Creating BPE tokenizer with vocab size {vocab_size}...")
- tokenizer, trainer = create_bpe_tokenizer(vocab_size=vocab_size)
-
- print(f"Loading dataset from {dataset_path}...")
- from chesstransformer.datasets.lichess import extract_games_optimized
-
- games = extract_games_optimized(dataset_path, num_games=num_samples)
-
- print(f"Extracting moves in {notation} notation...")
- print("Using token to separate moves...")
- if notation == "uci":
- move_sequences = ("".join([move.uci() for move in game.mainline_moves()]) for game in games)
- elif notation == "san":
- move_sequences = (
- "".join(
- [
- (lambda board, move: (board.san(move), board.push(move))[0])(board, move)
- for board in [game.board()]
- for move in game.mainline_moves()
- ]
- )
- for game in games
- )
- else:
- raise ValueError(f"Unsupported notation: {notation}")
-
- print(f"Training tokenizer on {num_samples} games...")
- tokenizer.train_from_iterator(move_sequences, trainer=trainer)
-
- tokenizer_path = output_dir / f"bpe_tokenizer_vocab{vocab_size}.json"
- print(f"Saving tokenizer to {tokenizer_path}...")
- tokenizer.save(str(tokenizer_path))
-
- print("Tokenizer training complete.")
-
-
-if __name__ == "__main__":
- main()
diff --git a/chesstransformer/models/transformer/__init__.py b/chesstransformer/models/transformer/__init__.py
deleted file mode 100644
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000
diff --git a/chesstransformer/models/transformer/attention.py b/chesstransformer/models/transformer/attention.py
deleted file mode 100644
index 9ab0951b2f45417c9b8c04401acad5b4830153a5..0000000000000000000000000000000000000000
--- a/chesstransformer/models/transformer/attention.py
+++ /dev/null
@@ -1,228 +0,0 @@
-"""This module implements attention mechanisms for LLMs."""
-
-import torch
-import torch.nn as nn
-import torch.nn.functional as F
-
-from chesstransformer.models.transformer.rope import RoPEEmbedding
-
-
-class MultiHeadAttention(nn.Module):
- def __init__(
- self, d_in, d_out, num_heads, context_length, qkv_bias=False, dropout=0.0, mask_future=True, apply_rope=False
- ):
- super().__init__()
- assert d_out % num_heads == 0, "d_out must be divisible by num_heads"
- self.num_heads = num_heads
- self.d_out = d_out
- self.d_head = d_out // num_heads
- self.mask_future = mask_future
- self.W_query = nn.Linear(d_in, d_out, bias=qkv_bias)
- self.W_key = nn.Linear(d_in, d_out, bias=qkv_bias)
- self.W_value = nn.Linear(d_in, d_out, bias=qkv_bias)
- self.out_proj = nn.Linear(d_out, d_out)
- self.dropout = nn.Dropout(dropout)
- self.apply_rope = apply_rope
-
- if self.apply_rope:
- self.rope = RoPEEmbedding(dim=self.d_head, max_position_embeddings=context_length)
-
- self.register_buffer("mask", torch.triu(torch.ones(context_length, context_length), diagonal=1))
-
- def forward(self, x):
- """
- Computes multi-head self-attention.
-
- Args:
- x: Input tensor of shape (batch_size, seq_len, d_in)
-
- Returns:
- Output tensor of shape (batch_size, seq_len, d_out)
- """
- bz, nb_tockens, _ = x.shape
-
- # Linear projections
- keys = self.W_key(x)
- values = self.W_value(x)
- queries = self.W_query(x)
-
- # Reshape for multi-head attention
- keys = keys.view(bz, nb_tockens, self.num_heads, self.d_head).transpose(1, 2)
- values = values.view(bz, nb_tockens, self.num_heads, self.d_head).transpose(1, 2)
- queries = queries.view(bz, nb_tockens, self.num_heads, self.d_head).transpose(1, 2)
-
- if self.apply_rope:
- keys = self.rope(keys)
- queries = self.rope(values)
-
- # Compute attention scores
- attention_scores = queries @ keys.transpose(2, 3) # (batch_size, num_heads, seq_len, seq_len)
-
- if self.mask_future:
- # Apply mask
- mask_bool = self.mask.bool()[:nb_tockens, :nb_tockens]
- attention_scores = attention_scores.masked_fill(mask_bool, -torch.inf)
-
- # Compute attention weights
- attention_weights = F.softmax(attention_scores / keys.shape[-1] ** 0.5, dim=-1)
- attention_weights = self.dropout(attention_weights)
-
- # apply attention weights to values
- context_vec = (attention_weights @ values).transpose(1, 2).contiguous()
-
- # Merge heads
- context_vec = context_vec.view(bz, nb_tockens, self.d_out)
- context_vec = self.out_proj(context_vec)
- return context_vec
-
-
-class PyTorchMultiHeadAttention(nn.Module):
- def __init__(self, d_in, d_out, num_heads, dropout=0.0, qkv_bias=False, mask_future=True, apply_rope=False):
- super().__init__()
-
- assert d_out % num_heads == 0, "d_out is indivisible by num_heads"
-
- self.num_heads = num_heads
- self.head_dim = d_out // num_heads
- self.d_out = d_out
- self.mask_future = mask_future
- self.apply_rope = apply_rope
-
- self.qkv = nn.Linear(d_in, 3 * d_out, bias=qkv_bias)
- self.proj = nn.Linear(d_out, d_out)
- self.dropout = dropout
- self.rope = None # Placeholder for RoPE, if needed
- if self.apply_rope:
- self.rope = RoPEEmbedding(dim=self.head_dim)
-
- def forward(self, x):
- batch_size, num_tokens, embed_dim = x.shape
-
- # (b, num_tokens, embed_dim) --> (b, num_tokens, 3 * embed_dim)
- qkv = self.qkv(x)
-
- # (b, num_tokens, 3 * embed_dim) --> (b, num_tokens, 3, num_heads, head_dim)
- qkv = qkv.view(batch_size, num_tokens, 3, self.num_heads, self.head_dim)
-
- # (b, num_tokens, 3, num_heads, head_dim) --> (3, b, num_heads, num_tokens, head_dim)
- qkv = qkv.permute(2, 0, 3, 1, 4)
-
- # (3, b, num_heads, num_tokens, head_dim) -> 3 times (b, num_heads, num_tokens, head_dim)
- queries, keys, values = qkv
-
- if self.apply_rope and self.rope is not None:
- keys = self.rope(keys)
- queries = self.rope(queries)
-
- use_dropout = 0.0 if not self.training else self.dropout
-
- context_vec = nn.functional.scaled_dot_product_attention(
- queries, keys, values, attn_mask=None, dropout_p=use_dropout, is_causal=self.mask_future
- )
-
- # Combine heads, where self.d_out = self.num_heads * self.head_dim
- context_vec = context_vec.transpose(1, 2).contiguous().view(batch_size, num_tokens, self.d_out)
-
- context_vec = self.proj(context_vec)
-
- return context_vec
-
-
-class GroupedQueryAttention(nn.Module):
- """
- Multi-head self-attention with grouped keys and values.
- This implementation allows multiple attention heads to share the same keys and values,
- reducing the number of parameters and computational cost.
-
- It works by dividing the attention heads into groups, where each group shares a set of keys and values.
-
- Example:
- If there are 8 attention heads and 4 key-value groups, then each group will be shared by 2 attention heads.
-
- Args:
- d_in: Input feature dimension
- d_out: Output feature dimension
- num_heads: Number of attention heads
- num_kv_groups: Number of key-value groups (must divide num_heads)
- dropout: Dropout probability
- qkv_bias: Whether to include bias terms in the linear projections
- mask_future: Whether to apply causal masking to prevent attention to future tokens
- apply_rope: Whether to apply Rotary Positional Embeddings (RoPE)
- Returns:
- Output tensor of shape (batch_size, seq_len, d_out)
- """
-
- def __init__(
- self, d_in, d_out, num_heads, num_kv_groups, dropout=0.0, qkv_bias=False, mask_future=True, apply_rope=False
- ):
- super().__init__()
-
- assert d_out % num_heads == 0, "d_out is indivisible by num_heads"
- assert num_heads % num_kv_groups == 0, "num_heads is indivisible by num_kv_groups"
-
- self.num_heads = num_heads
- self.num_kv_groups = num_kv_groups
- self.head_dim = d_out // num_heads
- self.d_out = d_out
-
- self.q_proj = nn.Linear(d_in, d_out, bias=qkv_bias)
- self.k_proj = nn.Linear(d_in, num_kv_groups * self.head_dim, bias=qkv_bias)
- self.v_proj = nn.Linear(d_in, num_kv_groups * self.head_dim, bias=qkv_bias)
- self.group_size = num_heads // num_kv_groups
- self.num_kv_groups = num_kv_groups
-
- self.proj = nn.Linear(d_out, d_out, bias=False)
- self.dropout = nn.Dropout(dropout)
-
- self.mask_future = mask_future
- self.apply_rope = apply_rope
- if self.apply_rope:
- self.rope = RoPEEmbedding(dim=self.head_dim)
-
- def forward(self, x):
- batch_size, num_tokens, embed_dim = x.shape
-
- queries = self.q_proj(x)
- keys = self.k_proj(x)
- values = self.v_proj(x)
-
- queries = queries.view(batch_size, num_tokens, self.num_heads, self.head_dim).transpose(1, 2)
- keys_new = keys.view(batch_size, num_tokens, self.num_kv_groups, self.head_dim).transpose(1, 2)
- values_new = values.view(batch_size, num_tokens, self.num_kv_groups, self.head_dim).transpose(1, 2)
-
- keys = keys_new.repeat_interleave(self.group_size, dim=1)
- values = values_new.repeat_interleave(self.group_size, dim=1)
-
- if self.apply_rope:
- keys = self.rope(keys)
- queries = self.rope(queries)
-
- use_dropout = 0.0 if not self.training else self.dropout.p
-
- context_vec = F.scaled_dot_product_attention(
- queries, keys, values, attn_mask=None, dropout_p=use_dropout, is_causal=self.mask_future
- )
-
- context_vec = context_vec.transpose(1, 2).contiguous()
- context_vec = context_vec.view(batch_size, num_tokens, self.d_out)
- context_vec = self.proj(context_vec)
-
- return context_vec
-
-
-if __name__ == "__main__":
- # Simple test
- batch_size = 2
- seq_len = 16
- d_in = 64
- d_out = 64
- num_heads = 8
- num_kv_groups = 4
-
- x = torch.randn(batch_size, seq_len, d_in)
-
- attn = GroupedQueryAttention(d_in, d_out, num_heads, num_kv_groups, apply_rope=True)
- output = attn(x)
-
- print("Input shape:", x.shape)
- print("Output shape:", output.shape)
diff --git a/chesstransformer/models/transformer/gpt.py b/chesstransformer/models/transformer/gpt.py
deleted file mode 100644
index f46b5607b8674f4d5aae32d450545b57e5066985..0000000000000000000000000000000000000000
--- a/chesstransformer/models/transformer/gpt.py
+++ /dev/null
@@ -1,79 +0,0 @@
-"""This module implements GPT model architecture."""
-
-import torch
-import torch.nn as nn
-from ..transformer.attention import MultiHeadAttention
-from ..transformer.utils import LayerNorm, GELU
-
-
-class FeedForward(nn.Module):
- def __init__(self, cfg):
- super().__init__()
- self.layers = nn.Sequential(
- nn.Linear(cfg["embed_dim"], 4 * cfg["embed_dim"]),
- GELU(),
- nn.Linear(4 * cfg["embed_dim"], cfg["embed_dim"]),
- )
-
- def forward(self, x):
- return self.layers(x)
-
-
-class TransformerBlock(nn.Module):
- def __init__(self, cfg):
- super().__init__()
- self.att = MultiHeadAttention(
- d_in=cfg["embed_dim"],
- d_out=cfg["embed_dim"],
- num_heads=cfg["num_heads"],
- context_length=cfg["context_size"],
- dropout=cfg["dropout"],
- qkv_bias=cfg["qkv_bias"],
- )
- self.feed_forward = FeedForward(cfg)
- self.norm1 = LayerNorm(cfg["embed_dim"])
- self.norm2 = LayerNorm(cfg["embed_dim"])
- self.dropout_shortcut = nn.Dropout(cfg["dropout"])
-
- def forward(self, x):
- shortcut = x
- x = self.norm1(x)
- x = self.att(x)
- x = self.dropout_shortcut(x)
- x = x + shortcut
-
- shortcut = x
- x = self.norm2(x)
- x = self.feed_forward(x)
- x = self.dropout_shortcut(x)
- x = x + shortcut
-
- return x
-
-
-class GPT2Model(nn.Module):
- def __init__(self, cfg):
- super().__init__()
- self.token_embedding = nn.Embedding(cfg["vocab_size"], cfg["embed_dim"])
- self.position_embedding = nn.Embedding(cfg["context_size"], cfg["embed_dim"])
- self.dropout_embed = nn.Dropout(cfg["dropout"])
- self.transfomer_blocks = nn.Sequential(*[TransformerBlock(cfg) for _ in range(cfg["num_layers"])])
- self.final_norm = LayerNorm(cfg["embed_dim"])
- self.lm_head = nn.Linear(cfg["embed_dim"], cfg["vocab_size"], bias=False)
-
- def forward(self, x):
- bz, sq_len = x.size()
-
- token_emb = self.token_embedding(x) # (bz, sq_len, embed_dim)
-
- positions = torch.arange(sq_len, device=x.device)
- pos_emb = self.position_embedding(positions) # (sq_len, embed_dim)
-
- x = token_emb + pos_emb # (bz, sq_len, embed_dim)
- x = self.dropout_embed(x)
-
- x = self.transfomer_blocks(x)
- x = self.final_norm(x)
-
- logits = self.lm_head(x) # (bz, sq_len, vocab_size)
- return logits
diff --git a/chesstransformer/models/transformer/pos2move_v2.py b/chesstransformer/models/transformer/pos2move_v2.py
deleted file mode 100644
index 8a96e1968d8e5b0dc8fc80fd991eacc62f88d25f..0000000000000000000000000000000000000000
--- a/chesstransformer/models/transformer/pos2move_v2.py
+++ /dev/null
@@ -1,311 +0,0 @@
-import torch
-import torch.nn as nn
-import torch.nn.functional as F
-
-NUM_ACTION_PLANES = 73
-BOARD_SQUARES = 64
-EXTRA_TOKENS = 3 # castling, en_passant, player
-CONTEXT_SIZE = BOARD_SQUARES + EXTRA_TOKENS # 67
-
-
-def build_chess_relation_index(context_size: int = CONTEXT_SIZE) -> torch.Tensor:
- """Precompute (T, T) relation index between every pair of tokens.
-
- Categories (8 total):
- 0: same square / same token
- 1: same file
- 2: same rank
- 3: same diagonal
- 4: knight-reachable
- 5: king-adjacent (Chebyshev <= 1)
- 6: nearby (Manhattan <= 3)
- 7: far
- Game-state tokens (>= 64) get category 0 toward themselves and
- a dedicated "global" category 7 toward all board squares.
- """
- rel = torch.zeros(context_size, context_size, dtype=torch.long)
- for i in range(context_size):
- for j in range(context_size):
- if i == j:
- rel[i, j] = 0
- continue
- if i >= BOARD_SQUARES or j >= BOARD_SQUARES:
- # Any pair involving a game-state token: "global" bucket
- rel[i, j] = 7
- continue
- fi, ri = i % 8, i // 8
- fj, rj = j % 8, j // 8
- df, dr = abs(fi - fj), abs(ri - rj)
- if df == 0:
- rel[i, j] = 1
- elif dr == 0:
- rel[i, j] = 2
- elif df == dr:
- rel[i, j] = 3
- elif {df, dr} == {1, 2}:
- rel[i, j] = 4
- elif df <= 1 and dr <= 1:
- rel[i, j] = 5
- elif df + dr <= 3:
- rel[i, j] = 6
- else:
- rel[i, j] = 7
- return rel
-
-
-_RELATION_INDEX: torch.Tensor = build_chess_relation_index()
-
-
-class ReluSquared(nn.Module):
- def forward(self, x):
- return F.relu(x) ** 2
-
-class ChessGroupedQueryAttention(nn.Module):
- def __init__(
- self, d_in, d_out, num_heads, num_kv_groups, context_size, dropout=0.0, qkv_bias=False
- ):
- super().__init__()
-
- assert d_out % num_heads == 0, "d_out is indivisible by num_heads"
- assert num_heads % num_kv_groups == 0, "num_heads is indivisible by num_kv_groups"
-
- self.num_heads = num_heads
- self.num_kv_groups = num_kv_groups
- self.head_dim = d_out // num_heads
- self.d_out = d_out
-
- self.q_proj = nn.Linear(d_in, d_out, bias=qkv_bias)
- self.k_proj = nn.Linear(d_in, num_kv_groups * self.head_dim, bias=qkv_bias)
- self.v_proj = nn.Linear(d_in, num_kv_groups * self.head_dim, bias=qkv_bias)
- self.group_size = num_heads // num_kv_groups
-
- self.proj = nn.Linear(d_out, d_out, bias=False)
- self.dropout = dropout
-
- self.q_norm = nn.RMSNorm(self.head_dim)
- self.k_norm = nn.RMSNorm(self.head_dim)
-
- # Chess-geometry relative bias: 8 relation categories per head
- # Massively fewer params than fully-learned (H, T, T) bias.
- self.num_relations = 8
- self.bias_table = nn.Parameter(torch.zeros(num_heads, self.num_relations))
- self.register_buffer("rel_idx", _RELATION_INDEX, persistent=False)
-
-
- def forward(self, x):
- batch_size, num_tokens, embed_dim = x.shape
-
- queries = self.q_proj(x)
- keys = self.k_proj(x)
- values = self.v_proj(x)
-
- queries = queries.view(batch_size, num_tokens, self.num_heads, self.head_dim).transpose(1, 2)
- keys_new = keys.view(batch_size, num_tokens, self.num_kv_groups, self.head_dim).transpose(1, 2)
- values_new = values.view(batch_size, num_tokens, self.num_kv_groups, self.head_dim).transpose(1, 2)
-
- queries = self.q_norm(queries)
- keys_new = self.k_norm(keys_new)
-
- # Expand K/V to full head count. At this tiny sequence length (67) the
- # repeat_interleave is cheaper than SDPA's native enable_gqa path, which
- # falls back to a slower kernel here (measured ~20% slower).
- keys = keys_new.repeat_interleave(self.group_size, dim=1)
- values = values_new.repeat_interleave(self.group_size, dim=1)
-
- # Build additive chess-geometry bias from the compact (H, 8) table.
- # Shape: (1, H, T, T) so SDPA broadcasts it across the batch.
- attn_bias = self.bias_table[:, self.rel_idx].unsqueeze(0)
-
- context_vec = F.scaled_dot_product_attention(
- queries, keys, values,
- attn_mask=attn_bias,
- dropout_p=self.dropout if self.training else 0.0,
- is_causal=False,
- )
-
- context_vec = context_vec.transpose(1, 2).contiguous()
- context_vec = context_vec.view(batch_size, num_tokens, self.d_out)
- context_vec = self.proj(context_vec)
-
- return context_vec
-
-class MLP(nn.Module):
- def __init__(self, d_in, d_hidden, d_out, dropout=0.0):
- super().__init__()
- self.fc1 = nn.Linear(d_in, d_hidden)
- self.act = ReluSquared()
- self.fc2 = nn.Linear(d_hidden, d_out)
- self.dropout = nn.Dropout(dropout)
-
- def forward(self, x):
- x = self.fc1(x)
- x = self.act(x)
- x = self.dropout(x)
- x = self.fc2(x)
- return x
-
-class ChessTransformerLayer(nn.Module):
- def __init__(self, dim, nb_heads, qkv_bias, context_size, dropout=0.1, layer_drop=0.0):
- super().__init__()
- self.attn = ChessGroupedQueryAttention(
- d_in=dim,
- d_out=dim,
- num_heads=nb_heads,
- num_kv_groups=nb_heads // 2,
- context_size=context_size,
- dropout=dropout,
- qkv_bias=qkv_bias,
- )
- self.mlp = MLP(dim, dim * 4, dim, dropout=dropout)
- self.norm1 = nn.RMSNorm(dim)
- self.norm2 = nn.RMSNorm(dim)
- self.layer_drop = layer_drop
-
- def forward(self, x):
- # Stochastic depth: with prob `layer_drop`, skip the entire layer.
- # Applied independently to attn and mlp branches (sub-layer LayerDrop).
- if self.training and self.layer_drop > 0.0:
- # Per-sample keep mask would be more standard, but per-layer scalar matches your old behavior.
- keep_attn = torch.empty(1, device=x.device, dtype=x.dtype).bernoulli_(1.0 - self.layer_drop)
- keep_mlp = torch.empty(1, device=x.device, dtype=x.dtype).bernoulli_(1.0 - self.layer_drop)
- x = x + keep_attn * self.attn(self.norm1(x))
- x = x + keep_mlp * self.mlp(self.norm2(x))
- else:
- x = x + self.attn(self.norm1(x))
- x = x + self.mlp(self.norm2(x))
- return x
-
-class Pos2MoveV2(nn.Module):
- def __init__(
- self,
- vocab_size: int = 13,
- embed_dim: int = 512,
- nb_transformer_layers: int = 2,
- num_heads: int = 8,
- dropout: float = 0.1,
- kvq_bias: bool = False,
- layer_drop: float = 0.0,
- value_bounded: bool = False,
- value_clamp: bool = False,
- ):
- super().__init__()
-
- self.token_embedding = nn.Embedding(vocab_size, embed_dim)
- self.position_embedding = nn.Embedding(67, embed_dim) # 64 squares + 3 extra tokens
- # Static position indices, kept as a buffer so we don't rebuild the
- # arange (and a host->device copy) on every forward pass.
- self.register_buffer("pos_index", torch.arange(67), persistent=False)
- self.castling_embedding = nn.Embedding(16, embed_dim) # 4-bit bitmask: 0-15
- self.en_passant_embedding = nn.Embedding(9, embed_dim) # 8 files + no en passant (8)
- self.player_embedding = nn.Embedding(2, embed_dim) # White or Black
-
- # Linearly increasing layer drop rate: deeper layers drop more.
- # This is the "Deep Networks with Stochastic Depth" schedule.
- drop_rates = [layer_drop * i / max(1, nb_transformer_layers - 1)
- for i in range(nb_transformer_layers)]
-
- self.transformer_layers = nn.ModuleList(
- [
- ChessTransformerLayer(
- dim=embed_dim,
- nb_heads=num_heads,
- qkv_bias=kvq_bias,
- context_size=67, # Max sequence length (64 squares + 3 extra tokens)
- dropout=dropout,
- layer_drop=drop_rates[i],
- )
- for i in range(nb_transformer_layers)
- ]
- )
-
- self.final_norm = nn.RMSNorm(embed_dim)
-
- policy_hidden = embed_dim // 2
- self.move_head = nn.Sequential(
- nn.Linear(embed_dim, policy_hidden),
- nn.GELU(),
- nn.Linear(policy_hidden, NUM_ACTION_PLANES),
- )
-
- # Deeper value head: D → D//4 → GELU → 1
- # value_bounded appends a tanh so the output cannot leave the target's
- # [-1, 1] range. Kept off by default: the shipped v2.1 weights were
- # trained without it, and squashing them post-hoc changes their scale.
- # The Tanh lands at index 3, so the Linear state-dict keys (0.*, 2.*)
- # are identical either way and checkpoints stay interchangeable.
- value_hidden = embed_dim // 4
- value_layers = [
- nn.Linear(embed_dim, value_hidden),
- nn.GELU(),
- nn.Linear(value_hidden, 1),
- ]
- if value_bounded:
- value_layers.append(nn.Tanh())
- elif value_clamp:
- # Minimal alternative to tanh for weights already trained unbounded:
- # a hard clamp leaves every in-range output untouched and only edits
- # the semantically impossible ones (a leaf "better than won"). v2.1
- # emits those on 0.76% of positions, reaching 1.039 -- and since MCTS
- # maximises over leaf values, that tail is precisely what it chases.
- # MEASURED 2026-08-19: on v2.1 this is a NO-OP -- the chosen move was
- # identical on 25/25 opening positions at both 4000 and 8000 sims. A
- # 4% excursion is too small to steer the search; it mattered for the
- # cp-trained head only because that one reached 1.211. Don't re-run.
- # Kept for unbounded heads that overshoot more; prefer value_bounded
- # (tanh) when training a head from scratch.
- value_layers.append(nn.Hardtanh(-1.0, 1.0))
- self.value_head = nn.Sequential(*value_layers)
-
- self._init_weights(nb_transformer_layers)
-
- def _init_weights(self, nb_layers: int):
- """GPT-2 style init with depth-scaled residual projections.
-
- - All Linear/Embedding: N(0, 0.02)
- - Residual projections (attn out + mlp out): scaled by 1/sqrt(2*N)
- to keep residual stream variance bounded with depth.
- """
- for module in self.modules():
- if isinstance(module, nn.Linear):
- nn.init.normal_(module.weight, mean=0.0, std=0.02)
- if module.bias is not None:
- nn.init.zeros_(module.bias)
- elif isinstance(module, nn.Embedding):
- nn.init.normal_(module.weight, mean=0.0, std=0.02)
-
- # Scale down the projections that write into the residual stream
- scale = (2 * nb_layers) ** -0.5
- for layer in self.transformer_layers:
- with torch.no_grad():
- layer.attn.proj.weight.mul_(scale)
- layer.mlp.fc2.weight.mul_(scale)
-
- def forward(self, board_tokens, player_token, castling_token, en_passant_token):
- batch_size = board_tokens.size(0)
-
- token_emb = self.token_embedding(board_tokens) # (B, 64, D)
- pos_emb = self.position_embedding(self.pos_index) # (67, D)
- castling_emb = self.castling_embedding(castling_token).unsqueeze(1) # (B, 1, D)
- en_passant_emb = self.en_passant_embedding(en_passant_token).unsqueeze(1) # (B, 1, D)
- player_emb = self.player_embedding(player_token).unsqueeze(1) # (B, 1, D)
-
- x = token_emb + pos_emb[:64] # Add positional embedding to board tokens
- x = torch.cat([
- x,
- castling_emb + pos_emb[64],
- en_passant_emb + pos_emb[65],
- player_emb + pos_emb[66],
- ], dim=1) # (B, 67, D)
-
- for layer in self.transformer_layers:
- x = layer(x)
-
- x = self.final_norm(x)
-
- state_token = x[:, -3:, :].mean(dim=1) # Average the last 3 game-state tokens: (B, D)
-
- board_tokens_out = x[:, :64, :] # (B, 64, D)
- move_logits = self.move_head(board_tokens_out) # (B, 64, NUM_ACTION_PLANES)
- value_pred = self.value_head(state_token) # (B, 1)
-
- return move_logits, value_pred
diff --git a/chesstransformer/models/transformer/position2move.py b/chesstransformer/models/transformer/position2move.py
deleted file mode 100644
index d5d7a449837a225fb74020b7783c2e49d95be982..0000000000000000000000000000000000000000
--- a/chesstransformer/models/transformer/position2move.py
+++ /dev/null
@@ -1,329 +0,0 @@
-import torch
-from torch import nn
-import torch.nn.functional as F
-
-from chesstransformer.models.transformer.attention import GroupedQueryAttention
-
-
-class RMSNorm(nn.Module):
- """Root Mean Square Layer Normalization.
-
- More efficient than LayerNorm as it doesn't compute mean.
- Used in Llama, Mistral, and other modern transformers.
- """
-
- def __init__(self, dim: int, eps: float = 1e-6):
- super().__init__()
- self.eps = eps
- self.weight = nn.Parameter(torch.ones(dim))
-
- def forward(self, x: torch.Tensor) -> torch.Tensor:
- # x: (batch, seq, dim)
- rms = torch.rsqrt(x.pow(2).mean(-1, keepdim=True) + self.eps)
- return x * rms * self.weight
-
-
-class SwiGLUFeedForward(nn.Module):
- """SwiGLU Feed-Forward Network.
-
- Combines Swish activation with Gated Linear Units.
- Better performance than standard GELU FFN.
- Used in Llama 2, PaLM, and other modern transformers.
- """
-
- def __init__(self, embed_dim: int, hidden_dim: int = None, dropout: float = 0.0):
- super().__init__()
- # Default hidden_dim is ~2.67x embed_dim (compensates for extra params from gate)
- hidden_dim = hidden_dim or int(embed_dim * 8 / 3)
- # Round to nearest multiple of 64 for efficiency
- hidden_dim = ((hidden_dim + 63) // 64) * 64
-
- self.w1 = nn.Linear(embed_dim, hidden_dim, bias=False) # Up projection
- self.w2 = nn.Linear(hidden_dim, embed_dim, bias=False) # Down projection
- self.w3 = nn.Linear(embed_dim, hidden_dim, bias=False) # Gate projection
- self.dropout = nn.Dropout(dropout)
-
- def forward(self, x: torch.Tensor) -> torch.Tensor:
- # SwiGLU: w2(SiLU(w1(x)) * w3(x))
- return self.dropout(self.w2(F.silu(self.w1(x)) * self.w3(x)))
-
-
-# Keep old FeedForward for backward compatibility
-class FeedForward(nn.Module):
- def __init__(self, embed_dim):
- super().__init__()
- self.layers = nn.Sequential(
- nn.Linear(embed_dim, 4 * embed_dim),
- nn.GELU(approximate="tanh"),
- nn.Linear(4 * embed_dim, embed_dim),
- )
-
- def forward(self, x):
- return self.layers(x)
-
-
-class TransformerBlock(nn.Module):
- def __init__(
- self,
- embed_dim,
- num_heads,
- num_kv_groups,
- context_length,
- dropout,
- qkv_bias,
- mask_future=False,
- apply_rope=True,
- use_swiglu=True,
- ):
- super().__init__()
- self.att = GroupedQueryAttention(
- d_in=embed_dim,
- d_out=embed_dim,
- num_heads=num_heads,
- num_kv_groups=num_kv_groups,
- dropout=dropout,
- qkv_bias=qkv_bias,
- mask_future=mask_future,
- apply_rope=apply_rope,
- )
- # Use SwiGLU by default, fallback to standard FFN
- if use_swiglu:
- self.feed_forward = SwiGLUFeedForward(embed_dim, dropout=dropout)
- else:
- self.feed_forward = FeedForward(embed_dim)
-
- # Use RMSNorm instead of LayerNorm
- self.norm1 = RMSNorm(embed_dim)
- self.norm2 = RMSNorm(embed_dim)
- self.dropout_shortcut = nn.Dropout(dropout)
-
- def forward(self, x):
- shortcut = x
- x = self.norm1(x)
- x = self.att(x)
- x = self.dropout_shortcut(x)
- x = x + shortcut
-
- shortcut = x
- x = self.norm2(x)
- x = self.feed_forward(x)
- x = self.dropout_shortcut(x)
- x = x + shortcut
-
- return x
-
-
-class Position2MoveModel(nn.Module):
- def __init__(
- self,
- vocab_size: int = 13,
- move_vocab_size: int = 1856,
- embed_dim: int = 512,
- nb_transformer_layers: int = 8,
- num_heads: int = 8,
- num_kv_groups: int = 4,
- dropout: float = 0.1,
- kvq_bias: bool = False,
- mask_future: bool = False,
- rope: bool = True,
- use_swiglu: bool = True,
- use_col_row_emb: bool = False,
- ):
- super().__init__()
- # Learnable CLS token for classification (instead of mean pooling)
- self.cls_token = nn.Parameter(torch.randn(1, 1, embed_dim) * 0.02)
-
- # Board piece embeddings
- self.token_embedding = nn.Embedding(vocab_size, embed_dim)
- if not rope:
- self.position_embedding = nn.Embedding(64, embed_dim) # 64 squares
- self.player_embedding = nn.Embedding(2, embed_dim) # 2 players: white and black
-
- self.use_col_row_emb = use_col_row_emb
-
- if use_col_row_emb:
- # Optional column and row embeddings for board squares
- self.col_embedding = nn.Embedding(8, embed_dim)
- self.row_embedding = nn.Embedding(8, embed_dim)
-
- # Additional game state embeddings
- # Castling rights: 4 binary flags (white_king, white_queen, black_king, black_queen)
- self.castling_embedding = nn.Embedding(16, embed_dim) # 2^4 = 16 combinations
-
- # En passant: 9 values (files a-h = 0-7, none = 8)
- self.en_passant_embedding = nn.Embedding(9, embed_dim)
-
- # Halfmove clock: bucketed into ranges for 50-move rule awareness
- # Buckets: 0, 1-5, 6-10, 11-20, 21-30, 31-40, 41-50, 50+
- self.halfmove_embedding = nn.Embedding(8, embed_dim)
-
- self.dropout = nn.Dropout(dropout)
- self.transformer_blocks = nn.Sequential(
- *[
- TransformerBlock(
- embed_dim=embed_dim,
- num_heads=num_heads,
- num_kv_groups=num_kv_groups,
- context_length=64 + 3 + 1, # 64 squares + 3 game state tokens + 1 CLS
- dropout=dropout,
- qkv_bias=kvq_bias,
- mask_future=mask_future,
- apply_rope=rope,
- use_swiglu=use_swiglu,
- )
- for _ in range(nb_transformer_layers)
- ]
- )
- # Final RMSNorm
- self.norm = RMSNorm(embed_dim)
-
- self.lm_head = nn.Linear(embed_dim, move_vocab_size, bias=False)
-
- self._init_weights()
-
- def _init_weights(self):
- torch.nn.init.normal_(self.token_embedding.weight, mean=0.0, std=0.02)
- if hasattr(self, "position_embedding"):
- torch.nn.init.normal_(self.position_embedding.weight, mean=0.0, std=0.02)
- if self.use_col_row_emb:
- torch.nn.init.normal_(self.col_embedding.weight, mean=0.0, std=0.02)
- torch.nn.init.normal_(self.row_embedding.weight, mean=0.0, std=0.02)
-
- torch.nn.init.normal_(self.player_embedding.weight, mean=0.0, std=0.02)
- torch.nn.init.normal_(self.castling_embedding.weight, mean=0.0, std=0.02)
- torch.nn.init.normal_(self.en_passant_embedding.weight, mean=0.0, std=0.02)
- torch.nn.init.normal_(self.halfmove_embedding.weight, mean=0.0, std=0.02)
-
- # init the q, k, v, and output projection layers using xavier initialization
- for layer in self.transformer_blocks:
- if isinstance(layer.att, GroupedQueryAttention):
- torch.nn.init.xavier_uniform_(layer.att.q_proj.weight)
- if layer.att.q_proj.bias is not None:
- torch.nn.init.zeros_(layer.att.q_proj.bias)
- torch.nn.init.xavier_uniform_(layer.att.k_proj.weight)
- if layer.att.k_proj.bias is not None:
- torch.nn.init.zeros_(layer.att.k_proj.bias)
- torch.nn.init.xavier_uniform_(layer.att.v_proj.weight)
- if layer.att.v_proj.bias is not None:
- torch.nn.init.zeros_(layer.att.v_proj.bias)
- torch.nn.init.xavier_uniform_(layer.att.proj.weight)
-
- # Initialize SwiGLU layers
- if isinstance(layer.feed_forward, SwiGLUFeedForward):
- torch.nn.init.xavier_uniform_(layer.feed_forward.w1.weight)
- torch.nn.init.xavier_uniform_(layer.feed_forward.w2.weight)
- torch.nn.init.xavier_uniform_(layer.feed_forward.w3.weight)
-
- torch.nn.init.xavier_uniform_(self.lm_head.weight)
-
- def _bucket_halfmove(self, halfmove: torch.Tensor) -> torch.Tensor:
- """Convert halfmove clock values to bucket indices.
-
- Buckets: 0, 1-5, 6-10, 11-20, 21-30, 31-40, 41-50, 50+
- Uses searchsorted to avoid torch.where (better for MPS compilation).
- """
- # Define bucket boundaries
- boundaries = torch.tensor([1, 6, 11, 21, 31, 41, 51], device=halfmove.device)
- # searchsorted gives us the bucket index directly
- # Values less than 1 -> bucket 0
- # Values in [1, 6) -> bucket 1
- # Values in [6, 11) -> bucket 2, etc.
- buckets = torch.searchsorted(boundaries, halfmove.float(), right=False)
- buckets = torch.clamp(buckets, 0, 7)
- return buckets.long()
-
- def forward(self, x, is_white, castling_rights=None, en_passant_file=None, halfmove_clock=None):
- """
- Args:
- x: Tensor of shape (batch_size, 64) containing token IDs for each square.
- is_white: Tensor of shape (batch_size,) indicating if the player to move is white (True) or black (False).
- castling_rights: Tensor of shape (batch_size,) with values 0-15 encoding 4 binary flags.
- Bit 0: white kingside, Bit 1: white queenside,
- Bit 2: black kingside, Bit 3: black queenside.
- If None, defaults to all castling allowed (15).
- en_passant_file: Tensor of shape (batch_size,) with values 0-7 for files a-h, or 8 for none.
- If None, defaults to no en passant (8).
- halfmove_clock: Tensor of shape (batch_size,) with halfmove clock values (0-100+).
- If None, defaults to 0.
- Returns:
- logits: Tensor of shape (batch_size, move_vocab_size) containing
- the logits for the next move prediction at each position.
- """
- batch_size, seq_len = x.size()
- assert seq_len == 64, "Input sequence length must be 64 (8x8 chess board)."
-
- device = x.device
-
- # Default values if not provided
- if castling_rights is None:
- castling_rights = torch.full((batch_size,), 15, dtype=torch.long, device=device)
- if en_passant_file is None:
- en_passant_file = torch.full((batch_size,), 8, dtype=torch.long, device=device)
- if halfmove_clock is None:
- halfmove_clock = torch.zeros(batch_size, dtype=torch.long, device=device)
-
- positions = torch.arange(seq_len, device=device).unsqueeze(0).expand(batch_size, -1)
- token_emb = self.token_embedding(x) # (batch_size, 64, embed_dim)
- if hasattr(self, "position_embedding"):
- pos_emb = self.position_embedding(positions) # (batch_size, 64, embed_dim)
- else:
- pos_emb = torch.zeros_like(token_emb) # Fallback if RoPE is used
-
- player_ids = is_white.long().unsqueeze(1).expand(-1, seq_len) # (batch_size, 64)
-
- player_emb = self.player_embedding(player_ids) # (batch_size, 64, embed_dim)
-
- board_emb = token_emb + pos_emb + player_emb # (batch_size, 64, embed_dim)
-
- if self.use_col_row_emb:
- cols = (positions % 8).long() # (batch_size, 64)
- rows = (positions // 8).long() # (batch_size, 64)
- col_emb = self.col_embedding(cols) # (batch_size, 64, embed_dim)
- row_emb = self.row_embedding(rows) # (batch_size, 64, embed_dim)
- board_emb = board_emb + col_emb + row_emb # (batch_size, 64, embed_dim)
-
- # Create game state tokens
- castling_emb = self.castling_embedding(castling_rights.long()) # (batch_size, embed_dim)
- en_passant_emb = self.en_passant_embedding(en_passant_file.long()) # (batch_size, embed_dim)
- halfmove_bucket = self._bucket_halfmove(halfmove_clock.long())
- halfmove_emb = self.halfmove_embedding(halfmove_bucket) # (batch_size, embed_dim)
-
- # Stack game state tokens: (batch_size, 3, embed_dim)
- game_state_tokens = torch.stack([castling_emb, en_passant_emb, halfmove_emb], dim=1)
-
- # Expand CLS token for batch: (batch_size, 1, embed_dim)
- cls_tokens = self.cls_token.expand(batch_size, -1, -1)
-
- # Concatenate: [CLS] + board + game_state: (batch_size, 68, embed_dim)
- x = torch.cat([cls_tokens, board_emb, game_state_tokens], dim=1)
- x = self.dropout(x)
-
- x = self.transformer_blocks(x) # (batch_size, 68, embed_dim)
- x = self.norm(x) # (batch_size, 68, embed_dim)
-
- # Use CLS token output for classification (instead of mean pooling)
- x = x[:, 0, :] # (batch_size, embed_dim)
-
- logits = self.lm_head(x) # (batch_size, move_vocab_size)
- return logits
-
-
-if __name__ == "__main__":
- # Example usage
- model = Position2MoveModel()
- batch_size = 2
- dummy_input = torch.randint(0, 13, (batch_size, 64)) # batch_size=2, seq_len=64
- is_white = torch.tensor([1, 0]) # First sample is white to move, second is black
-
- # With game state info
- castling_rights = torch.tensor([15, 3]) # First: all castling, Second: only white can castle
- en_passant_file = torch.tensor([8, 4]) # First: no EP, Second: EP on file e (4)
- halfmove_clock = torch.tensor([0, 25]) # Halfmove counters
-
- # Forward pass with all info
- logits = model(dummy_input, is_white, castling_rights, en_passant_file, halfmove_clock)
- print("Output shape:", logits.shape) # Should be (2, move_vocab_size)
-
- # Also works without game state (backward compatible)
- logits_simple = model(dummy_input, is_white)
- print("Output shape (simple):", logits_simple.shape)
diff --git a/chesstransformer/models/transformer/rope.py b/chesstransformer/models/transformer/rope.py
deleted file mode 100644
index 50b3d53d96629911ce2999f052c2f5716020a95a..0000000000000000000000000000000000000000
--- a/chesstransformer/models/transformer/rope.py
+++ /dev/null
@@ -1,104 +0,0 @@
-"""This module implements Rotary Positional Embeddings (RoPE) as described in
-the paper "RoFormer: Enhanced Transformer with Rotary Position Embedding" (Su et al., 2021).
-"""
-
-import torch
-import torch.nn as nn
-
-
-class RoPEEmbedding(nn.Module):
- """Rotary Positional Embedding module.
-
- This module applies rotary positional embeddings to the input tensor.
- Works with input shape (batch, num_heads, seq_len, head_dim).
- """
-
- def __init__(self, dim: int, base: int = 10_000, max_position_embeddings: int = 2048):
- """Initialize the RoPE embedding module.
-
- Args:
- dim (int): Dimension of each attention head (head_dim).
- base (int): Base for the frequency computation. Lower values = faster rotation.
- Default 10,000 works well for long sequences. For short sequences
- like chess (67 tokens), consider lower values like 1,000.
- max_position_embeddings (int): Maximum number of position embeddings.
- """
- super(RoPEEmbedding, self).__init__()
- self.dim = dim
- self.base = base
-
- # Compute inverse frequencies
- inv_freq = 1.0 / (base ** (torch.arange(0, dim, 2).float() / dim))
-
- # Compute position encodings for all positions
- position = torch.arange(max_position_embeddings, dtype=torch.float)
- # (max_pos, dim/2)
- idx_theta = torch.einsum("i, j -> i j", position, inv_freq)
-
- # Duplicate for both halves: (max_pos, dim)
- idx_theta2 = torch.cat([idx_theta, idx_theta], dim=-1)
-
- # Shape: (1, 1, max_pos, dim) for broadcasting with (batch, heads, seq, dim)
- self.register_buffer("cos_cached", idx_theta2.cos()[None, None, :, :], persistent=False)
- self.register_buffer("sin_cached", idx_theta2.sin()[None, None, :, :], persistent=False)
-
- def _neg_half(self, x: torch.Tensor):
- """Rotate tensor by swapping and negating halves.
-
- For RoPE, we need to compute: [-x2, x1] where x = [x1, x2]
- This implements the rotation matrix multiplication efficiently.
-
- Args:
- x (torch.Tensor): Input tensor of shape (..., dim).
-
- Returns:
- torch.Tensor: Rotated tensor.
- """
- d_2 = self.dim // 2
- x1 = x[..., :d_2]
- x2 = x[..., d_2:]
- return torch.cat([-x2, x1], dim=-1)
-
- def forward(self, x: torch.Tensor):
- """Apply RoPE to the input tensor.
-
- Args:
- x (torch.Tensor): Input tensor of shape (batch, num_heads, seq_len, head_dim).
-
- Returns:
- torch.Tensor: Tensor with RoPE applied, same shape as input.
- """
- # x shape: (batch, num_heads, seq_len, head_dim)
- seq_len = x.shape[2] # Get seq_len from the correct dimension
-
- # Slice cached values to sequence length: (1, 1, seq_len, dim)
- cos = self.cos_cached[:, :, :seq_len, :].to(x.device)
- sin = self.sin_cached[:, :, :seq_len, :].to(x.device)
-
- # Apply rotary embedding: x * cos + rotate(x) * sin
- x_rotated = self._neg_half(x)
- return x * cos + x_rotated * sin
-
-
-if __name__ == "__main__":
- # Test with chess-like dimensions
- batch_size = 2
- num_heads = 8
- seq_len = 67 # 64 squares + 3 game state tokens
- head_dim = 64
-
- rope = RoPEEmbedding(dim=head_dim, base=10_000, max_position_embeddings=128)
- x = torch.randn(batch_size, num_heads, seq_len, head_dim)
-
- output = rope(x)
- print(f"Input shape: {x.shape}")
- print(f"Output shape: {output.shape}")
-
- # Verify positions are encoded differently
- # Two different positions should produce different outputs
- x_same = torch.ones(1, 1, 10, head_dim)
- out = rope(x_same)
- print(f"\nPosition encoding test (should be different):")
- print(f" Position 0 norm: {out[0, 0, 0].norm():.4f}")
- print(f" Position 5 norm: {out[0, 0, 5].norm():.4f}")
- print(f" Position 0 vs 5 cosine sim: {torch.cosine_similarity(out[0, 0, 0], out[0, 0, 5], dim=0):.4f}")
diff --git a/chesstransformer/models/transformer/utils.py b/chesstransformer/models/transformer/utils.py
deleted file mode 100644
index 895616d7974df70f3c7951f4ec542339eb69e223..0000000000000000000000000000000000000000
--- a/chesstransformer/models/transformer/utils.py
+++ /dev/null
@@ -1,46 +0,0 @@
-"""Utility functions for models."""
-
-import torch
-import torch.nn as nn
-
-
-class LayerNorm(nn.Module):
- """Implements Layer Normalization"""
-
- def __init__(self, dim, eps=1e-5):
- super().__init__()
- self.eps = eps
- self.gamma = nn.Parameter(torch.ones(dim))
- self.beta = nn.Parameter(torch.zeros(dim))
-
- def forward(self, x):
- """
- Forward pass of layer normalization
- Args:
- x: Input tensor of shape (batch_size, seq_length, dim)
- Returns:
- out: Layer-normalized tensor of the same shape as input
- """
-
- mean = x.mean(-1, keepdim=True)
- var = x.var(-1, keepdim=True, unbiased=False)
- x_hat = (x - mean) / torch.sqrt(var + self.eps)
- out = self.gamma * x_hat + self.beta
- return out
-
-
-class GELU(nn.Module):
- """Implements the GELU activation function"""
-
- def __init__(self):
- super().__init__()
-
- def forward(self, x):
- """
- Forward pass of GELU activation
- Args:
- x: Input tensor
- Returns:
- out: Tensor after applying GELU activation
- """
- return 0.5 * x * (1 + torch.tanh(torch.sqrt(torch.tensor(2.0 / torch.pi)) * (x + 0.044715 * torch.pow(x, 3))))
diff --git a/chesstransformer/optimizer.py b/chesstransformer/optimizer.py
deleted file mode 100644
index 2a4dd38f763b11f7da3c92abf7630367d703d872..0000000000000000000000000000000000000000
--- a/chesstransformer/optimizer.py
+++ /dev/null
@@ -1,206 +0,0 @@
-"""Combined AdamW + Muon optimizer.
-
-The two underlying optimizers are kept as separate ``torch.optim`` instances
-internally; this class is a thin wrapper that exposes the standard
-``torch.optim.Optimizer`` interface (``step``, ``zero_grad``, ``param_groups``,
-``state_dict`` / ``load_state_dict``) so that LR schedulers, gradient
-accumulation, and HuggingFace ``Accelerator`` treat it as a single optimizer.
-
-Routing rule (when constructed from an ``nn.Module``):
-
-* parameters whose name contains an ``embedding_keyword`` → AdamW (low LR)
-* parameters whose name contains a ``head_keyword`` → AdamW (no decay)
-* remaining parameters with ``ndim >= 2`` → Muon
-* remaining parameters with ``ndim < 2`` (biases/norms) → AdamW (no decay)
-"""
-
-from __future__ import annotations
-
-from typing import Iterable
-
-import torch
-from torch import nn
-from torch.optim import AdamW, Muon, Optimizer
-
-
-class AdamWMuon(Optimizer):
- """Wrapper that drives an inner ``AdamW`` and ``Muon`` in lock-step.
-
- Parameters
- ----------
- model:
- ``nn.Module`` whose parameters will be auto-routed by name.
- lr:
- Base AdamW learning rate (used for the ``other`` 1D parameter group).
- lr_embedding, lr_head:
- Per-group AdamW LR overrides. Default to ``lr``.
- lr_muon:
- Muon LR for ≥2D body weights. Default ``= 3 * lr`` (Muon tolerates a
- higher LR than AdamW because updates are orthogonalised).
- weight_decay:
- AdamW weight decay applied to the embedding group. Heads and 1D
- params get ``0.0`` by convention.
- weight_decay_muon:
- Muon weight decay. Default = ``weight_decay``.
- embedding_keywords / head_keywords:
- Substrings used to classify parameters by ``named_parameters`` key.
- """
-
- def __init__(
- self,
- model: nn.Module,
- *,
- lr: float = 3e-4,
- lr_embedding: float | None = None,
- lr_head: float | None = None,
- lr_muon: float | None = None,
- weight_decay: float = 0.1,
- weight_decay_muon: float | None = None,
- betas: tuple[float, float] = (0.9, 0.95),
- eps: float = 1e-8,
- muon_momentum: float = 0.95,
- muon_nesterov: bool = True,
- muon_ns_steps: int = 5,
- embedding_keywords: Iterable[str] = ("embedding", "cls_token"),
- head_keywords: Iterable[str] = ("lm_head", "value_head"),
- ) -> None:
- if not isinstance(model, nn.Module):
- raise TypeError(
- f"AdamWMuon expects an nn.Module to auto-route parameters by name, "
- f"got {type(model).__name__}."
- )
-
- lr_embedding = lr if lr_embedding is None else lr_embedding
- lr_head = lr if lr_head is None else lr_head
- lr_muon = 3.0 * lr if lr_muon is None else lr_muon
- weight_decay_muon = weight_decay if weight_decay_muon is None else weight_decay_muon
-
- embedding_keywords = tuple(embedding_keywords)
- head_keywords = tuple(head_keywords)
-
- emb_params: list[torch.Tensor] = []
- head_params: list[torch.Tensor] = []
- other_params: list[torch.Tensor] = []
- muon_params: list[torch.Tensor] = []
-
- for name, param in model.named_parameters():
- if not param.requires_grad:
- continue
- if any(kw in name for kw in embedding_keywords):
- emb_params.append(param)
- elif any(kw in name for kw in head_keywords):
- head_params.append(param)
- elif param.ndim >= 2:
- muon_params.append(param)
- else:
- other_params.append(param)
-
- adamw_groups: list[dict] = []
- if emb_params:
- adamw_groups.append({
- "params": emb_params, "lr": lr_embedding,
- "weight_decay": weight_decay, "name": "adamw_embedding",
- })
- if other_params:
- adamw_groups.append({
- "params": other_params, "lr": lr,
- "weight_decay": 0.0, "name": "adamw_other",
- })
- if head_params:
- adamw_groups.append({
- "params": head_params, "lr": lr_head,
- "weight_decay": 0.0, "name": "adamw_head",
- })
-
- if not adamw_groups:
- raise ValueError("No AdamW-eligible parameters found in model.")
-
- self.adamw = AdamW(adamw_groups, lr=lr, betas=betas, eps=eps,
- weight_decay=weight_decay)
-
- if muon_params:
- self.muon: Muon | None = Muon(
- [{"params": muon_params, "lr": lr_muon,
- "weight_decay": weight_decay_muon, "name": "muon_body"}],
- lr=lr_muon, weight_decay=weight_decay_muon,
- momentum=muon_momentum, nesterov=muon_nesterov,
- ns_steps=muon_ns_steps,
- )
- else:
- self.muon = None
-
- # Initialise the ``Optimizer`` base with a placeholder group so
- # ``isinstance(opt, Optimizer)`` checks (Accelerate, schedulers) pass
- # and ``self.state`` / ``self.defaults`` exist. We then expose the
- # *inner* groups via ``param_groups`` so schedulers mutate them in
- # place and both sub-optimizers see the new LR.
- all_params = emb_params + other_params + head_params + muon_params
- self._init_done = False
- super().__init__([{"params": all_params}], defaults={"lr": lr})
- self.param_groups = self._combined_param_groups()
- self._init_done = True
-
- # ── helpers ─────────────────────────────────────────────────────────
- def _combined_param_groups(self) -> list[dict]:
- groups = list(self.adamw.param_groups)
- if self.muon is not None:
- groups += list(self.muon.param_groups)
- return groups
-
- # ── Optimizer API ───────────────────────────────────────────────────
- @torch.no_grad()
- def step(self, closure=None):
- loss = None
- if closure is not None:
- with torch.enable_grad():
- loss = closure()
- self.adamw.step()
- if self.muon is not None:
- self.muon.step()
- return loss
-
- def zero_grad(self, set_to_none: bool = True) -> None:
- self.adamw.zero_grad(set_to_none=set_to_none)
- if self.muon is not None:
- self.muon.zero_grad(set_to_none=set_to_none)
-
- def state_dict(self) -> dict:
- return {
- "adamw": self.adamw.state_dict(),
- "muon": self.muon.state_dict() if self.muon is not None else None,
- }
-
- def load_state_dict(self, state_dict: dict) -> None:
- self.adamw.load_state_dict(state_dict["adamw"])
- if self.muon is not None and state_dict.get("muon") is not None:
- self.muon.load_state_dict(state_dict["muon"])
- # Re-link in case sub-optimizers rebuilt their group dicts.
- self.param_groups = self._combined_param_groups()
-
- def add_param_group(self, param_group) -> None:
- if not getattr(self, "_init_done", False):
- # Called from ``Optimizer.__init__`` with the placeholder group.
- super().add_param_group(param_group)
- return
- raise NotImplementedError(
- "add_param_group is not supported on AdamWMuon after construction; "
- "build a new optimizer with the full parameter set instead."
- )
-
- def __repr__(self) -> str:
- n_emb = sum(p.numel() for g in self.adamw.param_groups
- if g.get("name") == "adamw_embedding" for p in g["params"])
- n_head = sum(p.numel() for g in self.adamw.param_groups
- if g.get("name") == "adamw_head" for p in g["params"])
- n_other = sum(p.numel() for g in self.adamw.param_groups
- if g.get("name") == "adamw_other" for p in g["params"])
- n_muon = (sum(p.numel() for g in self.muon.param_groups for p in g["params"])
- if self.muon is not None else 0)
- return (
- f"AdamWMuon(emb={n_emb:,}, other_1d={n_other:,}, "
- f"head={n_head:,}, muon_2d={n_muon:,})"
- )
-
-
-# Backwards-compat alias for the previous skeleton class.
-AdamWMuonOptim = AdamWMuon
\ No newline at end of file
diff --git a/chesstransformer/trainers/__init__.py b/chesstransformer/trainers/__init__.py
deleted file mode 100644
index 522d991ec55e20cf8bd642f5cd2ba2efc92dac8c..0000000000000000000000000000000000000000
--- a/chesstransformer/trainers/__init__.py
+++ /dev/null
@@ -1,11 +0,0 @@
-
-
-__all__ = [
- "SelfPlayTrainer",
- "SelfPlayEngine",
- "GameRecord",
- "MoveRecord",
- "OpeningPositionSampler",
- "compute_rewards",
- "COMMON_OPENINGS",
-]
diff --git a/chesstransformer/trainers/moveseq2moveseq_trainer.py b/chesstransformer/trainers/moveseq2moveseq_trainer.py
deleted file mode 100644
index a468f4b41b028e55c5c9eb7d5958817228f0de48..0000000000000000000000000000000000000000
--- a/chesstransformer/trainers/moveseq2moveseq_trainer.py
+++ /dev/null
@@ -1,342 +0,0 @@
-from pathlib import Path
-from datetime import datetime
-
-from tqdm.auto import tqdm
-import torch
-import accelerate
-from torch.utils.data import DataLoader
-from torch.utils.data import random_split
-from chesstransformer.datasets import LichessSimpleUciDataset
-from chesstransformer.models.transformer.gpt import GPT2Model
-
-
-ROOT_PATH = Path(__file__).parents[1]
-
-
-def get_next_run_number(log_dir="logs"):
- """Get the next run number by checking existing run directories."""
- log_path = Path(log_dir)
- if not log_path.exists():
- return 1
-
- # Find all directories that match the pattern "run_XXX"
- existing_runs = [d.name for d in log_path.iterdir() if d.is_dir() and d.name.startswith("run_")]
-
- if not existing_runs:
- return 1
-
- # Extract run numbers and find the maximum
- run_numbers = []
- for run_name in existing_runs:
- try:
- num = int(run_name.split("_")[1])
- run_numbers.append(num)
- except (IndexError, ValueError):
- continue
-
- return max(run_numbers) + 1 if run_numbers else 1
-
-
-def main():
- # Load the full dataset with LichessSimpleUciDataset
- print("Loading dataset...")
- full_dataset = LichessSimpleUciDataset(
- dataset_path=ROOT_PATH.parents[1] / "data/Lichess Rated Games 2017.pgn.zst",
- vocab_path=ROOT_PATH / "data/tokenizer_models/uci_vocab.json",
- num_samples=1_000_000, # Load all available games
- )
-
- vocab_size = len(full_dataset.vocab)
- print(f"Vocabulary size: {vocab_size}")
-
- # Split the dataset into training, validation, and test sets
- val = 20_000
- test = 20_000
- train_size = len(full_dataset) - val - test
- train_dataset, val_dataset, test_dataset = random_split(
- full_dataset,
- [train_size, val, test],
- generator=torch.Generator().manual_seed(42),
- )
-
- print(f"Train size: {len(train_dataset)}")
- print(f"Validation size: {len(val_dataset)}")
- print(f"Test size: {len(test_dataset)}")
-
- def collate_fn(batch):
- max_length = max(item["input_ids"].shape[0] for item in batch)
- input_ids = torch.full((len(batch), max_length), fill_value=-100, dtype=torch.long)
- target_ids = torch.full((len(batch), max_length), fill_value=-100, dtype=torch.long)
- legal_moves_mask = torch.zeros((len(batch), max_length, vocab_size), dtype=torch.float32)
-
- for i, item in enumerate(batch):
- length = item["input_ids"].shape[0]
- input_ids[i, :length] = item["input_ids"]
- target_ids[i, :length] = item["target_ids"]
- legal_moves_mask[i, :length, :] = item["legal_moves_mask"]
-
- if max_length > config["context_size"]:
- input_ids = input_ids[:, : config["context_size"]]
- target_ids = target_ids[:, : config["context_size"]]
- legal_moves_mask = legal_moves_mask[:, : config["context_size"], :]
-
- return {
- "input_ids": input_ids,
- "target_ids": target_ids,
- "legal_moves_mask": legal_moves_mask,
- }
-
- train_dl = DataLoader(train_dataset, batch_size=8, shuffle=True, collate_fn=collate_fn, drop_last=True)
- val_dl = DataLoader(val_dataset, batch_size=8, shuffle=False, collate_fn=collate_fn)
- test_dl = DataLoader(test_dataset, batch_size=8, shuffle=False, collate_fn=collate_fn)
-
- config = {
- "vocab_size": vocab_size,
- "context_size": 256,
- "embed_dim": 512,
- "num_heads": 8,
- "num_layers": 8,
- "dropout": 0.1,
- "qkv_bias": False,
- }
-
- model = GPT2Model(config)
- print(model)
-
- # Generate unique run identifier
- run_number = get_next_run_number("logs")
- timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
- run_name = f"run_{run_number:03d}_{timestamp}"
-
- print(f"Starting training run: {run_name}")
-
- # Gradient accumulation settings
- gradient_accumulation_steps = 8 # Effective batch size = 16 * 4 = 64
-
- # Initialize accelerator with logging and gradient accumulation
- accelerator = accelerate.Accelerator(
- log_with="tensorboard",
- project_dir=f"logs/{run_name}",
- mixed_precision="fp16" if torch.cuda.is_available() else "no",
- gradient_accumulation_steps=gradient_accumulation_steps,
- )
- device = accelerator.device
- print(f"Using device: {device}")
- print(f"Gradient accumulation steps: {gradient_accumulation_steps}")
- print(f"Effective batch size: {16 * gradient_accumulation_steps}")
- model.to(device)
-
- # Initialize TensorBoard tracker
- accelerator.init_trackers(
- project_name="chess_transformer",
- config={
- **config,
- "run_name": run_name,
- "run_number": run_number,
- "timestamp": timestamp,
- "learning_rate": 1e-3,
- "weight_decay": 1e-1,
- "num_epochs": 10,
- "batch_size": 16,
- "gradient_accumulation_steps": 8,
- "effective_batch_size": 128,
- "warmup_steps": 1000,
- },
- )
-
- optimizer = torch.optim.AdamW(model.parameters(), lr=5e-4, weight_decay=1e-1)
-
- # Calculate total training steps (accounting for gradient accumulation)
- num_epochs = 10
- total_steps = (len(train_dl) // gradient_accumulation_steps) * num_epochs
- warmup_steps = 1000
-
- print(f"Total training steps: {total_steps}")
- print(f"Warmup steps: {warmup_steps}")
-
- # Create linear scheduler with warmup
- def get_linear_schedule_with_warmup(optimizer, num_warmup_steps, num_training_steps):
- def lr_lambda(current_step):
- if current_step < num_warmup_steps:
- # Linear warmup
- return float(current_step) / float(max(1, num_warmup_steps))
- # Linear decay
- return max(
- 0.0, float(num_training_steps - current_step) / float(max(1, num_training_steps - num_warmup_steps))
- )
-
- return torch.optim.lr_scheduler.LambdaLR(optimizer, lr_lambda)
-
- scheduler = get_linear_schedule_with_warmup(optimizer, warmup_steps, total_steps)
-
- model, optimizer, train_dl, val_dl, scheduler = accelerator.prepare(model, optimizer, train_dl, val_dl, scheduler)
- criterion = torch.nn.CrossEntropyLoss(ignore_index=-100)
-
- best_val_loss = float("inf")
- global_step = 0
-
- for epoch in range(num_epochs):
- model.train()
- total_loss = 0.0
- total_llm_loss = 0.0
- total_l2_loss = 0.0
-
- loader = tqdm(train_dl, desc=f"Epoch {epoch + 1}/{num_epochs}")
- for batch_idx, batch in enumerate(loader):
- # Accelerate automatically handles gradient accumulation context
- with accelerator.accumulate(model):
- inputs = batch["input_ids"].to(device)
- targets = batch["target_ids"].to(device)
- legal_moves_mask = batch["legal_moves_mask"].to(device)
-
- logits = model(inputs)
-
- llm_loss = criterion(logits.flatten(0, 1), targets.flatten())
-
- probas = torch.nn.functional.softmax(logits, dim=-1)
-
- ilegal_moves_probas = probas * (1 - legal_moves_mask)
- l2_loss = torch.sum(ilegal_moves_probas**2, dim=-1).mean()
- loss = llm_loss + 10 * l2_loss
-
- loader.set_postfix({"llm_loss": llm_loss.item(), "l2_loss": l2_loss.item(), "total_loss": loss.item()})
-
- accelerator.backward(loss)
- optimizer.step()
- scheduler.step() # Update learning rate
- optimizer.zero_grad()
-
- total_loss += loss.item()
- total_llm_loss += llm_loss.item()
- total_l2_loss += l2_loss.item()
-
- # Log step metrics only on sync steps (when gradients are actually applied)
- if accelerator.sync_gradients:
- # Get current learning rate
- current_lr = scheduler.get_last_lr()[0]
-
- accelerator.log(
- {
- "train/step_loss": loss.item(),
- "train/step_llm_loss": llm_loss.item(),
- "train/step_l2_loss": l2_loss.item(),
- "train/learning_rate": current_lr,
- },
- step=global_step,
- )
-
- global_step += 1
-
- avg_train_loss = total_loss / len(train_dl)
- avg_train_llm_loss = total_llm_loss / len(train_dl)
- avg_train_l2_loss = total_l2_loss / len(train_dl)
- print(f"Epoch {epoch + 1}/{num_epochs}, Training Loss: {avg_train_loss:.4f}")
-
- # Validation
- model.eval()
- total_val_loss = 0.0
- total_val_llm_loss = 0.0
- total_val_l2_loss = 0.0
- with torch.no_grad():
- for batch in val_dl:
- inputs = batch["input_ids"].to(device)
- targets = batch["target_ids"].to(device)
- legal_moves_mask = batch["legal_moves_mask"].to(device)
-
- logits = model(inputs)
-
- llm_loss = criterion(logits.flatten(0, 1), targets.flatten())
-
- probas = torch.nn.functional.softmax(logits, dim=-1)
- ilegal_moves_probas = probas * (1 - legal_moves_mask)
- l2_loss = torch.sum(ilegal_moves_probas**2, dim=-1).mean()
- loss = llm_loss + l2_loss
-
- total_val_loss += loss.item()
- total_val_llm_loss += llm_loss.item()
- total_val_l2_loss += l2_loss.item()
-
- avg_val_loss = total_val_loss / len(val_dl)
- avg_val_llm_loss = total_val_llm_loss / len(val_dl)
- avg_val_l2_loss = total_val_l2_loss / len(val_dl)
- print(f"Epoch {epoch + 1}/{num_epochs}, Validation Loss: {avg_val_loss:.4f}")
-
- # Log epoch metrics
- accelerator.log(
- {
- "train/epoch_loss": avg_train_loss,
- "train/epoch_llm_loss": avg_train_llm_loss,
- "train/epoch_l2_loss": avg_train_l2_loss,
- "val/epoch_loss": avg_val_loss,
- "val/epoch_llm_loss": avg_val_llm_loss,
- "val/epoch_l2_loss": avg_val_l2_loss,
- "epoch": epoch + 1,
- },
- step=global_step,
- )
-
- # Save the best model
- if avg_val_loss < best_val_loss:
- best_val_loss = avg_val_loss
- unwrapped_model = accelerator.unwrap_model(model)
- torch.save(unwrapped_model.state_dict(), ROOT_PATH / "models/best_model.pth")
- print(f"Best model saved with validation loss: {best_val_loss:.4f}")
-
- # Log best metrics
- accelerator.log(
- {
- "best/val_loss": best_val_loss,
- "best/epoch": epoch + 1,
- },
- step=global_step,
- )
-
- # Testing
- model.eval()
- total_test_loss = 0.0
- total_test_llm_loss = 0.0
- total_test_l2_loss = 0.0
- with torch.no_grad():
- for batch in test_dl:
- inputs = batch["input_ids"].to(device)
- targets = batch["target_ids"].to(device)
- legal_moves_mask = batch["legal_moves_mask"].to(device)
-
- logits = model(inputs)
-
- llm_loss = criterion(logits.flatten(0, 1), targets.flatten())
-
- probas = torch.nn.functional.softmax(logits, dim=-1)
- ilegal_moves_probas = probas * (1 - legal_moves_mask)
- l2_loss = torch.sum(ilegal_moves_probas**2, dim=-1).mean()
- loss = llm_loss + l2_loss
-
- total_test_loss += loss.item()
- total_test_llm_loss += llm_loss.item()
- total_test_l2_loss += l2_loss.item()
-
- avg_test_loss = total_test_loss / len(test_dl)
- avg_test_llm_loss = total_test_llm_loss / len(test_dl)
- avg_test_l2_loss = total_test_l2_loss / len(test_dl)
- print(f"Test Loss: {avg_test_loss:.4f}")
-
- # Log test metrics
- accelerator.log(
- {
- "test/loss": avg_test_loss,
- "test/llm_loss": avg_test_llm_loss,
- "test/l2_loss": avg_test_l2_loss,
- },
- step=global_step,
- )
-
- # End tracking
- accelerator.end_training()
- print(f"\nTraining complete! Run: {run_name}")
- print(f"TensorBoard logs saved to 'logs/{run_name}/'")
- print("To view logs, run: tensorboard --logdir=logs")
- print(f"To view this run only: tensorboard --logdir=logs/{run_name}")
-
-
-if __name__ == "__main__":
- main()
diff --git a/chesstransformer/trainers/pos2move_v2_trainer.py b/chesstransformer/trainers/pos2move_v2_trainer.py
deleted file mode 100644
index ca89f7857d0b22778ef085d11ac28a0235a2ece9..0000000000000000000000000000000000000000
--- a/chesstransformer/trainers/pos2move_v2_trainer.py
+++ /dev/null
@@ -1,1128 +0,0 @@
-#!/usr/bin/env python3
-"""Training script for Pos2MoveV2.
-
-Trains the v2 transformer with GQA + QK-norm + chess-geometry attention bias,
-AlphaZero-style 64×73 action output and value head on HDF5 chess datasets.
-
-Usage:
- python src/chesstransformer/trainers/pos2move_v2_trainer.py --data data/elite_db.h5
-"""
-
-from pathlib import Path
-from datetime import datetime
-import argparse
-import json
-import os
-import random
-import shutil
-import sys
-
-# Make the checkout runnable without installing the package. `uv run` installs it into the
-# venv, so this never fires locally -- but a bare `git clone` on Kaggle or a rented box has
-# no chesstransformer on sys.path and every import below fails. scripts/build_shards.py,
-# scripts/verify_shards.py and tests/conftest.py already do this; the trainer did not, which
-# is exactly where it was first hit.
-sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
-
-import numpy as np
-import torch
-from torch.nn import functional as F
-from torch.utils.data import DataLoader, Subset, random_split
-from torch.optim.lr_scheduler import LambdaLR
-from tqdm.auto import tqdm
-import accelerate
-from accelerate.utils import broadcast_object_list, set_seed
-
-from chesstransformer.datasets.flat_shard_dataset import (
- FlatShardDataset,
- make_dataloader as make_shard_dataloader,
- scatter_legal_planes,
- worker_init_fn as shard_worker_init_fn,
-)
-from chesstransformer.datasets.h5_lichess_dataset import HDF5ChessDataset
-from chesstransformer.distributed_muon import (
- DistributedMuon,
- muon_momentum,
- muon_weight_decay,
-)
-from chesstransformer.models.tokenizer.position_tokenizer import PostionTokenizer
-from chesstransformer.models.transformer.pos2move_v2 import Pos2MoveV2, NUM_ACTION_PLANES
-
-
-#: Model sizes. DDP replicates weights *and* optimizer state on every rank.
-#:
-#: The static column is **measured**, not derived: one optimizer step then
-#: torch.cuda.memory_allocated(), on a 16 GB card. It comes out at ~8.4 bytes/param, not
-#: the ~18 a generic "fp32 master + two Adam moments + grads" estimate gives, because the
-#: optimizer is split -- 2D params go to Muon, which keeps a single momentum buffer, and
-#: the AdamW params (norms, biases, embeddings' non-2D parts) are a small remainder.
-#: An earlier version of this table used the 18 B/param estimate and overstated xl by 2.3x.
-#:
-#: Activations dominate and scale with micro-batch: measured ~76 MB/sample for `large`
-#: under torch.compile (~90 MB uncompiled). That is what actually decides whether a config
-#: fits, so the static column alone is not a fit test.
-#:
-#: ===== ============ ======== ============= =============================================
-#: name dim x layers params static/rank notes
-#: ===== ============ ======== ============= =============================================
-#: base 256 x 16 11.7 M 0.10 GB anything
-#: 46m 512 x 16 46.5 M 0.35 GB anything
-#: large 768 x 24 156.4 M 1.23 GB measured 8.5 GB/rank total at micro-batch 96
-#: on a T4; micro-batch 128 fits, 160 OOMs on 16 GB
-#: xl 1536 x 32 832.8 M ~6.4 GB extrapolated. Static alone would fit a 40 GB
-#: card under DDP; whether activations do is
-#: unmeasured -- measure before assuming FSDP.
-#: ===== ============ ======== ============= =============================================
-#:
-#: `xl` is deliberately oversized as a sharding/scaling testbed. It is not a bid for
-#: playing strength -- 46M already lost its head-to-head against v2.1 at 44.8%.
-PRESETS = {
- "base": {"embed_dim": 256, "num_layers": 16, "num_heads": 8},
- "46m": {"embed_dim": 512, "num_layers": 16, "num_heads": 8},
- "large": {"embed_dim": 768, "num_layers": 24, "num_heads": 12},
- "xl": {"embed_dim": 1536, "num_layers": 32, "num_heads": 16},
-}
-
-torch.set_float32_matmul_precision('high')
-
-
-def rank0_print(*a, **kw):
- """print() on the main process only.
-
- Keyed off the RANK env var rather than accelerator.is_main_process so it also works
- before the Accelerator is constructed -- dataset loading logs before that point. N
- ranks echoing the same line is noise; N ranks echoing *different* lines is a
- debugging trap, and the dataset banners are exactly where they would differ.
- """
- if int(os.environ.get("RANK", 0)) == 0:
- print(*a, **kw)
-
-
-def resolve_precision(requested: str) -> str:
- """Pick a mixed-precision mode this GPU can actually run.
-
- bf16 needs compute capability 8.0 (Ampere). Kaggle's T4 is Turing, 7.5 -- so the
- trainer's bf16 default silently does not apply there, and a run that looks fine is
- quietly not using tensor cores as intended. Fall back to fp16 (which Turing does have
- tensor cores for) and say so, rather than let the first multi-GPU session discover it.
- """
- # Accelerate spells "no mixed precision" as "no"; "fp32" is not a value it accepts.
- # Passing it through raises ValueError, which means --precision fp32 has never worked
- # and a CPU-only box (where the old code forced "fp32") could never start at all.
- if not torch.cuda.is_available() or os.environ.get("ACCELERATE_USE_CPU"):
- return "no"
- if requested == "fp32":
- return "no"
- if requested == "bf16" and not torch.cuda.is_bf16_supported():
- cap = torch.cuda.get_device_capability(0)
- rank0_print(f"WARNING: {torch.cuda.get_device_name(0)} is compute capability "
- f"{cap[0]}.{cap[1]}; bf16 needs 8.0+. Falling back to fp16 "
- f"(Accelerate adds a GradScaler). Pass --precision fp32 to opt out.")
- return "fp16"
- return requested
-
-
-def unpack_batch(batch, device):
- """Normalise a batch from either data path into the tensors the loss expects.
-
- The shard loader ships the legal mask as a ``(B, 64)`` uint16 index list — 64 KB per
- batch of 512 against 2.4 MB for the dense ``(64, 73)`` bool mask — so the mask is
- rebuilt here, on device, after the H2D copy. Doing it in the dataset would put the
- 2.4 MB straight back on the wire and undo the entire point of the shard format.
- """
- if "legal_idx" in batch:
- legal_planes = scatter_legal_planes(batch["legal_idx"].to(device, non_blocking=True))
- else:
- legal_planes = batch["legal_moves_planes"].to(device, non_blocking=True)
- return {
- "board": batch["position"].to(device, non_blocking=True).long(),
- "player": batch["is_white"].to(device, non_blocking=True).long(),
- "castling": batch["castling_rights"].to(device, non_blocking=True).long(),
- "en_passant": batch["en_passant_file"].to(device, non_blocking=True).long(),
- "from_sq": batch["from_square"].to(device, non_blocking=True).long(),
- "action_plane": batch["action_plane"].to(device, non_blocking=True).long(),
- "legal_planes": legal_planes,
- # Narrow on the wire (int8/int16 out of the shard), widened here so both data
- # paths hand compute_loss exactly the same dtypes.
- "result": batch["result"].to(device, non_blocking=True).long(),
- # Must stay bool: compute_loss uses it as a *mask* (`target_value[white_win &
- # is_white]`). An int tensor there would silently become fancy indexing and
- # scatter the value targets onto the wrong rows.
- "is_white": batch["is_white"].to(device, non_blocking=True).bool(),
- "move_number": batch["move_number"].to(device, non_blocking=True).long(),
- }
-
-
-def get_next_run_number(log_dir: str) -> int:
- log_path = Path(log_dir)
- if not log_path.exists():
- return 1
- run_numbers = []
- for d in log_path.iterdir():
- if d.is_dir() and d.name.startswith("run_"):
- try:
- run_numbers.append(int(d.name.split("_")[1]))
- except (IndexError, ValueError):
- continue
- return max(run_numbers, default=0) + 1
-
-
-def point_project_config_at(project_config, run_dir) -> None:
- """Move both project_dir and logging_dir onto the resolved run directory.
-
- Not ``set_directories(run_dir)``: that fills logging_dir only when it is None, and it is
- not. ``ProjectConfiguration.__post_init__`` already defaulted it to whatever project_dir
- the Accelerator was built with -- here the *parent*, logs/pos2move_v2, because the run
- name is not known until after the process group exists (see main()). Moving project_dir
- alone leaves the TensorBoard writer on the parent, where it appends the project name, so
- every run's events pile into one shared logs/pos2move_v2/pos2move_v2 instead of landing
- in the run directory beside that run's checkpoints.
- """
- project_config.project_dir = str(run_dir)
- project_config.logging_dir = str(run_dir)
-
-
-def compute_loss(
- move_logits,
- value,
- from_sq,
- action_plane,
- legal_moves_planes,
- result,
- is_white,
- move_number,
- value_loss_weight,
- value_rampup_halfmoves=40,
- label_smoothing: float = 0.0,
-):
- """Total loss = (label-smoothed) policy CE + value MSE.
-
- Returns
- -------
- total_loss : scalar tensor (the only thing that gets backprop'd)
- metrics : dict of detached scalar tensors for logging
- - ce : standard cross-entropy on the full action space
- - legal_ce : cross-entropy after masking to legal moves only
- - value_loss : weighted MSE on the value head
- - acc : top-1 accuracy on the full action space
- - legal_acc : top-1 accuracy after masking to legal moves only
- """
- B = move_logits.size(0)
-
- # Flatten (B, 64, 73) -> (B, 64*73). Upcast to fp32 for loss stability under bf16.
- flat_logits = move_logits.reshape(B, -1).float()
- flat_target = from_sq * NUM_ACTION_PLANES + action_plane
- flat_mask = legal_moves_planes.reshape(B, -1).float()
-
- # ── Policy loss: label-smoothed CE over LEGAL moves only ────────────
- if label_smoothing > 0:
- eps = label_smoothing
- legal_count = flat_mask.sum(dim=-1, keepdim=True).clamp(min=1.0)
- smooth = eps / legal_count
- target_dist = flat_mask * smooth
- idx = flat_target.unsqueeze(1)
- target_dist.scatter_add_(1, idx, torch.full_like(idx, 1.0 - eps, dtype=target_dist.dtype))
- log_pred = F.log_softmax(flat_logits, dim=-1)
- policy_loss = F.kl_div(log_pred, target_dist, reduction="batchmean")
- else:
- policy_loss = F.cross_entropy(flat_logits, flat_target)
-
- # ── Value loss: per-sample MSE, ramped up by move number ────────────
- target_value = torch.zeros_like(value)
- white_win = result == 1
- black_win = result == 2
- target_value[white_win & is_white] = 1.0
- target_value[white_win & ~is_white] = -1.0
- target_value[black_win & is_white] = -1.0
- target_value[black_win & ~is_white] = 1.0
-
- progress = (move_number.float() / value_rampup_halfmoves).clamp(max=1.0)
- per_sample_value_loss = (value.float() - target_value) ** 2
- value_loss = (progress * per_sample_value_loss).mean()
-
- total_loss = policy_loss + value_loss_weight * value_loss
-
- # ── Metrics (no grad) ───────────────────────────────────────────────
- with torch.no_grad():
- ce_metric = F.cross_entropy(flat_logits, flat_target)
-
- masked_logits = flat_logits.masked_fill(flat_mask == 0, float("-inf"))
- legal_ce_metric = F.cross_entropy(masked_logits, flat_target)
-
- preds = flat_logits.argmax(dim=-1)
- acc = (preds == flat_target).float().mean()
-
- legal_preds = masked_logits.argmax(dim=-1)
- legal_acc = (legal_preds == flat_target).float().mean()
-
- metrics = {
- "ce": ce_metric,
- "legal_ce": legal_ce_metric,
- "value_loss": value_loss.detach(),
- "acc": acc,
- "legal_acc": legal_acc,
- }
- return total_loss, metrics
-
-
-def create_lr_scheduler(optimizer, warmup_steps, total_steps, final_lr_ratio,
- schedule="linear", warmdown_ratio=0.2):
- """Build the LR multiplier schedule.
-
- ``wsd`` -- warmup, then **stable at peak**, then a linear warmdown over the final
- ``warmdown_ratio`` of the run (nanochat's ``get_lr_multiplier``). ``linear`` -- warmup
- then decay across the entire remainder, which is what this trainer used to do.
-
- WSD is the default for two reasons. It decouples the horizon from the schedule: peak LR
- is held flat, so a run can be stopped or extended at any point in the stable phase and
- only the warmdown re-run, where a decay-from-peak schedule bakes ``total_steps`` into
- every step it takes. And it is the phase structure Muon's momentum schedule was designed
- against -- momentum warms down *during the LR warmdown*, which only exists here.
-
- Note the asymmetry in defaults, which is deliberate: the CLI defaults to ``wsd`` for new
- runs, but this function defaults to ``linear``. scheduler_config dicts persisted in
- checkpoints from before this change carry no ``schedule`` key, and are replayed through
- here on resume -- so the function default is what those runs get. Defaulting it to
- ``wsd`` would silently change the LR shape of an in-flight run at the point it resumed.
- """
- warmup_steps = max(1, warmup_steps)
- total_steps = max(warmup_steps + 1, total_steps)
- warmdown = max(1, round(warmdown_ratio * total_steps))
-
- def lr_lambda(step):
- if step < warmup_steps:
- return float(step + 1) / float(warmup_steps)
- if schedule == "wsd":
- if step <= total_steps - warmdown:
- return 1.0
- progress = max(0.0, min(1.0, (total_steps - step) / warmdown))
- return progress * 1.0 + (1 - progress) * final_lr_ratio
- progress = (step - warmup_steps + 1) / float(max(1, total_steps - warmup_steps))
- return 1.0 - (1.0 - final_lr_ratio) * min(1.0, progress)
-
- return LambdaLR(optimizer, lr_lambda)
-
-
-def save_trainer_state(checkpoint_dir, data_pass, global_step, best_val_loss, scheduler_config=None):
- # "epoch" is written alongside "data_pass" only so a checkpoint from this trainer still
- # loads in anything that predates the step paradigm. Nothing here reads it back.
- state = {"data_pass": data_pass, "epoch": data_pass,
- "global_step": global_step, "best_val_loss": best_val_loss}
- if scheduler_config:
- state["scheduler_config"] = scheduler_config
- with (Path(checkpoint_dir) / "trainer_state.json").open("w") as f:
- json.dump(state, f)
-
-
-def load_trainer_state(checkpoint_dir):
- path = Path(checkpoint_dir) / "trainer_state.json"
- if not path.exists():
- return None
- with path.open() as f:
- return json.load(f)
-
-
-# ── EMA helpers ──────────────────────────────────────────────────────────
-# EMA state is kept in fp32 to avoid bf16 precision loss on the
-# (1 - decay) * param update term (which is ~0.001 * param at decay=0.999
-# and would round to zero for many small values in bf16).
-
-def create_ema_state(model):
- return {name: p.data.detach().float().clone() for name, p in model.named_parameters()}
-
-
-@torch.no_grad()
-def update_ema(model, ema_state, decay):
- for name, param in model.named_parameters():
- # Cast live param to fp32 for the lerp; ema_state stays fp32.
- ema_state[name].lerp_(param.data.float(), 1.0 - decay)
-
-
-def swap_ema_weights(model, ema_state):
- # Backup current (compute-dtype) weights into ema_state, load EMA weights
- # into the model — cast to the model's dtype on the way in.
- for name, param in model.named_parameters():
- tmp = param.data.detach().float().clone()
- param.data.copy_(ema_state[name].to(param.dtype))
- ema_state[name].copy_(tmp)
-
-
-def save_ema_state(ema_state, path):
- torch.save(ema_state, path)
-
-
-def load_ema_state(path, device):
- return torch.load(path, map_location=device, weights_only=True)
-
-
-def get_raw_model(model, accelerator):
- m = accelerator.unwrap_model(model)
- if hasattr(m, "_orig_mod"):
- m = m._orig_mod
- return m
-
-
-def cleanup_old_checkpoints(checkpoint_dir, max_checkpoints):
- checkpoints = sorted(
- [d for d in Path(checkpoint_dir).iterdir() if d.is_dir() and d.name.startswith("checkpoint_step_")],
- key=lambda x: int(x.name.split("_")[-1]),
- )
- while len(checkpoints) > max_checkpoints:
- shutil.rmtree(checkpoints.pop(0))
-
-def main():
- default_data = Path(__file__).parents[3] / "data" / "elite_db.h5"
-
- parser = argparse.ArgumentParser(description="Train Pos2MoveV2 (clean)")
- # Data
- parser.add_argument("--data", type=str, default=str(default_data))
- parser.add_argument("--shards", type=str, default=None,
- help="Directory of flat shards from scripts/build_shards.py. Bypasses "
- "--data/--min-elo/--sample-weighting entirely: the sampling "
- "distribution is frozen into the shard at build time.")
- parser.add_argument("--min-elo", type=int, default=None)
- parser.add_argument("--max-elo", type=int, default=None)
- parser.add_argument("--sample-weighting", type=str, default="middlegame", choices=["uniform", "middlegame"])
- parser.add_argument("--skip-opening-plies", type=int, default=0)
- parser.add_argument("--max-val-samples", type=int, default=50_000,
- help="Cap on val/test size. The HDF5 path has always applied this; "
- "the shard path needs it too since a reserved shard is much larger.")
- # Training
- parser.add_argument("--batch-size", type=int, default=1024)
- parser.add_argument("--grad-accum", type=int, default=4)
- parser.add_argument("--epochs", type=int, default=None,
- help="DEPRECATED and ignored. Training length is --max-steps. Kept so "
- "existing invocations do not hard-fail; it warns and does nothing.")
- parser.add_argument("--lr", type=float, default=1e-3)
- parser.add_argument("--lr-muon", type=float, default=2e-3)
- parser.add_argument("--lr-embedding", type=float, default=4e-4)
- parser.add_argument("--lr-head", type=float, default=None)
- parser.add_argument("--warmup-steps", type=int, default=2000)
- parser.add_argument("--final-lr-ratio", type=float, default=0.05)
- parser.add_argument("--lr-schedule", type=str, default="wsd", choices=["wsd", "linear"],
- help="'wsd' = warmup / stable at peak / linear warmdown over the last "
- "--warmdown-ratio of the run. 'linear' = the previous behaviour, "
- "decay from peak across the whole remainder.")
- parser.add_argument("--warmdown-ratio", type=float, default=0.2,
- help="Fraction of the run spent decaying, for --lr-schedule wsd. Muon's "
- "momentum warmdown is aligned to the same window.")
- parser.add_argument("--max-grad-norm", type=float, default=1.0)
- parser.add_argument("--weight-decay", type=float, default=0.1)
- # Loss weights
- parser.add_argument("--value-loss-weight", type=float, default=5.0)
- parser.add_argument("--label-smoothing", type=float, default=0.1)
- # Model
- parser.add_argument("--preset", type=str, default="base", choices=list(PRESETS),
- help="Model size. See PRESETS for per-GPU DDP memory. Individual "
- "--embed-dim/--num-layers/--num-heads override the preset.")
- parser.add_argument("--embed-dim", type=int, default=None)
- parser.add_argument("--num-layers", type=int, default=None)
- parser.add_argument("--num-heads", type=int, default=None)
- parser.add_argument("--dropout", type=float, default=0.05)
- parser.add_argument("--layer-drop", type=float, default=0.1,
- help="Stochastic depth rate. Linearly scales from 0 (first layer) to this value (last layer). Try 0.1.") # Checkpointing
- parser.add_argument("--max-steps", type=int, default=50_000,
- help="Total optimizer steps to train for. This is THE training-length "
- "knob: it bounds the loop and sets the LR decay horizon, and it "
- "means the same thing on both data paths and at any world size.")
- parser.add_argument("--save-every", type=int, default=None,
- help="DEPRECATED and ignored (was epoch-based checkpointing). "
- "Use --save-steps.")
- parser.add_argument("--save-steps", type=int, default=5000)
- parser.add_argument("--eval-steps", type=int, default=1000,
- help="Validate every N optimizer steps. Val loss selects the best "
- "checkpoint, so this sets the resolution of that choice. 0 "
- "disables periodic validation (a final one still runs).")
- parser.add_argument("--max-checkpoints", type=int, default=5)
- parser.add_argument("--resume-from", type=str, default=None)
- # EMA
- parser.add_argument("--ema-decay", type=float, default=0.999)
- # Muon
- parser.add_argument("--distributed-muon", action=argparse.BooleanOptionalAction, default=True,
- help="Partition Newton-Schulz across ranks by whole parameter, so the "
- "NS work and momentum state are divided instead of every rank "
- "redundantly computing the same thing. No-op at world size 1.")
- parser.add_argument("--muon-schedules", action=argparse.BooleanOptionalAction, default=True,
- help="Schedule Muon momentum (0.85->0.97, down to 0.90 during warmdown) "
- "and cosine-decay its weight decay to zero, as nanochat does.")
- # Misc
- parser.add_argument("--seed", type=int, default=42)
- parser.add_argument("--precision", type=str, default="bf16", choices=["bf16", "fp16", "fp32"])
- parser.add_argument("--num-workers", type=int, default=12)
- # store_true with default=True could never be turned off. BooleanOptionalAction
- # gives a real --no-compile, which CPU/gloo testing and debugging both need.
- parser.add_argument("--compile", action=argparse.BooleanOptionalAction, default=True)
- parser.add_argument("--compile-mode", type=str, default="default",
- choices=["default", "reduce-overhead", "max-autotune"])
- parser.add_argument("--compile-order", type=str, default="auto",
- choices=["auto", "before-prepare", "after-prepare"],
- help="Whether torch.compile runs before or after accelerator.prepare(). "
- "'auto' = before on a single process (the ordering the 2.2x "
- "single-GPU speedup was measured with), after under DDP so "
- "Dynamo's DDPOptimizer engages. See the note at the call site.")
- args = parser.parse_args()
-
- # Epochs are gone as a unit of training length. They were never comparable between the
- # two data paths — the HDF5 dataset is indexed by *game* (738 optimizer steps/epoch) and
- # the shard dataset by *sample* (11,495 for elite_k16, 50,505 for full_k4) — so the same
- # --epochs N bought a 15-68x different amount of training, LR decay horizon and
- # validation cadence depending only on which --data/--shards flag you passed.
- for dead, replacement in (("epochs", "--max-steps"), ("save_every", "--save-steps")):
- if getattr(args, dead) is not None:
- rank0_print(f"WARNING: --{dead.replace('_', '-')} is deprecated and ignored; "
- f"use {replacement}.")
- if args.max_steps <= 0:
- parser.error("--max-steps must be > 0")
-
- # Preset supplies the geometry; an explicit flag always wins over it.
- preset = PRESETS[args.preset]
- for flag, key in (("embed_dim", "embed_dim"), ("num_layers", "num_layers"),
- ("num_heads", "num_heads")):
- if getattr(args, flag) is None:
- setattr(args, flag, preset[key])
-
- set_seed(args.seed)
- random.seed(args.seed)
- np.random.seed(args.seed)
- torch.manual_seed(args.seed)
- torch.cuda.manual_seed_all(args.seed)
-
- # ── Logging ──────────────────────────────────────────────────────────
- # The run directory has to be decided by ONE rank and told to the others. Every rank
- # scanning logs/ for the next free run number is a race: they can pick the same number
- # (two ranks writing one run) or different ones (N runs, N-1 of them silently empty),
- # and mkdir(exist_ok=False) then crashes whichever rank loses. The timestamp in the
- # name makes independent derivation impossible too -- ranks start milliseconds apart.
- #
- # Ordering note: the Accelerator must exist before the broadcast (it is what sets up
- # the process group), but init_trackers needs the run dir. Hence build it against the
- # parent, then point it at the resolved run dir with set_directories().
- log_dir = Path("logs") / "pos2move_v2"
- log_dir.mkdir(parents=True, exist_ok=True)
-
- precision = resolve_precision(args.precision)
- accelerator = accelerate.Accelerator(
- log_with="tensorboard",
- project_dir=str(log_dir),
- mixed_precision=precision,
- gradient_accumulation_steps=args.grad_accum,
- )
-
- if accelerator.is_main_process:
- run_number = get_next_run_number(str(log_dir))
- run_name = f"run_{run_number:03d}_{datetime.now().strftime('%Y%m%d_%H%M%S')}"
- else:
- run_name = None
- run_name = broadcast_object_list([run_name], from_process=0)[0]
-
- log_path = log_dir / run_name
- checkpoint_dir = log_path / "checkpoints"
- if accelerator.is_main_process:
- checkpoint_dir.mkdir(parents=True, exist_ok=False)
- # Every rank must see the directory before anyone writes a checkpoint into it.
- accelerator.wait_for_everyone()
- point_project_config_at(accelerator.project_configuration, log_path)
-
- # ── Dataset ──────────────────────────────────────────────────────────
- use_shards = args.shards is not None
- if use_shards:
- # Flat memmap shards: the replay, tokenize and legal-move enumeration were all
- # paid once at build time, and the val/test split is by whole shard (== whole
- # game range) rather than by sample, so no game straddles the split.
- train_set = FlatShardDataset(args.shards, "train")
- val_set = FlatShardDataset(args.shards, "val")
- test_set = FlatShardDataset(args.shards, "test")
-
- # Whole reserved shards hold far more than validation needs (478k samples at
- # K=16). Cap them the way the HDF5 path does, with a fixed seed so val loss stays
- # comparable across runs — it is what selects the best checkpoint.
- def cap(ds, name):
- if len(ds) <= args.max_val_samples:
- return ds
- g = np.random.default_rng(0xC0FFEE)
- keep = np.sort(g.choice(len(ds), size=args.max_val_samples, replace=False))
- rank0_print(f" {name}: capped {len(ds):,} -> {args.max_val_samples:,} samples")
- return Subset(ds, keep.tolist())
-
- val_set = cap(val_set, "val")
- test_set = cap(test_set, "test")
-
- # The shard loaders bypass accelerator.prepare() (see the note at prepare()), so
- # rank splitting is NOT automatic here the way it is for the HDF5 path -- it has to
- # be asked for. Without this every rank draws the identical batches from the same
- # seeded RandomSampler, and DDP degenerates into N ranks computing one gradient N
- # times: the loss curve looks healthy, the effective batch never grows, and the
- # extra GPUs buy nothing.
- # This is exactly why the Accelerator is constructed above the dataset section:
- # DistributedSampler requires an initialised process group, and the Accelerator is
- # what initialises it. Building the loaders first raises "Default process group has
- # not been initialized".
- world_size = accelerator.num_processes
- train_loader = make_shard_dataloader(train_set, batch_size=args.batch_size, shuffle=True,
- num_workers=args.num_workers, drop_last=True,
- seed=args.seed, distributed=world_size > 1)
- # Val/test are deliberately NOT sharded. Each rank evaluates the whole (capped)
- # set and therefore computes an identical val loss, so best-model selection agrees
- # on every rank by construction and needs no gather or padding-trim. The cost is
- # N x redundant work on <=10k samples -- about two seconds -- against a class of
- # bug where ranks disagree about which checkpoint is best and race to write it.
- val_loader = make_shard_dataloader(val_set, batch_size=args.batch_size, shuffle=False,
- num_workers=args.num_workers)
- test_loader = make_shard_dataloader(test_set, batch_size=args.batch_size, shuffle=False,
- num_workers=args.num_workers)
- vocab_size = PostionTokenizer().vocab_size
- rank0_print(f"Shards: {args.shards} (K={train_set.meta['k']}, "
- f"mode={train_set.meta['sample_mode']}, seed={train_set.meta['seed']})")
- else:
- dataset = HDF5ChessDataset(
- hdf5_path=args.data,
- min_elo=args.min_elo,
- max_elo=args.max_elo,
- sample_weighting=args.sample_weighting,
- skip_opening_plies=args.skip_opening_plies,
- )
-
- val_size = min(int(0.1 * len(dataset)), args.max_val_samples)
- test_size = min(int(0.1 * len(dataset)), args.max_val_samples)
- train_size = len(dataset) - val_size - test_size
- train_set, val_set, test_set = random_split(dataset, [train_size, val_size, test_size])
-
- # pin_memory lets the H2D copy run on the DMA engine and overlap with compute;
- # without it `.to(device, non_blocking=True)` is silently synchronous.
- train_loader = DataLoader(train_set, batch_size=args.batch_size, shuffle=True,
- num_workers=args.num_workers, drop_last=True,
- persistent_workers=True, prefetch_factor=4, pin_memory=True,
- worker_init_fn=shard_worker_init_fn)
- val_loader = DataLoader(val_set, batch_size=args.batch_size, shuffle=False,
- num_workers=args.num_workers, pin_memory=True,
- worker_init_fn=shard_worker_init_fn)
- test_loader = DataLoader(test_set, batch_size=args.batch_size, shuffle=False,
- num_workers=args.num_workers, pin_memory=True,
- worker_init_fn=shard_worker_init_fn)
- vocab_size = dataset.position_tokenizer.vocab_size
-
- rank0_print(f"Train: {len(train_set):,} | Val: {len(val_set):,} | Test: {len(test_set):,}")
-
- # ── Model ────────────────────────────────────────────────────────────
- model_config = {
- "vocab_size": vocab_size,
- "embed_dim": args.embed_dim,
- "nb_transformer_layers": args.num_layers,
- "num_heads": args.num_heads,
- "dropout": args.dropout,
- "kvq_bias": False,
- "layer_drop": args.layer_drop,
- }
- if accelerator.is_main_process:
- with (log_path / "model_config.json").open("w") as f:
- json.dump(model_config, f, indent=2)
-
- # Effective batch scales with world size: every rank contributes its own micro-batch
- # to the same all-reduced gradient. Reporting it without num_processes understates the
- # real batch by N, which is exactly the number linear LR scaling is derived from.
- effective_bs = args.batch_size * args.grad_accum * accelerator.num_processes
- rank0_print(f"Effective batch size: {effective_bs} (micro={args.batch_size} × "
- f"accum={args.grad_accum} × ranks={accelerator.num_processes})")
- accelerator.init_trackers(
- project_name="pos2move_v2",
- config={
- "batch_size": args.batch_size,
- "grad_accum": args.grad_accum,
- "effective_batch_size": effective_bs,
- "max_steps": args.max_steps,
- "lr": args.lr,
- "weight_decay": args.weight_decay,
- "value_loss_weight": args.value_loss_weight,
- **model_config,
- },
- )
-
- device = accelerator.device
- rank0_print(f"Device: {device} | Precision: {precision} | Logging to: {log_path}")
-
- model = Pos2MoveV2(**model_config)
- total_params = sum(p.numel() for p in model.parameters())
- rank0_print(f"Model parameters: {total_params:,}")
- model.to(device)
-
- # torch.compile vs accelerator.prepare() ordering is not cosmetic under DDP.
- #
- # Dynamo's DDPOptimizer (torch._dynamo.backends.distributed, on by default) splits the
- # graph at DDP's gradient all-reduce bucket boundaries so the collectives can overlap
- # with backward compute. Its own docs: "DDPOptimizer applies when dynamo compiles
- # models wrapped in DistributedDataParallel". Compiling first gives DDP(OptimizedModule)
- # -- the wrap happens outside the compiled graph, DDPOptimizer never sees a DDP module,
- # and the overlap is lost. Compiling after prepare() gives compile(DDP(model)), which is
- # the shape it wants.
- #
- # Single process has no buckets to align to, so 'before' is kept there: it is the
- # ordering the measured 2.2x speedup and the compile-time numbers come from.
- compile_order = args.compile_order
- if compile_order == "auto":
- compile_order = "after-prepare" if accelerator.num_processes > 1 else "before-prepare"
-
- def maybe_compile(m, when):
- if args.compile and compile_order == when:
- rank0_print(f"torch.compile (mode={args.compile_mode}, {when})")
- return torch.compile(m, mode=args.compile_mode, dynamic=False) # shapes are static
- return m
-
- model = maybe_compile(model, "before-prepare")
-
- # ── Optimizer (Muon for 2D weights, AdamW for embeddings/heads/1D) ──
- lr_emb = args.lr_embedding or args.lr
- lr_head = args.lr_head or args.lr
-
- muon_params = []
- adamw_emb_params = []
- adamw_head_params = []
- adamw_other_params = []
-
- for name, param in model.named_parameters():
- if "embedding" in name:
- adamw_emb_params.append(param)
- elif "move_head" in name or "value_head" in name:
- adamw_head_params.append(param)
- elif param.ndim == 2:
- muon_params.append(param)
- else:
- adamw_other_params.append(param)
-
- rank0_print(f"LR: emb={lr_emb:.2e} | muon={args.lr_muon:.2e} | head={lr_head:.2e}")
- rank0_print(
- f"Params: emb={sum(p.numel() for p in adamw_emb_params):,} | "
- f"muon={sum(p.numel() for p in muon_params):,} | "
- f"adamw_other={sum(p.numel() for p in adamw_other_params):,} | "
- f"head={sum(p.numel() for p in adamw_head_params):,}"
- )
-
- # DistributedMuon degenerates to exactly torch.optim.Muon at world size 1 (verified in
- # tests/test_distributed_muon.py), so there is one code path rather than two.
- muon_cls = DistributedMuon if args.distributed_muon else torch.optim.Muon
- muon_optimizer = muon_cls(
- muon_params,
- lr=args.lr_muon,
- momentum=0.95,
- weight_decay=args.weight_decay,
- )
- if args.distributed_muon and accelerator.num_processes > 1:
- rank0_print(muon_optimizer.sharding_summary().split(" | ")[0])
- adamw_optimizer = torch.optim.AdamW([
- {"params": adamw_emb_params, "lr": lr_emb, "weight_decay": args.weight_decay},
- {"params": adamw_other_params, "lr": args.lr, "weight_decay": 0.0},
- {"params": adamw_head_params, "lr": lr_head, "weight_decay": 0.0},
- ])
-
- if use_shards:
- # The shard loaders are deliberately kept out of prepare(). They run with
- # batch_size=None and a BatchSampler so that a whole index list reaches
- # __getitem__ in one call; Accelerate reads batch_size=None as "this iterable
- # already yields batches" and would re-shard at the wrong granularity. Rank
- # splitting is done by the DistributedSampler inside make_shard_dataloader, and
- # the H2D copy by unpack_batch.
- model, muon_optimizer, adamw_optimizer = accelerator.prepare(
- model, muon_optimizer, adamw_optimizer
- )
- else:
- model, muon_optimizer, adamw_optimizer, train_loader, val_loader, test_loader = accelerator.prepare(
- model, muon_optimizer, adamw_optimizer, train_loader, val_loader, test_loader
- )
-
- model = maybe_compile(model, "after-prepare")
-
- # ── EMA ──────────────────────────────────────────────────────────────
- use_ema = args.ema_decay > 0
- ema_state = None
- if use_ema:
- ema_state = create_ema_state(get_raw_model(model, accelerator))
- rank0_print(f"EMA enabled (decay={args.ema_decay})")
-
- # ── Resume ───────────────────────────────────────────────────────────
- best_val_loss = float("inf")
- start_pass = 0
- global_step = 0
- trainer_state = None
-
- if args.resume_from:
- trainer_state = load_trainer_state(args.resume_from)
- if trainer_state:
- # "epoch" is the pre-step-paradigm key; read it so old checkpoints still resume.
- start_pass = trainer_state.get("data_pass", trainer_state.get("epoch", 0)) + 1
- global_step = trainer_state["global_step"]
- best_val_loss = trainer_state["best_val_loss"]
-
- # ── Scheduler ────────────────────────────────────────────────────────
- total_steps = args.max_steps
- scheduler_config = {
- "warmup_steps": args.warmup_steps,
- "total_steps": total_steps,
- "final_lr_ratio": args.final_lr_ratio,
- "schedule": args.lr_schedule,
- "warmdown_ratio": args.warmdown_ratio,
- }
- if trainer_state and "scheduler_config" in trainer_state:
- # A resume keeps the schedule it started with, so the LR curve stays continuous.
- # That means --max-steps still bounds the *loop* but no longer sets the LR horizon,
- # and the two can silently disagree. Say so rather than let it be discovered later.
- scheduler_config = trainer_state["scheduler_config"]
- resumed_total = scheduler_config.get("total_steps")
- if resumed_total != args.max_steps:
- rank0_print(f"WARNING: resuming with the checkpoint's LR schedule "
- f"(total_steps={resumed_total:,}) while --max-steps={args.max_steps:,} "
- f"bounds the loop. The LR decays over {resumed_total:,} steps regardless.")
- total_steps = resumed_total
-
- # An "epoch" is not comparable between the two data paths: the HDF5 dataset is indexed
- # by *game* (738 optimizer steps/epoch) and the shard dataset by *sample* (11.5k for
- # elite_k16, 50.5k for full_k4). So --epochs N silently buys a 15-68x longer LR decay
- # on shards, and epoch-end validation fires that much less often. Print both so the
- # schedule is never a surprise, and say so out loud when --max-steps is not pinning it.
- steps_per_pass = len(train_loader) // max(1, args.grad_accum)
- samples_seen = args.max_steps * effective_bs
- warmdown_note = (f"warmdown {round(args.warmdown_ratio * total_steps):,} | "
- if args.lr_schedule == "wsd" else "")
- rank0_print(f"Schedule: {total_steps:,} optimizer steps | {args.lr_schedule} | "
- f"warmup {args.warmup_steps:,} | {warmdown_note}"
- f"eval every {args.eval_steps:,} | save every {args.save_steps:,}")
- rank0_print(f" one pass over the training data = {steps_per_pass:,} steps/rank "
- f"({args.max_steps / max(1, steps_per_pass):.2f} passes, "
- f"{samples_seen:,} samples consumed)")
-
- muon_scheduler = create_lr_scheduler(muon_optimizer, **scheduler_config)
- adamw_scheduler = create_lr_scheduler(adamw_optimizer, **scheduler_config)
- accelerator.register_for_checkpointing(muon_scheduler, adamw_scheduler)
-
- if args.resume_from:
- accelerator.load_state(args.resume_from)
- if use_ema:
- ema_path = Path(args.resume_from) / "ema_state.pt"
- if ema_path.exists():
- ema_state = load_ema_state(ema_path, device)
- rank0_print(f"Restored EMA state from {ema_path}")
- else:
- ema_state = create_ema_state(get_raw_model(model, accelerator))
- rank0_print("EMA state not found in checkpoint, re-initialized from model")
- rank0_print(f"Resumed from {args.resume_from} (step {global_step:,}/{args.max_steps:,}, "
- f"data pass {start_pass})")
-
- # ── Validation ───────────────────────────────────────────────────────
- # Runs on the --eval-steps counter. It used to run only at epoch end, which was a fine
- # cadence on the HDF5 path (738 steps/epoch) and a bad one on shards (11,495 steps for
- # elite_k16, 50,505 for full_k4) — the dataset index is a sample there, not a game. Val
- # loss selects the best checkpoint, so that gap mattered. Hence the step paradigm.
-
- def run_validation():
- """Evaluate on val_loader (under EMA weights if enabled). Restores train mode."""
- model.eval()
- unwrapped = get_raw_model(model, accelerator)
- if use_ema:
- swap_ema_weights(unwrapped, ema_state)
-
- # No cross-rank reduction here on purpose: val_loader is replicated, not sharded,
- # so every rank walks the identical set and arrives at the identical number. That
- # is what keeps best-model selection consistent across ranks without a gather.
- sums = dict(loss=0.0, ce=0.0, legal_ce=0.0, value=0.0, correct=0, legal_correct=0, total=0)
- with torch.no_grad():
- for batch in tqdm(val_loader, desc="Validation", leave=False,
- disable=not accelerator.is_main_process):
- b = unpack_batch(batch, device)
- move_logits, value = model(b["board"], b["player"], b["castling"], b["en_passant"])
- loss, metrics = compute_loss(
- move_logits, value,
- b["from_sq"], b["action_plane"],
- b["legal_planes"], b["result"], b["is_white"], b["move_number"],
- args.value_loss_weight,
- )
- B = move_logits.size(0)
- sums["loss"] += loss.item() * B
- sums["ce"] += metrics["ce"].item() * B
- sums["legal_ce"] += metrics["legal_ce"].item() * B
- sums["value"] += metrics["value_loss"].item() * B
- sums["correct"] += int(metrics["acc"].item() * B)
- sums["legal_correct"] += int(metrics["legal_acc"].item() * B)
- sums["total"] += B
-
- if use_ema:
- swap_ema_weights(unwrapped, ema_state)
- model.train()
-
- n = max(1, sums["total"])
- return {
- "val/loss": sums["loss"] / n,
- "val/ce": sums["ce"] / n,
- "val/legal_ce": sums["legal_ce"] / n,
- "val/value_loss": sums["value"] / n,
- "val/accuracy": sums["correct"] / n,
- "val/legal_accuracy": sums["legal_correct"] / n,
- }
-
- def record_validation(vals, data_pass, extra_log=None, label=""):
- """Log, print, and update best-model selection. Returns True if it was a new best."""
- nonlocal best_val_loss
- log_dict = dict(extra_log or {})
- log_dict.update(vals)
- if use_ema:
- log_dict["val/ema_loss"] = vals["val/loss"]
- log_dict["val/ema_accuracy"] = vals["val/accuracy"]
- accelerator.log(log_dict, step=global_step)
-
- ema_tag = " (EMA)" if use_ema else ""
- rank0_print(f"{label}val_loss={vals['val/loss']:.4f}{ema_tag} "
- f"| val_acc={vals['val/accuracy']:.4f} "
- f"| val_legal_acc={vals['val/legal_accuracy']:.4f}")
-
- if vals["val/loss"] < best_val_loss:
- best_val_loss = vals["val/loss"]
- best_path = checkpoint_dir / "best_model"
- accelerator.save_state(str(best_path))
- save_trainer_state(best_path, data_pass, global_step, best_val_loss, scheduler_config)
- if use_ema and accelerator.is_main_process:
- save_ema_state(ema_state, best_path / "ema_state.pt")
- rank0_print(f" -> New best model (val_loss={vals['val/loss']:.4f})")
- return True
- return False
-
- # ── Training loop (step-driven) ──────────────────────────────────────
- # The loop is bounded by optimizer steps and cycles the dataloader as many times as
- # that takes. Nothing downstream keys off "epoch" any more: LR decay, validation and
- # checkpointing are all on step counters, which mean the same thing whichever dataset
- # is mounted and at any world size.
-
- def start_pass_shuffle(loader, n):
- """Reshuffle for a fresh pass over the data.
-
- DistributedSampler needs set_epoch or every rank replays the identical order on
- every pass. RandomSampler carries its generator across iterators and reshuffles on
- its own, so this is a no-op there.
- """
- for obj in (loader,
- getattr(loader, "sampler", None),
- getattr(getattr(loader, "sampler", None), "sampler", None)):
- if obj is not None and hasattr(obj, "set_epoch"):
- obj.set_epoch(n)
- return
-
- def new_window():
- """Train metrics accumulated since the last validation, sample-weighted."""
- return dict(loss=0.0, ce=0.0, legal_ce=0.0, value=0.0, acc=0.0, legal_acc=0.0, n=0)
-
- WINDOW_KEYS = ("loss", "ce", "legal_ce", "value", "acc", "legal_acc")
-
- def flush_window(w):
- """Average the window across ranks, not just within one.
-
- Each rank trains on a different slice, so these sums are rank-local: rank 0's
- numbers alone are a 1/N sample of the step. Reducing sum-of-(metric x batch) and
- sum-of-batch separately, then dividing, is exact regardless of how unevenly the
- last batches fall -- which is why this does not need gather_for_metrics and its
- padding-trim bookkeeping (that machinery also would not work here, since the shard
- loaders never went through prepare()).
- """
- vals = torch.tensor([w[k] for k in WINDOW_KEYS] + [float(w["n"])],
- dtype=torch.float64, device=accelerator.device)
- if accelerator.num_processes > 1:
- vals = accelerator.reduce(vals, reduction="sum")
- total = max(1.0, vals[-1].item())
- return {f"train/{'value_loss' if k == 'value' else k}": vals[i].item() / total
- for i, k in enumerate(WINDOW_KEYS)}
-
- last_val_step = -1
-
- def validate_now(window, data_pass, label_prefix):
- """Flush the train window, validate, log both against the same step.
-
- No-ops if this step has already been validated, which happens whenever max_steps is
- an exact multiple of eval_steps and the final validation lands on the same step as
- the last periodic one. Worth guarding rather than tolerating: on the HDF5 path the
- two calls do not even agree, because ``HDF5ChessDataset.__getitem__`` samples a
- fresh ply per call, so its val set re-rolls on every pass. (The shard val set is
- fixed, so shard val loss is comparable across evaluations and HDF5 val loss is not
- — worth remembering when reading either curve.)
- """
- nonlocal last_val_step
- if global_step == last_val_step:
- return
- last_val_step = global_step
- extra = flush_window(window) if window["n"] else {}
- extra["data_pass"] = data_pass
- train_tag = f"train_loss={extra['train/loss']:.4f} | " if window["n"] else ""
- record_validation(run_validation(), data_pass, extra_log=extra,
- label=f"{label_prefix}{train_tag}")
-
- model.train()
- window = new_window()
- data_pass = start_pass
- pbar = tqdm(total=args.max_steps, initial=global_step, desc="train", unit="step",
- disable=not accelerator.is_main_process)
-
- while global_step < args.max_steps:
- start_pass_shuffle(train_loader, data_pass)
- for batch in train_loader:
- with accelerator.accumulate(model):
- b = unpack_batch(batch, device)
- board, player = b["board"], b["player"]
- castling, en_passant = b["castling"], b["en_passant"]
- from_sq, action_plane = b["from_sq"], b["action_plane"]
- legal_planes, result = b["legal_planes"], b["result"]
- is_white, move_number = b["is_white"], b["move_number"]
-
- move_logits, value = model(board, player, castling, en_passant)
- loss, metrics = compute_loss(
- move_logits, value,
- from_sq, action_plane,
- legal_planes, result, is_white, move_number,
- args.value_loss_weight,
- label_smoothing=args.label_smoothing,
- )
-
- accelerator.backward(loss)
- if accelerator.sync_gradients and args.max_grad_norm > 0:
- accelerator.clip_grad_norm_(model.parameters(), args.max_grad_norm)
- if args.muon_schedules:
- # Momentum is not a constant in a well-tuned Muon run: a long memory
- # early averages over a fast-moving gradient, and a slightly shorter one
- # late helps the run settle. Weight decay cosine-decays to zero.
- mom = muon_momentum(global_step, args.max_steps,
- warmdown_ratio=args.warmdown_ratio)
- wdk = muon_weight_decay(global_step, args.max_steps, args.weight_decay)
- for g in muon_optimizer.param_groups:
- g["momentum"], g["weight_decay"] = mom, wdk
- muon_optimizer.step()
- adamw_optimizer.step()
- if accelerator.sync_gradients:
- muon_scheduler.step()
- adamw_scheduler.step()
- muon_optimizer.zero_grad()
- adamw_optimizer.zero_grad()
-
- # EMA is redundant under DDP but not wrong, and that is worth stating.
- # Gradients are all-reduced before the step, so every rank holds identical
- # params and computes an identical EMA from an identical decay -- no drift.
- # swap_ema_weights in run_validation is likewise symmetric because every
- # rank validates (val_loader is replicated, not sharded). The failure mode
- # to avoid is any rank taking a different branch here; sync_gradients is
- # world-uniform, so none does. tests/test_distributed.py asserts it.
- if use_ema and accelerator.sync_gradients:
- update_ema(get_raw_model(model, accelerator), ema_state, args.ema_decay)
-
- bs = board.size(0)
- window["loss"] += loss.item() * bs
- window["ce"] += metrics["ce"].item() * bs
- window["legal_ce"] += metrics["legal_ce"].item() * bs
- window["value"] += metrics["value_loss"].item() * bs
- window["acc"] += metrics["acc"].item() * bs
- window["legal_acc"] += metrics["legal_acc"].item() * bs
- window["n"] += bs
-
- pbar.set_postfix(
- loss=f"{loss.item():.4f}",
- ce=f"{metrics['ce'].item():.4f}",
- acc=f"{metrics['acc'].item():.3f}",
- val_l=f"{metrics['value_loss'].item():.4f}",
- )
-
- if not accelerator.sync_gradients:
- continue
-
- muon_lrs = muon_scheduler.get_last_lr()
- adamw_lrs = adamw_scheduler.get_last_lr()
- accelerator.log(
- {
- "train/step_loss": loss.item(),
- "train/step_ce": metrics["ce"].item(),
- "train/step_legal_ce": metrics["legal_ce"].item(),
- "train/step_value_loss": metrics["value_loss"].item(),
- "train/step_acc": metrics["acc"].item(),
- "train/step_legal_acc": metrics["legal_acc"].item(),
- "train/lr_muon": muon_lrs[0],
- "train/lr_emb": adamw_lrs[0],
- "train/lr_head": adamw_lrs[2],
- **({"train/muon_momentum": muon_optimizer.param_groups[0]["momentum"],
- "train/muon_wd": muon_optimizer.param_groups[0]["weight_decay"]}
- if args.muon_schedules else {}),
- },
- step=global_step,
- )
-
- global_step += 1
- pbar.update(1)
-
- if args.save_steps > 0 and global_step % args.save_steps == 0:
- ckpt = checkpoint_dir / f"checkpoint_step_{global_step:07d}"
- accelerator.save_state(str(ckpt))
- save_trainer_state(ckpt, data_pass, global_step, best_val_loss, scheduler_config)
- if use_ema and accelerator.is_main_process:
- save_ema_state(ema_state, ckpt / "ema_state.pt")
- if accelerator.is_main_process:
- rank0_print(f"\n Saved step checkpoint at step {global_step}")
- cleanup_old_checkpoints(checkpoint_dir, args.max_checkpoints)
-
- # Validation is on a step counter, and global_step is identical on every rank,
- # so all ranks enter the collective forward together.
- if args.eval_steps > 0 and global_step % args.eval_steps == 0:
- validate_now(window, data_pass,
- f"\nStep {global_step:,}/{args.max_steps:,}: ")
- window = new_window()
-
- if global_step >= args.max_steps:
- break
-
- data_pass += 1
-
- pbar.close()
-
- # Always finish on a validation, so the final steps can still win best_model even when
- # max_steps is not a multiple of eval_steps.
- rank0_print(f"\nReached max-steps ({args.max_steps:,}) after {data_pass} pass(es) over the data.")
- validate_now(window, data_pass, f"Final (step {global_step:,}): ")
-
- # ── Final test (using EMA weights) ───────────────────────────────────
- rank0_print("\nFinal test evaluation...")
- model.eval()
- if use_ema:
- swap_ema_weights(get_raw_model(model, accelerator), ema_state)
- test_loss_sum = 0.0
- test_correct = 0
- test_legal_correct = 0
- test_total = 0
-
- with torch.no_grad():
- for batch in tqdm(test_loader, desc="Testing", leave=False,
- disable=not accelerator.is_main_process):
- b = unpack_batch(batch, device)
- board, player = b["board"], b["player"]
- castling, en_passant = b["castling"], b["en_passant"]
- from_sq, action_plane = b["from_sq"], b["action_plane"]
- legal_planes, result = b["legal_planes"], b["result"]
- is_white, move_number = b["is_white"], b["move_number"]
-
- move_logits, value = model(board, player, castling, en_passant)
- loss, metrics = compute_loss(
- move_logits, value,
- from_sq, action_plane,
- legal_planes, result, is_white, move_number,
- args.value_loss_weight,
- )
-
- B = move_logits.size(0)
- test_loss_sum += loss.item() * B
- test_correct += int(metrics["acc"].item() * B)
- test_legal_correct += int(metrics["legal_acc"].item() * B)
- test_total += B
-
- test_loss = test_loss_sum / test_total
- test_acc = test_correct / test_total
- test_legal_acc = test_legal_correct / test_total
-
- accelerator.log(
- {"test/loss": test_loss, "test/accuracy": test_acc, "test/legal_accuracy": test_legal_acc},
- step=global_step,
- )
- rank0_print(f"Test: loss={test_loss:.4f} | accuracy={test_acc:.4f} | legal_accuracy={test_legal_acc:.4f}")
-
- accelerator.end_training()
- rank0_print(f"\nTraining complete. Logs: {log_path}")
-
-
-if __name__ == "__main__":
- main()
diff --git a/chesstransformer/trainers/position2move_trainer.py b/chesstransformer/trainers/position2move_trainer.py
deleted file mode 100644
index 8ef3c193a16362f981a5c5bb8584f86f3e1f91c0..0000000000000000000000000000000000000000
--- a/chesstransformer/trainers/position2move_trainer.py
+++ /dev/null
@@ -1,682 +0,0 @@
-#!/usr/bin/env python3
-"""
-Training script for Position2Move transformer model.
-
-This script trains a transformer to predict chess moves from board positions.
-Uses the new efficient HDF5 dataset format with game-based storage and random sampling.
-
-Usage:
- python src/chesstransformer/trainers/position2move_trainer.py --data data/Lichess2017.h5
-
-Options:
- --data: Path to HDF5 dataset
- --min-elo: Minimum average ELO for filtering games
- --max-elo: Maximum average ELO for filtering games
- --batch-size: Batch size (default: 32)
- --epochs: Number of epochs (default: 10)
- --lr: Learning rate (default: 3e-4)
- --save-every: Save checkpoint every N epochs (default: 5)
-"""
-
-from pathlib import Path
-from datetime import datetime
-import argparse
-import copy
-import random
-import numpy as np
-import json
-
-from tqdm.auto import tqdm
-import torch
-from torch.nn import functional as F
-from torch.utils.data import DataLoader, random_split
-import accelerate
-from accelerate.utils import set_seed
-from torch.optim.lr_scheduler import LambdaLR
-
-from chesstransformer.datasets.h5_lichess_dataset import HDF5ChessDataset
-from chesstransformer.models.transformer.position2move import Position2MoveModel
-
-
-def get_next_run_number(log_dir="logs"):
- """Get the next run number by checking existing run directories."""
- log_path = Path(log_dir)
- if not log_path.exists():
- return 1
-
- existing_runs = [d.name for d in log_path.iterdir() if d.is_dir() and d.name.startswith("run_")]
-
- if not existing_runs:
- return 1
-
- run_numbers = []
- for run_name in existing_runs:
- try:
- num = int(run_name.split("_")[1])
- run_numbers.append(num)
- except (IndexError, ValueError):
- continue
-
- return max(run_numbers) + 1 if run_numbers else 1
-
-
-def compute_loss(logits, moves, legal_moves_mask, beta, legal_move_loss_weight):
- llm_loss = F.cross_entropy(logits, moves)
- illegal_prob = F.softmax(logits, dim=-1) * (1.0 - legal_moves_mask.float())
- legal_move_loss = torch.sum(illegal_prob**2, dim=-1).mean()
- filtered_logits = torch.where(legal_moves_mask.bool(), logits, -torch.inf)
- legal_move_cross_entropy = F.cross_entropy(filtered_logits, moves)
- loss = beta * llm_loss + legal_move_loss_weight * legal_move_loss + (1 - beta) * legal_move_cross_entropy
- return loss, llm_loss, legal_move_loss, legal_move_cross_entropy
-
-
-def save_trainer_state(checkpoint_dir, epoch, global_step, best_val_loss, scheduler_config=None):
- state = {
- "epoch": epoch,
- "global_step": global_step,
- "best_val_loss": best_val_loss,
- }
- if scheduler_config:
- state["scheduler_config"] = scheduler_config
- state_path = Path(checkpoint_dir) / "trainer_state.json"
- with state_path.open("w", encoding="utf-8") as fp:
- json.dump(state, fp)
-
-
-def load_trainer_state(checkpoint_dir):
- state_path = Path(checkpoint_dir) / "trainer_state.json"
- if not state_path.exists():
- return None
- with state_path.open("r", encoding="utf-8") as fp:
- return json.load(fp)
-
-
-def cleanup_old_step_checkpoints(checkpoint_dir, max_checkpoints):
- """Remove old step checkpoints, keeping only the most recent ones."""
- checkpoint_path = Path(checkpoint_dir)
- step_checkpoints = sorted(
- [d for d in checkpoint_path.iterdir() if d.is_dir() and d.name.startswith("checkpoint_step_")],
- key=lambda x: int(x.name.split("_")[-1]),
- )
-
- # Remove oldest checkpoints if we exceed max_checkpoints
- while len(step_checkpoints) > max_checkpoints:
- oldest = step_checkpoints.pop(0)
- import shutil
-
- shutil.rmtree(oldest)
- print(f" Removed old checkpoint: {oldest.name}")
-
-
-def create_linear_warmup_decay_scheduler(optimizer, warmup_steps, total_steps, final_lr_ratio):
- final_lr_ratio = max(0.0, final_lr_ratio)
- warmup_steps = max(1, warmup_steps)
- total_steps = max(warmup_steps + 1, total_steps)
-
- def lr_lambda(current_step):
- if current_step < warmup_steps:
- return float(current_step + 1) / float(warmup_steps)
- progress = (current_step - warmup_steps + 1) / float(max(1, total_steps - warmup_steps))
- return 1.0 - (1.0 - final_lr_ratio) * min(1.0, progress)
-
- return LambdaLR(optimizer, lr_lambda)
-
-
-def find_best_learning_rate(
- model, train_loader, accelerator, lr_min=1e-6, lr_max=1e-2, num_steps=100, beta=0.7, legal_move_loss_weight=10.0
-):
- model.train()
- device = accelerator.device
- initial_state = copy.deepcopy(model.state_dict())
- # Use base learning rate for LR finder
- optimizer = torch.optim.AdamW(model.parameters(), lr=lr_min)
- multiplier = (lr_max / lr_min) ** (1 / max(num_steps - 1, 1))
- best_lr = lr_min
- best_loss = float("inf")
- lr = lr_min
- data_iter = iter(train_loader)
-
- for step in range(num_steps):
- try:
- batch = next(data_iter)
- except StopIteration:
- data_iter = iter(train_loader)
- batch = next(data_iter)
-
- optimizer.zero_grad()
- positions = batch["position"].to(device)
- moves = batch["move"].to(device)
- is_white = batch["is_white"].to(device)
- legal_moves_mask = batch["legal_moves_mask"].to(device)
- castling_rights = batch["castling_rights"].to(device)
- en_passant_file = batch["en_passant_file"].to(device)
- halfmove_clock = batch["halfmove_clock"].to(device)
-
- logits = model(positions, is_white, castling_rights, en_passant_file, halfmove_clock)
- loss, llm_loss, legal_move_loss, legal_move_cross_entropy = compute_loss(
- logits, moves, legal_moves_mask, beta, legal_move_loss_weight
- )
-
- accelerator.backward(loss)
- optimizer.step()
-
- loss_value = accelerator.gather_for_metrics(loss.detach()).mean().item()
- if loss_value < best_loss:
- best_loss = loss_value
- best_lr = lr
-
- lr *= multiplier
- for param_group in optimizer.param_groups:
- param_group["lr"] = lr
-
- model.load_state_dict(initial_state)
- model.zero_grad(set_to_none=True)
- accelerator.free_memory()
- return best_lr, best_loss
-
-
-def main():
- default_data_path = Path(__file__).parents[2] / "data" / "Lichess2017.h5"
-
- parser = argparse.ArgumentParser(description="Train Position2Move Transformer")
- parser.add_argument("--data", type=str, default=default_data_path, help="Path to HDF5 dataset")
- parser.add_argument("--min-elo", type=int, default=None, help="Minimum average ELO for filtering games")
- parser.add_argument("--max-elo", type=int, default=None, help="Maximum average ELO for filtering games")
- parser.add_argument("--batch-size", type=int, default=32, help="Batch size")
- parser.add_argument("--epochs", type=int, default=10, help="Number of epochs")
- parser.add_argument("--lr", type=float, default=3e-4, help="Learning rate for transformers (base LR)")
- parser.add_argument(
- "--lr-embedding", type=float, default=None, help="Learning rate for embeddings (default: same as --lr)"
- )
- parser.add_argument(
- "--lr-head", type=float, default=None, help="Learning rate for output head (default: same as --lr)"
- )
- parser.add_argument("--save-every", type=int, default=5, help="Save checkpoint every N epochs")
- parser.add_argument("--lr-find", action="store_true", help="Run learning rate finder before training")
- parser.add_argument("--lr-min", type=float, default=1e-6, help="Minimum LR for finder sweep")
- parser.add_argument("--lr-max", type=float, default=1e-2, help="Maximum LR for finder sweep")
- parser.add_argument("--lr-steps", type=int, default=100, help="Number of steps for LR finder sweep")
- parser.add_argument("--beta", type=float, default=0.7, help="Weight for LLM loss in combined loss function")
- parser.add_argument(
- "--legal-move-loss-weight",
- type=float,
- default=10.0,
- help="Weight for legal move loss in combined loss function",
- )
- parser.add_argument("--random-seed", type=int, default=42, help="Random seed for reproducibility")
- parser.add_argument("--warmup-steps", type=int, default=1000, help="Number of warmup steps for LR scheduler")
- parser.add_argument("--final-lr-ratio", type=float, default=0.01, help="Final LR multiplier after linear decay")
- parser.add_argument("--resume-from", type=str, default=None, help="Path to checkpoint directory to resume from")
- parser.add_argument(
- "--max-grad-norm", type=float, default=1.0, help="Maximum gradient norm for clipping (0 to disable)"
- )
- parser.add_argument(
- "--max_examples", type=int, default=None, help="Maximum number of examples to use from the dataset"
- )
- parser.add_argument("--save-steps", type=int, default=5000, help="Save checkpoint every N steps")
- parser.add_argument(
- "--max-checkpoints",
- type=int,
- default=5,
- help="Maximum number of step checkpoints to keep (excluding best model)",
- )
- parser.add_argument(
- "--precision", type=str, default="fp16", choices=["bf16", "fp16", "fp32"], help="Precision for training"
- )
- args = parser.parse_args()
-
- set_seed(args.random_seed)
- random.seed(args.random_seed)
- np.random.seed(args.random_seed)
- torch.manual_seed(args.random_seed)
- torch.cuda.manual_seed_all(args.random_seed)
-
- # Setup dataset and dataloaders
- dataset = HDF5ChessDataset(
- hdf5_path=args.data,
- min_elo=args.min_elo,
- max_elo=args.max_elo,
- )
-
- MAX_VALIDATION_SAMPLES = 10000
-
- val_size = min(int(0.1 * len(dataset)), MAX_VALIDATION_SAMPLES)
- test_size = min(int(0.1 * len(dataset)), MAX_VALIDATION_SAMPLES)
- train_size = len(dataset) - val_size - test_size
- train_set, val_set, test_set = random_split(dataset, [train_size, val_size, test_size])
-
- train_loader = DataLoader(train_set, batch_size=args.batch_size, shuffle=True, num_workers=12, drop_last=True)
- val_loader = DataLoader(val_set, batch_size=args.batch_size, shuffle=False, num_workers=12)
- test_loader = DataLoader(test_set, batch_size=args.batch_size, shuffle=False, num_workers=12)
-
- print(f"Train samples: {len(train_set)}, Val samples: {len(val_set)}, Test samples: {len(test_set)}")
-
- # Model configuration
- model_config = {
- "vocab_size": dataset.position_tokenizer.vocab_size,
- "move_vocab_size": dataset.move_tokenizer.vocab_size,
- "embed_dim": 512,
- "nb_transformer_layers": 8,
- "num_heads": 8,
- "num_kv_groups": 4, # GQA: 8 heads / 4 groups = 2 heads per KV group
- "dropout": 0.1,
- "kvq_bias": False,
- "mask_future": False,
- "rope": True, # Use RoPE for positional encoding
- "use_swiglu": True, # Use SwiGLU activation (Llama-style)
- "use_col_row_emb": True, # Use separate column and row embeddings
- }
-
- # Setup logging
- log_dir = Path("logs") / "position2move"
- log_dir.mkdir(parents=True, exist_ok=True)
- run_number = get_next_run_number(str(log_dir))
- log_dir_full = f"run_{run_number:03d}_{datetime.now().strftime('%Y%m%d_%H%M%S')}"
- log_path = log_dir / log_dir_full
- log_path.mkdir(parents=True, exist_ok=False)
- print(f"Logging to: {log_path}")
-
- # Create checkpoints directory
- checkpoint_dir = log_path / "checkpoints"
- checkpoint_dir.mkdir(exist_ok=True)
-
- # Save model config to JSON
- config_path = log_path / "model_config.json"
- with config_path.open("w", encoding="utf-8") as f:
- json.dump(model_config, f, indent=2)
- print(f"Model config saved to: {config_path}")
-
- precision = args.precision if torch.cuda.is_available() or torch.backends.mps.is_available() else "fp32"
- print(f"Using precision: {precision}")
-
- accelerator = accelerate.Accelerator(log_with="tensorboard", project_dir=str(log_path), mixed_precision=precision)
-
- # Initialize trackers
- accelerator.init_trackers(
- project_name="position2move",
- config={
- "batch_size": args.batch_size,
- "epochs": args.epochs,
- "learning_rate": args.lr,
- "lr_embedding": args.lr_embedding if args.lr_embedding is not None else args.lr,
- "lr_head": args.lr_head if args.lr_head is not None else args.lr,
- "min_elo": args.min_elo,
- "max_elo": args.max_elo,
- "max_grad_norm": args.max_grad_norm,
- **model_config,
- "warmup_steps": args.warmup_steps,
- "final_lr_ratio": args.final_lr_ratio,
- "precision": args.precision,
- },
- )
-
- device = accelerator.device
- print(f"Using device: {device}")
-
- model = Position2MoveModel(**model_config)
-
- # torch.compile has limited support on MPS backend - disable if compilation fails
- if device.type == "mps":
- print("Warning: Skipping torch.compile on MPS device (Mac GPU) due to backend limitations")
- print("Training will proceed without compilation optimizations")
- else:
- try:
- model = torch.compile(
- model,
- )
- print("Model compiled successfully")
- except Exception as e:
- print(f"Warning: torch.compile failed ({e}), proceeding without compilation")
-
- model.to(device)
-
- if args.lr_find:
- if accelerator.is_main_process:
- print("Running learning rate finder...")
- best_lr, best_loss = find_best_learning_rate(
- model,
- train_loader,
- accelerator,
- lr_min=args.lr_min,
- lr_max=args.lr_max,
- num_steps=args.lr_steps,
- beta=args.beta,
- legal_move_loss_weight=args.legal_move_loss_weight,
- )
- accelerator.log({"lr_finder/best_lr": best_lr, "lr_finder/best_loss": best_loss}, step=0)
- args.lr = best_lr
- if accelerator.is_main_process:
- print(f"Best LR found: {best_lr:.6g} (loss {best_loss:.4f})")
-
- # Set up discriminative learning rates
- lr_embedding = args.lr_embedding if args.lr_embedding is not None else args.lr
- lr_head = args.lr_head if args.lr_head is not None else args.lr
- lr_transformer = args.lr
-
- # Create parameter groups
- embedding_params = []
- transformer_params = []
- head_params = []
-
- # Get underlying model (unwrap from accelerator if needed)
- unwrapped_model = accelerator.unwrap_model(model)
-
- # Categorize parameters
- for name, param in unwrapped_model.named_parameters():
- if "embedding" in name:
- embedding_params.append(param)
- elif "lm_head" in name:
- head_params.append(param)
- else:
- transformer_params.append(param)
-
- param_groups = [
- {"params": embedding_params, "lr": lr_embedding, "name": "embeddings"},
- {"params": transformer_params, "lr": lr_transformer, "name": "transformers"},
- {"params": head_params, "lr": lr_head, "name": "head"},
- ]
-
- if accelerator.is_main_process:
- emb_count = sum(p.numel() for p in embedding_params)
- trans_count = sum(p.numel() for p in transformer_params)
- head_count = sum(p.numel() for p in head_params)
- print(f"\nLearning rates:")
- print(f" Embeddings: {lr_embedding:.6g}")
- print(f" Transformers: {lr_transformer:.6g}")
- print(f" Head: {lr_head:.6g}")
- print(f" Param counts: Emb={emb_count:,}, Trans={trans_count:,}, Head={head_count:,}\n")
-
- optimizer = torch.optim.AdamW(param_groups, lr=args.lr)
- model, optimizer, train_loader, val_loader, test_loader = accelerator.prepare(
- model, optimizer, train_loader, val_loader, test_loader
- )
-
- # Check if resuming to load scheduler config from checkpoint
- best_val_loss = float("inf")
- start_epoch = 1
- global_step = 0
- resume_path = Path(args.resume_from) if args.resume_from else None
- trainer_state = None
-
- if resume_path:
- trainer_state = load_trainer_state(resume_path)
- if trainer_state:
- start_epoch = trainer_state.get("epoch", 0) + 1
- global_step = trainer_state.get("global_step", 0)
- best_val_loss = trainer_state.get("best_val_loss", float("inf"))
-
- # Determine scheduler parameters - use saved config when resuming for consistency
- has_scheduler_config = trainer_state and "scheduler_config" in trainer_state
-
- if has_scheduler_config:
- saved_config = trainer_state["scheduler_config"]
- scheduler_warmup_steps = saved_config["warmup_steps"]
- scheduler_total_steps = saved_config["total_steps"]
- scheduler_final_lr_ratio = saved_config["final_lr_ratio"]
- if accelerator.is_main_process:
- print(
- f"Using scheduler config from checkpoint: warmup={scheduler_warmup_steps}, total={scheduler_total_steps}, final_lr_ratio={scheduler_final_lr_ratio}"
- )
- else:
- scheduler_warmup_steps = args.warmup_steps
- scheduler_total_steps = max(1, len(train_loader)) * max(1, args.epochs)
- scheduler_final_lr_ratio = args.final_lr_ratio
- if resume_path and accelerator.is_main_process:
- print(
- f"Old checkpoint without scheduler_config - using new scheduler params: warmup={scheduler_warmup_steps}, total={scheduler_total_steps}"
- )
-
- # Store scheduler config for saving in checkpoints
- scheduler_config = {
- "warmup_steps": scheduler_warmup_steps,
- "total_steps": scheduler_total_steps,
- "final_lr_ratio": scheduler_final_lr_ratio,
- }
-
- scheduler = create_linear_warmup_decay_scheduler(
- optimizer,
- warmup_steps=scheduler_warmup_steps,
- total_steps=scheduler_total_steps,
- final_lr_ratio=scheduler_final_lr_ratio,
- )
-
- # Always register scheduler for checkpointing (required for load_state to work)
- accelerator.register_for_checkpointing(scheduler)
-
- # Load the full accelerator state (model, optimizer, scheduler) if resuming
- if resume_path:
- accelerator.load_state(str(resume_path))
-
- # For old checkpoints without scheduler_config, the loaded scheduler state
- # has a mismatched lr_lambda (old total_steps). We need to recreate it.
- if not has_scheduler_config:
- # Recreate scheduler with correct parameters
- scheduler = create_linear_warmup_decay_scheduler(
- optimizer,
- warmup_steps=scheduler_warmup_steps,
- total_steps=scheduler_total_steps,
- final_lr_ratio=scheduler_final_lr_ratio,
- )
- # Manually advance to the correct step
- scheduler.last_epoch = global_step - 1 # -1 because step() will increment it
- scheduler.step() # This sets it to global_step and computes correct LR
- if accelerator.is_main_process:
- current_lrs = scheduler.get_last_lr()
- print(f" Recreated scheduler and advanced to step {global_step}")
- print(f" Current LRs: emb={current_lrs[0]:.6g}, trans={current_lrs[1]:.6g}, head={current_lrs[2]:.6g}")
-
- if accelerator.is_main_process:
- print(f"Resumed from checkpoint: {resume_path}")
- print(f" Starting from epoch {start_epoch}, global_step {global_step}, best_val_loss {best_val_loss:.4f}")
-
- for epoch in range(start_epoch, args.epochs + 1):
- # Training phase
- model.train()
- train_loss = 0.0
- train_llm_loss = 0.0
- train_legal_move_loss = 0.0
- train_legal_move_ce = 0.0
-
- progress_bar = tqdm(train_loader, desc=f"Epoch {epoch}/{args.epochs} - Training", leave=False)
- for batch in progress_bar:
- optimizer.zero_grad()
- positions = batch["position"]
- moves = batch["move"]
- is_white = batch["is_white"]
- legal_moves_mask = batch["legal_moves_mask"]
- castling_rights = batch["castling_rights"]
- en_passant_file = batch["en_passant_file"]
- halfmove_clock = batch["halfmove_clock"]
-
- logits = model(positions, is_white, castling_rights, en_passant_file, halfmove_clock)
- loss, llm_loss, legal_move_loss, legal_move_cross_entropy = compute_loss(
- logits, moves, legal_moves_mask, args.beta, args.legal_move_loss_weight
- )
-
- accelerator.backward(loss)
-
- # Gradient clipping
- if args.max_grad_norm > 0:
- accelerator.clip_grad_norm_(model.parameters(), args.max_grad_norm)
-
- optimizer.step()
- scheduler.step()
-
- batch_size = positions.size(0)
- train_loss += loss.item() * batch_size
- train_llm_loss += llm_loss.item() * batch_size
- train_legal_move_loss += legal_move_loss.item() * batch_size
- train_legal_move_ce += legal_move_cross_entropy.item() * batch_size
-
- progress_bar.set_postfix(
- {
- "loss": loss.item(),
- "llm_loss": llm_loss.item(),
- "legal_move_loss": legal_move_loss.item(),
- "legal_move_ce": legal_move_cross_entropy.item(),
- }
- )
-
- # Log step-level metrics
- current_lrs = scheduler.get_last_lr()
- log_dict = {
- "train/step_loss": loss.item(),
- "train/step_llm_loss": llm_loss.item(),
- "train/step_legal_move_loss": legal_move_loss.item(),
- "train/step_legal_move_ce": legal_move_cross_entropy.item(),
- "train/lr_embeddings": current_lrs[0],
- "train/lr_transformers": current_lrs[1],
- "train/lr_head": current_lrs[2],
- }
- accelerator.log(log_dict, step=global_step)
-
- global_step += 1
-
- # Save step checkpoint
- if args.save_steps > 0 and global_step % args.save_steps == 0:
- step_checkpoint_path = checkpoint_dir / f"checkpoint_step_{global_step:07d}"
- accelerator.save_state(str(step_checkpoint_path))
- save_trainer_state(step_checkpoint_path, epoch, global_step, best_val_loss, scheduler_config)
- if accelerator.is_main_process:
- print(f"\n Saved step checkpoint at step {global_step}")
- cleanup_old_step_checkpoints(checkpoint_dir, args.max_checkpoints)
-
- # Calculate average training metrics
- train_loss /= len(train_set)
- train_llm_loss /= len(train_set)
- train_legal_move_loss /= len(train_set)
- train_legal_move_ce /= len(train_set)
-
- # Validation phase
- model.eval()
- val_loss = 0.0
- val_llm_loss = 0.0
- val_legal_move_loss = 0.0
- val_legal_move_ce = 0.0
- val_correct = 0
- val_total = 0
-
- with torch.no_grad():
- progress_bar = tqdm(val_loader, desc=f"Epoch {epoch}/{args.epochs} - Validation", leave=False)
- for batch in progress_bar:
- positions = batch["position"]
- moves = batch["move"]
- is_white = batch["is_white"]
- legal_moves_mask = batch["legal_moves_mask"]
- castling_rights = batch["castling_rights"]
- en_passant_file = batch["en_passant_file"]
- halfmove_clock = batch["halfmove_clock"]
-
- logits = model(positions, is_white, castling_rights, en_passant_file, halfmove_clock)
- loss, llm_loss, legal_move_loss, legal_move_cross_entropy = compute_loss(
- logits, moves, legal_moves_mask, args.beta, args.legal_move_loss_weight
- )
-
- batch_size = positions.size(0)
- val_loss += loss.item() * batch_size
- val_llm_loss += llm_loss.item() * batch_size
- val_legal_move_loss += legal_move_loss.item() * batch_size
- val_legal_move_ce += legal_move_cross_entropy.item() * batch_size
-
- # Calculate accuracy
- predictions = torch.argmax(logits, dim=-1)
- val_correct += (predictions == moves).sum().item()
- val_total += batch_size
-
- progress_bar.set_postfix({"val_loss": loss.item()})
-
- val_loss /= len(val_set)
- val_llm_loss /= len(val_set)
- val_legal_move_loss /= len(val_set)
- val_legal_move_ce /= len(val_set)
- val_accuracy = val_correct / val_total if val_total > 0 else 0.0
-
- # Log epoch-level metrics
- accelerator.log(
- {
- "train/epoch_loss": train_loss,
- "train/epoch_llm_loss": train_llm_loss,
- "train/epoch_legal_move_loss": train_legal_move_loss,
- "train/epoch_legal_move_ce": train_legal_move_ce,
- "val/loss": val_loss,
- "val/llm_loss": val_llm_loss,
- "val/legal_move_loss": val_legal_move_loss,
- "val/legal_move_ce": val_legal_move_ce,
- "val/accuracy": val_accuracy,
- "epoch": epoch,
- "train/lr_embeddings_epoch_end": scheduler.get_last_lr()[0],
- "train/lr_transformers_epoch_end": scheduler.get_last_lr()[1],
- "train/lr_head_epoch_end": scheduler.get_last_lr()[2],
- },
- step=global_step,
- )
-
- print(f"\nEpoch {epoch}/{args.epochs}:")
- print(f" Train Loss: {train_loss:.4f} | Val Loss: {val_loss:.4f} | Val Accuracy: {val_accuracy:.4f}")
-
- # Save best model
- if val_loss < best_val_loss:
- best_val_loss = val_loss
- best_checkpoint_path = checkpoint_dir / "best_model"
- accelerator.save_state(str(best_checkpoint_path))
- save_trainer_state(best_checkpoint_path, epoch, global_step, best_val_loss, scheduler_config)
- print(f" Saved best model (val_loss: {val_loss:.4f})")
-
- # Periodic checkpoint saving
- if epoch % args.save_every == 0:
- checkpoint_path = checkpoint_dir / f"checkpoint_epoch_{epoch:03d}"
- accelerator.save_state(str(checkpoint_path))
- save_trainer_state(checkpoint_path, epoch, global_step, best_val_loss, scheduler_config)
- print(f" Saved checkpoint at epoch {epoch}")
-
- # Final test evaluation
- print("\nRunning final test evaluation...")
- model.eval()
- test_loss = 0.0
- test_correct = 0
- test_total = 0
-
- with torch.no_grad():
- progress_bar = tqdm(test_loader, desc="Testing", leave=False)
- for batch in progress_bar:
- positions = batch["position"]
- moves = batch["move"]
- is_white = batch["is_white"]
- legal_moves_mask = batch["legal_moves_mask"]
- castling_rights = batch["castling_rights"]
- en_passant_file = batch["en_passant_file"]
- halfmove_clock = batch["halfmove_clock"]
-
- logits = model(positions, is_white, castling_rights, en_passant_file, halfmove_clock)
- loss, llm_loss, legal_move_loss, legal_move_cross_entropy = compute_loss(
- logits, moves, legal_moves_mask, args.beta, args.legal_move_loss_weight
- )
-
- batch_size = positions.size(0)
- test_loss += loss.item() * batch_size
-
- predictions = torch.argmax(logits, dim=-1)
- test_correct += (predictions == moves).sum().item()
- test_total += batch_size
-
- test_loss /= len(test_set)
- test_accuracy = test_correct / test_total if test_total > 0 else 0.0
-
- accelerator.log(
- {
- "test/loss": test_loss,
- "test/accuracy": test_accuracy,
- },
- step=global_step,
- )
-
- print(f"\nTest Results:")
- print(f" Test Loss: {test_loss:.4f} | Test Accuracy: {test_accuracy:.4f}")
-
- # End tracking
- accelerator.end_training()
- print(f"\nTraining complete! Logs saved to: {log_path}")
-
-
-if __name__ == "__main__":
- main()
diff --git a/chesstransformer/utils/__init__.py b/chesstransformer/utils/__init__.py
deleted file mode 100644
index 15f4eaa33cae3fe69e6e54482617726e6cb37ba9..0000000000000000000000000000000000000000
--- a/chesstransformer/utils/__init__.py
+++ /dev/null
@@ -1,3 +0,0 @@
-from .chess_utils import generate_board_image, generate_board_ascii, generate_game_gif
-
-__all__ = ["generate_board_image", "generate_board_ascii", "generate_game_gif"]
diff --git a/chesstransformer/utils/chess_utils.py b/chesstransformer/utils/chess_utils.py
deleted file mode 100644
index c086cfd8ae01e6bbfbafdcd1d577a3e39bd0d5c0..0000000000000000000000000000000000000000
--- a/chesstransformer/utils/chess_utils.py
+++ /dev/null
@@ -1,123 +0,0 @@
-import chess
-from PIL import Image
-from io import BytesIO
-import cairosvg
-
-
-def _create_board_from_moves(moves: list[str]) -> chess.Board:
- """
- Create a chess board and apply a sequence of moves.
-
- Args:
- moves: List of moves in UCI format (e.g., ['e2e4', 'd7d5'])
-
- Returns:
- Chess board with moves applied
- """
- board = chess.Board()
- for move_str in moves:
- try:
- move = chess.Move.from_uci(move_str)
- board.push(move)
- except ValueError:
- pass # Skip invalid moves
- return board
-
-
-def generate_board_image(moves: list[str], size: int = 400) -> Image.Image:
- """
- Generate a PIL image of a chess board after applying a sequence of moves.
-
- Args:
- moves: List of moves in UCI format (e.g., ['e2e4', 'd7d5'])
- size: Size of the output image in pixels (default: 400)
-
- Returns:
- PIL Image of the chess board
- """
- import chess.svg
-
- board = _create_board_from_moves(moves)
-
- # Generate SVG using chess library
- svg_data = chess.svg.board(board, size=size)
-
- # Convert SVG to PNG using cairosvg
- png_data = cairosvg.svg2png(bytestring=svg_data.encode("utf-8"))
-
- # Load PNG into PIL Image
- img = Image.open(BytesIO(png_data))
-
- return img
-
-
-def generate_board_ascii(moves: list[str]) -> str:
- """
- Generate an ASCII representation of a chess board after applying a sequence of moves.
-
- Args:
- moves: List of moves in UCI format (e.g., ['e2e4', 'd7d5'])
-
- Returns:
- String representation of the chess board
- """
- board = _create_board_from_moves(moves)
- return str(board)
-
-
-def generate_game_gif(moves: list[str], size: int = 400, duration: int = 500) -> bytes:
- """
- Generate an animated GIF of a chess game from a sequence of moves.
-
- Args:
- moves: List of moves in UCI format (e.g., ['e2e4', 'd7d5'])
- size: Size of each frame in pixels (default: 400)
- duration: Duration of each frame in milliseconds (default: 500)
-
- Returns:
- Bytes of the animated GIF
- """
- frames = []
-
- # Initial position
- img = generate_board_image([], size)
- frames.append(img)
-
- # Apply moves and capture frames
- moves_so_far = []
- for move_str in moves:
- try:
- move = chess.Move.from_uci(move_str)
- moves_so_far.append(move_str)
- img = generate_board_image(moves_so_far, size)
- frames.append(img)
- except ValueError:
- pass # Skip invalid moves
-
- # Save frames as GIF in memory
- gif_bytes = BytesIO()
- frames[0].save(
- gif_bytes,
- format="GIF",
- save_all=True,
- append_images=frames[1:],
- duration=duration,
- loop=0,
- )
- gif_bytes.seek(0)
-
- return gif_bytes.getvalue()
-
-
-if __name__ == "__main__":
- moves = ["e2e4"]
- img = generate_board_image(moves)
- img.show()
-
- ascii_board = generate_board_ascii(moves)
- print(ascii_board)
-
- coup_du_berget = ["e2e4", "e7e5", "f1c4", "b8c6", "d1h5", "g8f6", "h5f7"]
- gif = generate_game_gif(coup_du_berget)
- with open("coup_du_berget.gif", "wb") as f:
- f.write(gif)
diff --git a/chesstransformer/utils/evaluate_puzzles.py b/chesstransformer/utils/evaluate_puzzles.py
deleted file mode 100644
index 77d29bf7e1f33d99fd241d7ac26afb702f8d84b5..0000000000000000000000000000000000000000
--- a/chesstransformer/utils/evaluate_puzzles.py
+++ /dev/null
@@ -1,426 +0,0 @@
-#!/usr/bin/env python3
-"""
-Evaluation script for chess puzzle solving with Position2Move model.
-
-This script evaluates how well a trained model can solve chess puzzles by:
-1. Loading a trained model checkpoint
-2. Testing it on puzzles from the Lichess puzzle database
-3. Measuring accuracy for first move, full solution, and by rating/theme
-4. Generating detailed reports
-
-Usage:
- python src/chesstransformer/utils/evaluate_puzzles.py --model data/models/puzzle_training/run_001/best_model.pth
-
-Options:
- --model: Path to model checkpoint (required)
- --puzzle-data: Path to puzzle .csv.zst file (default: data/lichess_db_puzzle.csv.zst)
- --num-puzzles: Number of puzzles to evaluate (default: 1000)
- --min-rating: Minimum puzzle rating (default: None)
- --max-rating: Maximum puzzle rating (default: None)
- --themes: Comma-separated puzzle themes to filter by (default: None)
- --output: Output JSON file for results (default: results/puzzle_eval_{timestamp}.json)
-"""
-
-from pathlib import Path
-from datetime import datetime
-import argparse
-import json
-
-import torch
-import chess
-from tqdm.auto import tqdm
-
-from chesstransformer.datasets.puzzle_dataset import LichessPuzzleFullSolutionDataset
-from chesstransformer.models.transformer.position2move import Position2MoveModel
-from chesstransformer.models.tokenizer.position_tokenizer import PostionTokenizer
-from chesstransformer.models.tokenizer.move_tokenizer import MoveTokenizer
-
-
-class PuzzleEvaluator:
- """Evaluates a Position2Move model on chess puzzles."""
-
- def __init__(self, model, device="cpu"):
- self.model = model
- self.device = device
- self.model.to(device)
- self.model.eval()
-
- self.position_tokenizer = PostionTokenizer()
- self.move_tokenizer = MoveTokenizer()
-
- def predict_move(self, board: chess.Board, top_k=5):
- """
- Predict the best move for a given board position.
-
- Args:
- board: chess.Board object
- top_k: Return top-k predictions
-
- Returns:
- List of (move_uci, probability, is_legal) tuples
- """
- # Encode position
- position_tokens = self.position_tokenizer.encode(board)
- position_tensor = torch.tensor(position_tokens, dtype=torch.long).unsqueeze(0).to(self.device)
- is_white = torch.tensor([board.turn == chess.WHITE], dtype=torch.long).to(self.device)
-
- # Get predictions
- with torch.no_grad():
- logits = self.model(position_tensor, is_white)
-
- # Create legal moves mask
- legal_moves = [m.uci() for m in board.legal_moves]
- legal_move_indices = []
- for move_uci in legal_moves:
- try:
- idx = self.move_tokenizer.encode(move_uci)
- legal_move_indices.append(idx)
- except ValueError:
- continue
-
- # Apply mask to logits (set illegal moves to -inf before softmax)
- mask = torch.full_like(logits, float("-inf"))
- if legal_move_indices:
- mask[0, legal_move_indices] = 0
- masked_logits = logits + mask
-
- probs = torch.softmax(masked_logits, dim=-1)
-
- # Get top-k predictions
- top_probs, top_indices = torch.topk(probs[0], k=top_k)
-
- predictions = []
-
- for prob, idx in zip(top_probs.cpu().numpy(), top_indices.cpu().numpy()):
- try:
- move_uci = self.move_tokenizer.decode(int(idx))
- is_legal = move_uci in legal_moves
- predictions.append((move_uci, float(prob), is_legal))
- except ValueError:
- continue
-
- return predictions
-
- def evaluate_puzzle(self, puzzle_data, max_moves=10):
- """
- Evaluate a single puzzle.
-
- Args:
- puzzle_data: Dictionary with puzzle information
- max_moves: Maximum number of moves to try in the solution
-
- Returns:
- Dictionary with evaluation results
- """
- board = chess.Board(puzzle_data["fen"])
- solution_moves = puzzle_data["moves_uci"]
-
- results = {
- "puzzle_id": puzzle_data["puzzle_id"],
- "rating": puzzle_data["rating"],
- "themes": puzzle_data["themes"],
- "solution_length": len(solution_moves),
- "moves_tried": [],
- "first_move_correct": False,
- "fully_solved": False,
- "moves_correct": 0,
- "predictions": [],
- }
-
- # Try to solve the puzzle move by move
- for move_idx in range(0, len(solution_moves), 2): # Only predict our moves (every other move)
- if move_idx >= max_moves:
- break
-
- # Apply opponent's move first (if not the first move)
- if move_idx > 0:
- opponent_move = chess.Move.from_uci(solution_moves[move_idx - 1])
- if opponent_move in board.legal_moves:
- board.push(opponent_move)
- else:
- # Opponent move is illegal - puzzle data might be corrupted
- results["error"] = "Illegal opponent move in solution"
- break
-
- # Predict our move
- expected_move = solution_moves[move_idx]
- predictions = self.predict_move(board, top_k=5)
-
- if not predictions:
- results["error"] = "Model produced no valid predictions"
- break
-
- predicted_move = predictions[0][0]
- predicted_prob = predictions[0][1]
- is_correct = predicted_move == expected_move
-
- results["moves_tried"].append(
- {
- "move_number": move_idx // 2 + 1,
- "expected": expected_move,
- "predicted": predicted_move,
- "probability": predicted_prob,
- "correct": is_correct,
- "top_5_predictions": [{"move": m, "prob": p, "legal": l} for m, p, l in predictions],
- }
- )
-
- if is_correct:
- results["moves_correct"] += 1
- if move_idx == 0:
- results["first_move_correct"] = True
-
- # Apply our correct move
- our_move = chess.Move.from_uci(predicted_move)
- board.push(our_move)
- else:
- # Wrong move - puzzle failed
- break
-
- # Check if fully solved
- if results["moves_correct"] == (len(solution_moves) + 1) // 2:
- results["fully_solved"] = True
-
- return results
-
- def evaluate_dataset(self, dataset, num_puzzles=None):
- """
- Evaluate the model on a puzzle dataset.
-
- Args:
- dataset: LichessPuzzleFullSolutionDataset
- num_puzzles: Number of puzzles to evaluate (None = all)
-
- Returns:
- Dictionary with aggregate results
- """
- num_puzzles = min(num_puzzles or len(dataset), len(dataset))
-
- print(f"Evaluating {num_puzzles} puzzles...")
-
- results = {
- "num_puzzles": num_puzzles,
- "first_move_accuracy": 0,
- "full_solution_accuracy": 0,
- "average_moves_correct": 0,
- "by_rating": {},
- "by_theme": {},
- "puzzle_results": [],
- }
-
- first_move_correct = 0
- fully_solved = 0
- total_moves_correct = 0
- total_moves = 0
-
- # Track by rating buckets
- rating_buckets = {
- "0-1000": {"first": 0, "full": 0, "total": 0},
- "1000-1500": {"first": 0, "full": 0, "total": 0},
- "1500-2000": {"first": 0, "full": 0, "total": 0},
- "2000-2500": {"first": 0, "full": 0, "total": 0},
- "2500+": {"first": 0, "full": 0, "total": 0},
- }
-
- # Track by theme
- theme_stats = {}
-
- for i in tqdm(range(num_puzzles), desc="Evaluating puzzles"):
- puzzle_data = dataset[i]
- puzzle_result = self.evaluate_puzzle(puzzle_data)
-
- # Aggregate statistics
- if puzzle_result["first_move_correct"]:
- first_move_correct += 1
- if puzzle_result["fully_solved"]:
- fully_solved += 1
-
- total_moves_correct += puzzle_result["moves_correct"]
- total_moves += puzzle_result["solution_length"] // 2 + 1
-
- # Rating bucket
- rating = puzzle_result["rating"]
- if rating < 1000:
- bucket = "0-1000"
- elif rating < 1500:
- bucket = "1000-1500"
- elif rating < 2000:
- bucket = "1500-2000"
- elif rating < 2500:
- bucket = "2000-2500"
- else:
- bucket = "2500+"
-
- rating_buckets[bucket]["total"] += 1
- if puzzle_result["first_move_correct"]:
- rating_buckets[bucket]["first"] += 1
- if puzzle_result["fully_solved"]:
- rating_buckets[bucket]["full"] += 1
-
- # Theme statistics
- for theme in puzzle_result["themes"]:
- if theme not in theme_stats:
- theme_stats[theme] = {"first": 0, "full": 0, "total": 0}
- theme_stats[theme]["total"] += 1
- if puzzle_result["first_move_correct"]:
- theme_stats[theme]["first"] += 1
- if puzzle_result["fully_solved"]:
- theme_stats[theme]["full"] += 1
-
- results["puzzle_results"].append(puzzle_result)
-
- # Calculate aggregate metrics
- results["first_move_accuracy"] = 100.0 * first_move_correct / num_puzzles
- results["full_solution_accuracy"] = 100.0 * fully_solved / num_puzzles
- results["average_moves_correct"] = total_moves_correct / num_puzzles
- results["move_accuracy"] = 100.0 * total_moves_correct / total_moves if total_moves > 0 else 0
-
- # Rating bucket statistics
- for bucket, stats in rating_buckets.items():
- if stats["total"] > 0:
- results["by_rating"][bucket] = {
- "count": stats["total"],
- "first_move_accuracy": 100.0 * stats["first"] / stats["total"],
- "full_solution_accuracy": 100.0 * stats["full"] / stats["total"],
- }
-
- # Theme statistics (top 20 themes by frequency)
- sorted_themes = sorted(theme_stats.items(), key=lambda x: x[1]["total"], reverse=True)[:20]
- for theme, stats in sorted_themes:
- results["by_theme"][theme] = {
- "count": stats["total"],
- "first_move_accuracy": 100.0 * stats["first"] / stats["total"],
- "full_solution_accuracy": 100.0 * stats["full"] / stats["total"],
- }
-
- return results
-
-
-def main():
- parser = argparse.ArgumentParser(description="Evaluate Position2Move model on chess puzzles")
- parser.add_argument("--model", type=str, required=True, help="Path to model checkpoint")
- parser.add_argument(
- "--puzzle-data", type=str, default="data/lichess_db_puzzle.csv.zst", help="Path to puzzle .csv.zst file"
- )
- parser.add_argument("--num-puzzles", type=int, default=1000, help="Number of puzzles to evaluate")
- parser.add_argument("--min-rating", type=int, default=None, help="Minimum puzzle rating")
- parser.add_argument("--max-rating", type=int, default=None, help="Maximum puzzle rating")
- parser.add_argument("--themes", type=str, default=None, help="Comma-separated puzzle themes to filter by")
- parser.add_argument("--output", type=str, default=None, help="Output JSON file for results")
-
- args = parser.parse_args()
-
- # Parse themes
- themes = args.themes.split(",") if args.themes else None
-
- # Setup device
- device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
- print(f"Using device: {device}")
-
- # Load model
- print(f"\nLoading model from {args.model}...")
- model_path = Path(args.model)
-
- # Support both .pth and .safetensors formats
- if model_path.suffix == ".safetensors":
- # Load config from same directory
- from safetensors import safe_open
- import json
-
- config_path = model_path.parent / "config.json"
- with open(config_path, "r") as f:
- config = json.load(f)
-
- model = Position2MoveModel(**config)
-
- # Load weights from safetensors
- with safe_open(str(model_path), framework="pt", device=str(device)) as f:
- state_dict = {k: f.get_tensor(k) for k in f.keys()}
-
- # Handle compiled model prefix (_orig_mod.)
- if any(k.startswith("_orig_mod.") for k in state_dict.keys()):
- state_dict = {k.replace("_orig_mod.", ""): v for k, v in state_dict.items()}
-
- model.load_state_dict(state_dict)
-
- print(f"Model loaded from safetensors")
- else:
- # Load from .pth checkpoint
- checkpoint = torch.load(args.model, map_location=device, weights_only=False)
- config = checkpoint["config"]
-
- model = Position2MoveModel(**config)
- model.load_state_dict(checkpoint["model_state_dict"])
- print(f"Model loaded (epoch {checkpoint.get('epoch', 'unknown')})")
-
- # Load puzzle dataset
- print(f"\nLoading puzzle dataset...")
- dataset = LichessPuzzleFullSolutionDataset(
- puzzle_path=args.puzzle_data,
- min_rating=args.min_rating,
- max_rating=args.max_rating,
- themes=themes,
- max_puzzles=args.num_puzzles,
- )
-
- # Evaluate
- print("\n" + "=" * 70)
- print("Starting evaluation")
- print("=" * 70)
-
- evaluator = PuzzleEvaluator(model, device=device)
- results = evaluator.evaluate_dataset(dataset, num_puzzles=args.num_puzzles)
-
- # Print summary
- print("\n" + "=" * 70)
- print("EVALUATION RESULTS")
- print("=" * 70)
- print(f"Total puzzles: {results['num_puzzles']}")
- print(f"First move accuracy: {results['first_move_accuracy']:.2f}%")
- print(f"Full solution accuracy: {results['full_solution_accuracy']:.2f}%")
- print(f"Move accuracy: {results['move_accuracy']:.2f}%")
- print(f"Average moves correct: {results['average_moves_correct']:.2f}")
-
- print("\n" + "-" * 70)
- print("By Rating:")
- print("-" * 70)
- for rating, stats in sorted(results["by_rating"].items()):
- print(
- f" {rating:>12}: {stats['count']:4} puzzles | "
- f"First: {stats['first_move_accuracy']:5.2f}% | "
- f"Full: {stats['full_solution_accuracy']:5.2f}%"
- )
-
- print("\n" + "-" * 70)
- print("Top Themes:")
- print("-" * 70)
- for theme, stats in list(results["by_theme"].items())[:10]:
- print(
- f" {theme:>20}: {stats['count']:4} puzzles | "
- f"First: {stats['first_move_accuracy']:5.2f}% | "
- f"Full: {stats['full_solution_accuracy']:5.2f}%"
- )
-
- # Save results
- if args.output:
- output_path = args.output
- else:
- timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
- output_path = f"results/puzzle_eval_{timestamp}.json"
-
- output_path = Path(output_path)
- output_path.parent.mkdir(parents=True, exist_ok=True)
-
- # Don't save individual puzzle results to keep file size manageable
- results_summary = {k: v for k, v in results.items() if k != "puzzle_results"}
- results_summary["num_puzzles_detailed"] = len(results["puzzle_results"])
-
- with open(output_path, "w") as f:
- json.dump(results_summary, f, indent=2)
-
- print(f"\nResults saved to {output_path}")
- print("=" * 70)
-
-
-if __name__ == "__main__":
- main()
diff --git a/frontend/.dockerignore b/frontend/.dockerignore
new file mode 100644
index 0000000000000000000000000000000000000000..0cf40507c703ad69c16e23c68d6237302cca9234
--- /dev/null
+++ b/frontend/.dockerignore
@@ -0,0 +1,36 @@
+# Dependencies
+node_modules
+.pnp
+.pnp.js
+
+# Build outputs
+.next
+out
+build
+
+# Testing
+coverage
+
+# Misc
+.DS_Store
+*.pem
+
+# Debug
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+
+# Local env files
+.env*.local
+.env
+
+# Vercel
+.vercel
+
+# TypeScript
+*.tsbuildinfo
+next-env.d.ts
+
+# IDE
+.idea
+.vscode
diff --git a/frontend/.gitignore b/frontend/.gitignore
new file mode 100644
index 0000000000000000000000000000000000000000..5ef6a520780202a1d6addd833d800ccb1ecac0bb
--- /dev/null
+++ b/frontend/.gitignore
@@ -0,0 +1,41 @@
+# See https://help.github.com/articles/ignoring-files/ for more about ignoring files.
+
+# dependencies
+/node_modules
+/.pnp
+.pnp.*
+.yarn/*
+!.yarn/patches
+!.yarn/plugins
+!.yarn/releases
+!.yarn/versions
+
+# testing
+/coverage
+
+# next.js
+/.next/
+/out/
+
+# production
+/build
+
+# misc
+.DS_Store
+*.pem
+
+# debug
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+.pnpm-debug.log*
+
+# env files (can opt-in for committing if needed)
+.env*
+
+# vercel
+.vercel
+
+# typescript
+*.tsbuildinfo
+next-env.d.ts
diff --git a/frontend/Dockerfile b/frontend/Dockerfile
new file mode 100644
index 0000000000000000000000000000000000000000..3b9f657d15df41262010e717ea2d7583edf58aad
--- /dev/null
+++ b/frontend/Dockerfile
@@ -0,0 +1,50 @@
+# Frontend Dockerfile for ChessTransformer Next.js app
+FROM node:20-alpine AS base
+
+# Install dependencies only when needed
+FROM base AS deps
+RUN apk add --no-cache libc6-compat
+WORKDIR /app
+
+# Copy package files
+COPY package.json package-lock.json* ./
+
+# Install dependencies
+RUN npm ci
+
+# Rebuild the source code only when needed
+FROM base AS builder
+WORKDIR /app
+COPY --from=deps /app/node_modules ./node_modules
+COPY . .
+
+# Set the API URL for build time
+ARG NEXT_PUBLIC_API_URL=http://localhost:5001
+ENV NEXT_PUBLIC_API_URL=$NEXT_PUBLIC_API_URL
+
+# Build the application
+RUN npm run build
+
+# Production image, copy all the files and run next
+FROM base AS runner
+WORKDIR /app
+
+ENV NODE_ENV=production
+
+# Create non-root user
+RUN addgroup --system --gid 1001 nodejs
+RUN adduser --system --uid 1001 nextjs
+
+# Copy built application
+COPY --from=builder /app/public ./public
+COPY --from=builder --chown=nextjs:nodejs /app/.next/standalone ./
+COPY --from=builder --chown=nextjs:nodejs /app/.next/static ./.next/static
+
+USER nextjs
+
+EXPOSE 3000
+
+ENV PORT=3000
+ENV HOSTNAME="0.0.0.0"
+
+CMD ["node", "server.js"]
diff --git a/frontend/README.md b/frontend/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..df9922c569e7df198bb3a3068ae1f103ebee5d9e
--- /dev/null
+++ b/frontend/README.md
@@ -0,0 +1,157 @@
+# ChessTransformer Frontend
+
+A Next.js web application for testing the ChessTransformer bot interactively. Play chess as a human against the AI-powered chess bot.
+
+## Features
+
+- 🎮 Interactive chessboard UI
+- 🤖 Real-time bot integration
+- 📊 Move history display
+- ✅ Game status tracking (check, checkmate, draw)
+- 🎯 Intuitive and beginner-friendly interface
+
+## Prerequisites
+
+- Node.js 18+ and npm
+- Python 3.12+
+- UV package manager (for backend dependencies)
+
+## Setup
+
+### 1. Install Backend Dependencies
+
+From the project root directory:
+
+```bash
+cd /path/to/ChessTransformer
+uv sync
+```
+
+This will install all Python dependencies including FastAPI and uvicorn.
+
+### 2. Install Frontend Dependencies
+
+```bash
+cd frontend
+npm install
+```
+
+## Running the Application
+
+You need to run both the backend API server and the frontend development server.
+
+### Start the Backend API Server
+
+From the project root directory:
+
+```bash
+cd backend
+uv run python api.py
+```
+
+The API server will start on `http://localhost:5000`.
+
+**Note:** The backend will automatically fall back to using RandomBot if the Position2MoveBot model is not available.
+
+### Start the Frontend Development Server
+
+In a separate terminal, from the project root:
+
+```bash
+cd frontend
+npm run dev
+```
+
+The frontend will start on `http://localhost:3000`.
+
+## Usage
+
+1. Open your browser and navigate to `http://localhost:3000`
+2. You'll see a chessboard with you playing as White
+3. Make your move by dragging and dropping pieces
+4. The bot (playing as Black) will automatically respond
+5. The move history is displayed on the right side
+6. Click "New Game" to start over at any time
+
+## Configuration
+
+The frontend API URL can be configured via environment variable:
+
+Create a `.env.local` file in the `frontend` directory:
+
+```
+NEXT_PUBLIC_API_URL=http://localhost:5000
+```
+
+## Building for Production
+
+### Build the Frontend
+
+```bash
+cd frontend
+npm run build
+npm start
+```
+
+This will create an optimized production build.
+
+### Deploy
+
+The frontend can be deployed to platforms like Vercel, Netlify, or any Node.js hosting service.
+
+The backend API should be deployed separately and the `NEXT_PUBLIC_API_URL` environment variable should be updated to point to the production API URL.
+
+## Technology Stack
+
+- **Frontend:**
+ - Next.js 15 (React framework)
+ - TypeScript
+ - Tailwind CSS
+ - chess.js (Chess logic)
+ - react-chessboard (Chessboard UI)
+
+- **Backend:**
+ - FastAPI (Python web framework)
+ - python-chess (Chess engine)
+ - ChessTransformer models
+
+## Troubleshooting
+
+### Backend Connection Error
+
+If you see "Failed to connect to the backend" error:
+- Ensure the backend API server is running on port 5000
+- Check that CORS is enabled in the API
+- Verify the API URL in your environment variables
+
+### Bot Not Loading
+
+If the bot type shows "RandomBot" instead of "Position2MoveBot":
+- This is expected if the model files are not available
+- The RandomBot will work fine for testing the interface
+- To use the full Position2MoveBot, ensure model files are in the correct location
+
+## API Endpoints
+
+The backend provides the following endpoints:
+
+- `GET /api/health` - Health check and bot type
+- `POST /api/move` - Get bot's next move
+- `POST /api/new-game` - Start a new game
+- `POST /api/validate-move` - Validate a move
+
+## Learn More
+
+To learn more about Next.js, take a look at the following resources:
+
+- [Next.js Documentation](https://nextjs.org/docs)
+- [Learn Next.js](https://nextjs.org/learn)
+
+## Contributing
+
+Contributions are welcome! Please feel free to submit issues or pull requests.
+
+## License
+
+MIT
+
diff --git a/frontend/app/components/Avatars.tsx b/frontend/app/components/Avatars.tsx
new file mode 100644
index 0000000000000000000000000000000000000000..603c4628d5f6384f9dea2937cb7cf0a9cfe64fb2
--- /dev/null
+++ b/frontend/app/components/Avatars.tsx
@@ -0,0 +1,91 @@
+'use client';
+
+import { useEffect, useState } from 'react';
+import type { Color } from 'chess.js';
+import { defaultPieces } from 'react-chessboard';
+
+/**
+ * How strongly a query square attends to another, using the chess-geometry
+ * relations the model's attention bias is built on: king-adjacent, knight
+ * reach, shared file/rank, shared diagonal, everything else.
+ */
+function attention(q: number, s: number): number {
+ if (q === s) return 1;
+ const df = Math.abs((q % 8) - (s % 8));
+ const dr = Math.abs((q >> 3) - (s >> 3));
+ const d = Math.max(df, dr);
+ if (d === 1) return 0.7;
+ if (df * dr === 2) return 0.56;
+ if (df === 0 || dr === 0) return 0.48 - d * 0.045;
+ if (df === dr) return 0.42 - d * 0.04;
+ return 0.05;
+}
+
+const IDLE_QUERY = 28; // e4 from the avatar's point of view
+
+/**
+ * The engine's avatar: an 8×8 attention map. While the engine searches, the
+ * query square wanders the board and the map follows it.
+ */
+export function BotAvatar({ size = 40, thinking = false }: { size?: number; thinking?: boolean }) {
+ const [query, setQuery] = useState(IDLE_QUERY);
+
+ useEffect(() => {
+ if (!thinking || window.matchMedia('(prefers-reduced-motion: reduce)').matches) return;
+ let seed = 17;
+ const id = window.setInterval(() => {
+ seed = (seed * 1103515245 + 12345) & 0x7fffffff;
+ setQuery((q) => {
+ // a knight-ish hop, wrapped onto the board, keeps the motion legible
+ const hops = [17, 15, 10, 6, -6, -10, -15, -17, 9, -9, 7, -7];
+ return (q + hops[seed % hops.length] + 64) % 64;
+ });
+ }, 240);
+ return () => {
+ window.clearInterval(id);
+ setQuery(IDLE_QUERY);
+ };
+ }, [thinking]);
+
+ const gap = size >= 64 ? 1 : 0;
+ return (
+
+ );
+}
diff --git a/frontend/app/components/SearchView.tsx b/frontend/app/components/SearchView.tsx
new file mode 100644
index 0000000000000000000000000000000000000000..9492d11208f08326ff6186497b52220fbce83e6c
--- /dev/null
+++ b/frontend/app/components/SearchView.tsx
@@ -0,0 +1,227 @@
+'use client';
+
+import { useMemo } from 'react';
+import { Chess, type Color } from 'chess.js';
+import { formatEval } from '../core/chess';
+import type { SearchInfo, SearchNode } from '../core/game';
+
+/** A searched move with its SAN and the UCI path that leads to it from the searched position. */
+interface Branch {
+ id: string;
+ san: string;
+ color: Color;
+ visits: number;
+ q: number;
+ path: string[];
+ children: Branch[];
+}
+
+function annotate(fen: string, nodes: SearchNode[], path: string[] = []): Branch[] {
+ const out: Branch[] = [];
+ for (const n of nodes) {
+ const g = new Chess(fen);
+ try {
+ const m = g.move({ from: n.move.slice(0, 2), to: n.move.slice(2, 4), promotion: n.move[4] });
+ const p = [...path, n.move];
+ out.push({
+ id: p.join(' '),
+ san: m.san,
+ color: m.color,
+ visits: n.visits,
+ q: n.q,
+ path: p,
+ children: annotate(g.fen(), n.children, p),
+ });
+ } catch {
+ // a move that doesn't parse here would mean a stale tree; skip it
+ }
+ }
+ return out;
+}
+
+/** The branch followed by always taking the most-visited reply. */
+const mainLine = (b: Branch): Branch[] => (b.children[0] ? [b, ...mainLine(b.children[0])] : [b]);
+
+// ---------------------------------------------------------------- tree layout
+
+const MAX_DEPTH = 6;
+const ROW = 20;
+const COL = 56;
+const ROOT_X = 8;
+const FIRST_X = 26;
+const LABEL_W = 40;
+
+interface Placed {
+ branch: Branch;
+ depth: number;
+ x: number;
+ y: number;
+ parent: Placed | null;
+}
+
+/** Tidy tree, left to right: leaves take consecutive rows, parents centre on their children. */
+function layout(roots: Branch[]) {
+ const placed: Placed[] = [];
+ let row = 0;
+ const place = (branch: Branch, depth: number, parent: Placed | null): Placed => {
+ const me: Placed = { branch, depth, x: FIRST_X + (depth - 1) * COL, y: 0, parent };
+ placed.push(me);
+ const kids = depth < MAX_DEPTH ? branch.children : [];
+ if (!kids.length) {
+ me.y = row++ * ROW + ROW / 2;
+ } else {
+ const ys = kids.map((k) => place(k, depth + 1, me).y);
+ me.y = (ys[0] + ys[ys.length - 1]) / 2;
+ }
+ return me;
+ };
+ const tops = roots.map((r) => place(r, 1, null));
+ const rootY = tops.length ? (tops[0].y + tops[tops.length - 1].y) / 2 : ROW / 2;
+ return { placed, rootY, height: Math.max(row, 1) * ROW };
+}
+
+// ---------------------------------------------------------------- view
+
+interface Props {
+ search: SearchInfo;
+ /** The searched position. */
+ fen: string;
+ /** SAN of the move the engine played from it, once known. */
+ played?: string;
+ /** Path being previewed on the board, if any. */
+ active: string[] | null;
+ onPreview: (path: string[] | null, pin?: boolean) => void;
+}
+
+export default function SearchView({ search, fen, played, active, onPreview }: Props) {
+ const roots = useMemo(() => annotate(fen, search.tree), [fen, search.tree]);
+ const { placed, rootY, height } = useMemo(() => layout(roots), [roots]);
+ const total = Math.max(1, search.visits);
+ const activeId = active?.join(' ') ?? null;
+ const chosen = roots.find((r) => r.san === played) ?? roots[0];
+ const mainIds = useMemo(() => new Set(chosen ? mainLine(chosen).map((b) => b.id) : []), [chosen]);
+ const onActive = (id: string) => !!activeId && (activeId === id || activeId.startsWith(`${id} `));
+
+ const hover = (path: string[]) => ({
+ onMouseEnter: () => onPreview(path),
+ onMouseLeave: () => onPreview(null),
+ onFocus: () => onPreview(path),
+ onBlur: () => onPreview(null),
+ onClick: () => onPreview(path, true),
+ });
+
+ return (
+
+
+ {played ? (
+ <>
+ Chose {played} after
+ >
+ ) : (
+ 'Weighed'
+ )}{' '}
+ {search.visits.toLocaleString()} positions in {(search.elapsedMs / 1000).toFixed(1)} s. Bars show each
+ move's share of the search; hover or tap one to replay its line on the board.
+
+ {search.depth ? (
+
+ Its main line runs {search.depth.mainLine} half-moves deep and
+ the deepest branch {search.depth.max}; a typical simulation stops after {search.depth.mean.toFixed(0)}. The
+ diagram below shows only the first {MAX_DEPTH}.
+
+ {engine?.elo ? `Rated about ${engine.elo}` : offline ? 'Engine offline' : 'Connecting to the engine'}
+
+
+ A transformer that learned chess from human games only, then searches{' '}
+ {engine?.sims ? `${engine.sims} positions` : 'ahead'} before each move.
+