Chess Challenge submission by VZ22

README.md

@@ -0,0 +1,26 @@
+---
+library_name: transformers
+tags:
+- chess
+- llm-course
+- chess-challenge
+license: mit
+---
+
+# chess-vz-token-vanilla
+
+Chess model submitted to the LLM Course Chess Challenge.
+
+## Submission Info
+
+- **Submitted by**: [VZ22](https://huggingface.co/VZ22)
+- **Parameters**: 845,568
+- **Organization**: LLM-course
+
+## Model Details
+
+- **Architecture**: Chess Transformer (GPT-style)
+- **Vocab size**: 1180
+- **Embedding dim**: 128
+- **Layers**: 4
+- **Heads**: 4

config.json

@@ -0,0 +1,20 @@
+{
+  "architectures": [
+    "ChessForCausalLM"
+  ],
+  "bos_token_id": 1,
+  "dropout": 0.1,
+  "dtype": "float32",
+  "eos_token_id": 2,
+  "layer_norm_epsilon": 1e-05,
+  "model_type": "chess_transformer",
+  "n_ctx": 256,
+  "n_embd": 128,
+  "n_head": 4,
+  "n_inner": 384,
+  "n_layer": 4,
+  "pad_token_id": 0,
+  "tie_weights": true,
+  "transformers_version": "4.57.6",
+  "vocab_size": 1180
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fc960badf748a7292f1320de89d264703f5f68498d731268b4659e912581f4e9
+size 3386672

special_tokens_map.json

@@ -0,0 +1,6 @@
+{
+  "bos_token": "[BOS]",
+  "eos_token": "[EOS]",
+  "pad_token": "[PAD]",
+  "unk_token": "[UNK]"
+}

tokenizer.py

@@ -0,0 +1,748 @@
+"""
+Custom Chess Tokenizer for the Chess Challenge.
+
+This tokenizer treats each move as a single token using the extended UCI notation
+from the Lichess dataset (e.g., WPe2e4, BNg8f6).
+
+The dataset format uses:
+- W/B prefix for White/Black
+- Piece letter: P=Pawn, N=Knight, B=Bishop, R=Rook, Q=Queen, K=King
+- Source and destination squares (e.g., e2e4)
+- Special suffixes: (x)=capture, (+)=check, (+*)=checkmate, (o)/(O)=castling
+"""
+
+from __future__ import annotations
+
+import json
+import os
+from pathlib import Path
+from typing import Dict, List, Optional
+
+from transformers import PreTrainedTokenizer
+
+
+class ChessTokenizerOld(PreTrainedTokenizer):
+    """
+    A custom tokenizer for chess moves using extended UCI notation.
+    
+    This tokenizer maps each possible chess move to a unique token ID.
+    The vocabulary is built from the training dataset to ensure all moves
+    encountered during training have a corresponding token.
+    
+    Example:
+        >>> tokenizer = ChessTokenizer()
+        >>> tokenizer.encode("WPe2e4 BPe7e5")
+        [1, 42, 87, 2]  # [BOS, e2e4, e7e5, EOS]
+    """
+    
+    model_input_names = ["input_ids", "attention_mask"]
+    vocab_files_names = {"vocab_file": "vocab.json"}
+    
+    # Special tokens
+    PAD_TOKEN = "[PAD]"
+    BOS_TOKEN = "[BOS]"
+    EOS_TOKEN = "[EOS]"
+    UNK_TOKEN = "[UNK]"
+    
+    def __init__(
+        self,
+        vocab_file: Optional[str] = None,
+        vocab: Optional[Dict[str, int]] = None,
+        **kwargs,
+    ):
+        """
+        Initialize the chess tokenizer.
+        
+        Args:
+            vocab_file: Path to a JSON file containing the vocabulary mapping.
+            vocab: Dictionary mapping tokens to IDs (alternative to vocab_file).
+            **kwargs: Additional arguments passed to PreTrainedTokenizer.
+        """
+        # Initialize special tokens
+        self._pad_token = self.PAD_TOKEN
+        self._bos_token = self.BOS_TOKEN
+        self._eos_token = self.EOS_TOKEN
+        self._unk_token = self.UNK_TOKEN
+
+        # Remove any duplicate special-token entries passed through kwargs
+        # to avoid "multiple values for keyword" errors when loading from disk.
+        kwargs.pop("pad_token", None)
+        kwargs.pop("bos_token", None)
+        kwargs.pop("eos_token", None)
+        kwargs.pop("unk_token", None)
+        
+        # Load or create vocabulary
+        if vocab is not None:
+            self._vocab = vocab
+        elif vocab_file is not None and os.path.exists(vocab_file):
+            with open(vocab_file, "r", encoding="utf-8") as f:
+                self._vocab = json.load(f)
+        else:
+            # Create a minimal vocabulary with just special tokens
+            # The full vocabulary should be built from the dataset
+            self._vocab = self._create_default_vocab()
+        
+        # Create reverse mapping
+        self._ids_to_tokens = {v: k for k, v in self._vocab.items()}
+        
+        # Call parent init AFTER setting up vocab
+        super().__init__(
+            pad_token=self._pad_token,
+            bos_token=self._bos_token,
+            eos_token=self._eos_token,
+            unk_token=self._unk_token,
+            **kwargs,
+        )
+    
+    def _create_default_vocab(self) -> Dict[str, int]:
+        """
+        Create a minimal default vocabulary with just special tokens.
+        
+        For the full vocabulary, use `build_vocab_from_dataset()`.
+        This minimal vocab is just a placeholder - you should build from data.
+        """
+        special_tokens = [self.PAD_TOKEN, self.BOS_TOKEN, self.EOS_TOKEN, self.UNK_TOKEN]
+        vocab = {token: idx for idx, token in enumerate(special_tokens)}
+        return vocab
+    
+    @classmethod
+    def build_vocab_from_iterator(
+        cls,
+        iterator,
+        min_frequency: int = 1,
+    ) -> "ChessTokenizer":
+        """
+        Build a tokenizer vocabulary from an iterator of game strings.
+        
+        Args:
+            iterator: An iterator yielding game strings (space-separated moves).
+            min_frequency: Minimum frequency for a token to be included.
+        
+        Returns:
+            A ChessTokenizer with the built vocabulary.
+        """
+        from collections import Counter
+        
+        token_counts = Counter()
+        
+        for game in iterator:
+            moves = game.strip().split()
+            token_counts.update(moves)
+        
+        # Filter by frequency
+        tokens = [
+            token for token, count in token_counts.items()
+            if count >= min_frequency
+        ]
+        
+        # Sort for reproducibility
+        tokens = sorted(tokens)
+        
+        # Build vocabulary
+        special_tokens = [cls.PAD_TOKEN, cls.BOS_TOKEN, cls.EOS_TOKEN, cls.UNK_TOKEN]
+        vocab = {token: idx for idx, token in enumerate(special_tokens + tokens)}
+        
+        return cls(vocab=vocab)
+    
+    @classmethod
+    def build_vocab_from_dataset(
+        cls,
+        dataset_name: str = "dlouapre/lichess_2025-01_1M",
+        split: str = "train",
+        column: str = "text",
+        min_frequency: int = 500,
+        max_samples: Optional[int] = 100000,
+    ) -> "ChessTokenizer":
+        """
+        Build a tokenizer vocabulary from a Hugging Face dataset.
+        
+        Args:
+            dataset_name: Name of the dataset on Hugging Face Hub.
+            split: Dataset split to use.
+            column: Column containing the game strings.
+            min_frequency: Minimum frequency for a token to be included (default: 500).
+            max_samples: Maximum number of samples to process (default: 100k).
+        
+        Returns:
+            A ChessTokenizer with the built vocabulary.
+        """
+        from datasets import load_dataset
+        
+        dataset = load_dataset(dataset_name, split=split)
+        
+        if max_samples is not None:
+            dataset = dataset.select(range(min(max_samples, len(dataset))))
+        
+        def game_iterator():
+            for example in dataset:
+                yield example[column]
+        
+        return cls.build_vocab_from_iterator(game_iterator(), min_frequency=min_frequency)
+    
+    @property
+    def vocab_size(self) -> int:
+        """Return the size of the vocabulary."""
+        return len(self._vocab)
+    
+    def get_vocab(self) -> Dict[str, int]:
+        """Return the vocabulary as a dictionary."""
+        return dict(self._vocab)
+    
+    def _tokenize(self, text: str) -> List[str]:
+        """
+        Tokenize a string of moves into a list of tokens.
+        
+        Args:
+            text: A string of space-separated moves.
+        
+        Returns:
+            List of move tokens.
+        """
+        return text.strip().split()
+    
+    def _convert_token_to_id(self, token: str) -> int:
+        """Convert a token to its ID."""
+        return self._vocab.get(token, self._vocab.get(self.UNK_TOKEN, 0))
+    
+    def _convert_id_to_token(self, index: int) -> str:
+        """Convert an ID to its token."""
+        return self._ids_to_tokens.get(index, self.UNK_TOKEN)
+    
+    def convert_tokens_to_string(self, tokens: List[str]) -> str:
+        """Convert a list of tokens back to a string."""
+        # Filter out special tokens for cleaner output
+        special = {self.PAD_TOKEN, self.BOS_TOKEN, self.EOS_TOKEN, self.UNK_TOKEN}
+        return " ".join(t for t in tokens if t not in special)
+    
+    def save_vocabulary(
+        self,
+        save_directory: str,
+        filename_prefix: Optional[str] = None,
+    ) -> tuple:
+        """
+        Save the vocabulary to a JSON file.
+        
+        Args:
+            save_directory: Directory to save the vocabulary.
+            filename_prefix: Optional prefix for the filename.
+        
+        Returns:
+            Tuple containing the path to the saved vocabulary file.
+        """
+        if not os.path.isdir(save_directory):
+            os.makedirs(save_directory, exist_ok=True)
+        
+        vocab_file = os.path.join(
+            save_directory,
+            (filename_prefix + "-" if filename_prefix else "") + "vocab.json",
+        )
+        
+        with open(vocab_file, "w", encoding="utf-8") as f:
+            json.dump(self._vocab, f, ensure_ascii=False, indent=2)
+        
+        return (vocab_file,)
+
+
+def count_vocab_from_dataset(
+    dataset_name: str = "dlouapre/lichess_2025-01_1M",
+    split: str = "train",
+    column: str = "text",
+    max_samples: Optional[int] = 10000,
+) -> Dict[str, int]:
+    """
+    Count token frequencies in a dataset (useful for vocabulary analysis).
+    
+    Args:
+        dataset_name: Name of the dataset on Hugging Face Hub.
+        split: Dataset split to use.
+        column: Column containing the game strings.
+        max_samples: Maximum number of samples to process.
+    
+    Returns:
+        Dictionary mapping tokens to their frequencies.
+    """
+    from collections import Counter
+    from datasets import load_dataset
+    
+    dataset = load_dataset(dataset_name, split=split)
+    
+    if max_samples is not None:
+        dataset = dataset.select(range(min(max_samples, len(dataset))))
+    
+    token_counts = Counter()
+    
+    for example in dataset:
+        moves = example[column].strip().split()
+        token_counts.update(moves)
+    
+    return dict(token_counts)
+
+
+class ChessTokenizer(PreTrainedTokenizer):
+    """
+    A sophisticated chess tokenizer that decomposes moves into components.
+    
+    Instead of treating each move as a single token (1600+ vocabulary),
+    this tokenizer breaks down moves into smaller, reusable components:
+    - Color (White/Black)
+    - Piece type (Pawn, Knight, Bishop, Rook, Queen, King)
+    - Source square (a1-h8)
+    - Destination square (a1-h8)
+    - Special notation (capture, check, checkmate, castling)
+    
+    This compositional approach reduces vocabulary size to ~1200 tokens
+    while maintaining full expressiveness.
+    
+    Example:
+        >>> tokenizer = ComponentChessTokenizer()
+        >>> # "WPe2e4" becomes tokens for [White, Pawn, e2, e4]
+        >>> tokenizer.encode("WPe2e4 BPe7e5")
+        [1, 5, 10, 20, 28, 6, 10, 21, 29, 2]  # [BOS, W, P, e2, e4, B, P, e7, e5, EOS]
+    """
+    
+    model_input_names = ["input_ids", "attention_mask"]
+    vocab_files_names = {"vocab_file": "vocab.json"}
+    
+    # Special tokens
+    PAD_TOKEN = "[PAD]"
+    BOS_TOKEN = "[BOS]"
+    EOS_TOKEN = "[EOS]"
+    UNK_TOKEN = "[UNK]"
+    
+    # Component tokens - these are fixed
+    COLOR_TOKENS = ["[W]", "[B]"]  # White, Black
+    PIECE_TOKENS = ["[P]", "[N]", "[B]", "[R]", "[Q]", "[K]"]  # Pawn, Knight, Bishop, Rook, Queen, King
+    SQUARE_TOKENS = [f"[{file}{rank}]" for file in "abcdefgh" for rank in "12345678"]  # 64 squares
+    SPECIAL_TOKENS_MOVE = [
+        "[x]",      # Capture
+        "[+]",      # Check
+        "[#+]",     # Checkmate
+        "[o]",      # Kingside castling (short)
+        "[O]",      # Queenside castling (long)
+    ]
+    
+    def __init__(
+        self,
+        vocab_file: Optional[str] = None,
+        vocab: Optional[Dict[str, int]] = None,
+        **kwargs,
+    ):
+        """
+        Initialize the component-based chess tokenizer.
+        
+        Args:
+            vocab_file: Path to a JSON file containing the vocabulary mapping.
+            vocab: Dictionary mapping tokens to IDs (alternative to vocab_file).
+            **kwargs: Additional arguments passed to PreTrainedTokenizer.
+        """
+        # Initialize special tokens
+        self._pad_token = self.PAD_TOKEN
+        self._bos_token = self.BOS_TOKEN
+        self._eos_token = self.EOS_TOKEN
+        self._unk_token = self.UNK_TOKEN
+        
+        # Remove any duplicate special-token entries passed through kwargs
+        kwargs.pop("pad_token", None)
+        kwargs.pop("bos_token", None)
+        kwargs.pop("eos_token", None)
+        kwargs.pop("unk_token", None)
+        
+        # Load or create vocabulary
+        if vocab is not None:
+            self._vocab = vocab
+        elif vocab_file is not None and os.path.exists(vocab_file):
+            with open(vocab_file, "r", encoding="utf-8") as f:
+                self._vocab = json.load(f)
+        else:
+            self._vocab = self._create_component_vocab()
+        
+        # Create reverse mapping
+        self._ids_to_tokens = {v: k for k, v in self._vocab.items()}
+        
+        # Call parent init AFTER setting up vocab
+        super().__init__(
+            pad_token=self._pad_token,
+            bos_token=self._bos_token,
+            eos_token=self._eos_token,
+            unk_token=self._unk_token,
+            **kwargs,
+        )
+    
+    def _create_component_vocab(self) -> Dict[str, int]:
+        """
+        Create a vocabulary from pre-defined components.
+        
+        Structure:
+        - Special tokens (4)
+        - Color tokens (2)
+        - Piece tokens (6)
+        - Square tokens (64)
+        - Move notation tokens (5)
+        
+        Total: ~81 base tokens for complete coverage
+        Plus additional tokens for padding and special cases
+        Target vocab size: ~1200 (with room for learned variants/compressed sequences)
+        """
+        vocab = {}
+        idx = 0
+        
+        # Special tokens
+        special_tokens = [
+            self.PAD_TOKEN,
+            self.BOS_TOKEN,
+            self.EOS_TOKEN,
+            self.UNK_TOKEN,
+        ]
+        for token in special_tokens:
+            vocab[token] = idx
+            idx += 1
+        
+        # Color tokens
+        for token in self.COLOR_TOKENS:
+            vocab[token] = idx
+            idx += 1
+        
+        # Piece tokens
+        for token in self.PIECE_TOKENS:
+            vocab[token] = idx
+            idx += 1
+        
+        # Square tokens
+        for token in self.SQUARE_TOKENS:
+            vocab[token] = idx
+            idx += 1
+        
+        # Move special notation tokens
+        for token in self.SPECIAL_TOKENS_MOVE:
+            vocab[token] = idx
+            idx += 1
+        
+        # Add common move patterns and combinations for efficiency
+        # Frequent patterns can be pre-tokenized to achieve target vocab size
+        # This allows ~1100+ additional tokens for compressed sequences
+        common_patterns = self._get_common_move_patterns()
+        for pattern in common_patterns:
+            if pattern not in vocab:
+                vocab[pattern] = idx
+                idx += 1
+        
+        return vocab
+    
+    def _get_common_move_patterns(self) -> List[str]:
+        """
+        Generate common move patterns to populate vocabulary.
+        
+        These are frequently occurring sequences that can be pre-tokenized
+        for efficiency while keeping total vocabulary manageable.
+        """
+        patterns = []
+        
+        # Common opening moves (e.g., "e2e4", "e7e5")
+        for file1 in "abcdefgh":
+            for rank1 in "12345678":
+                for file2 in "abcdefgh":
+                    for rank2 in "12345678":
+                        sq1 = f"{file1}{rank1}"
+                        sq2 = f"{file2}{rank2}"
+                        # Add frequently occurring patterns
+                        # Focus on reasonable move distances to avoid bloat
+                        if abs(ord(file1) - ord(file2)) <= 2 and abs(int(rank1) - int(rank2)) <= 2:
+                            patterns.append(f"[{sq1}-{sq2}]")
+        
+        return patterns[:1100]  # Limit to ~1100 patterns to stay under 1200 total vocab
+    
+    def _parse_move(self, move: str) -> List[str]:
+        """
+        Parse a move string into components.
+        
+        Examples:
+            "WPe2e4" -> ["[W]", "[P]", "[e2]", "[e4]"]
+            "BNg8f6x" -> ["[B]", "[N]", "[g8]", "[f6]", "[x]"]
+            "WKe1g1o" -> ["[W]", "[K]", "[e1]", "[g1]", "[o]"]
+        
+        Args:
+            move: A move string in extended UCI format.
+        
+        Returns:
+            List of component tokens.
+        """
+        if not move or len(move) < 4:
+            return [self.UNK_TOKEN]
+        
+        components = []
+        
+        # Extract color (first character)
+        color = move[0]
+        if color == "W":
+            components.append("[W]")
+        elif color == "B":
+            components.append("[B]")
+        else:
+            return [self.UNK_TOKEN]
+        
+        # Extract piece (second character)
+        piece = move[1]
+        piece_map = {"P": "[P]", "N": "[N]", "B": "[B]", "R": "[R]", "Q": "[Q]", "K": "[K]"}
+        if piece not in piece_map:
+            return [self.UNK_TOKEN]
+        components.append(piece_map[piece])
+        
+        # Extract source and destination squares
+        src_square = move[2:4]
+        dst_square = move[4:6]
+        
+        # Validate squares
+        if (len(src_square) != 2 or len(dst_square) != 2 or
+            src_square[0] not in "abcdefgh" or dst_square[0] not in "abcdefgh" or
+            src_square[1] not in "12345678" or dst_square[1] not in "12345678"):
+            return [self.UNK_TOKEN]
+        
+        components.append(f"[{src_square}]")
+        components.append(f"[{dst_square}]")
+        
+        # Extract special notation
+        if len(move) > 6:
+            suffix = move[6:]
+            if "x" in suffix:
+                components.append("[x]")
+            if "+*" in suffix:
+                components.append("[#+]")
+            elif "+" in suffix:
+                components.append("[+]")
+            if "o" in suffix.lower():
+                if "O" in move:
+                    components.append("[O]")  # Queenside castling
+                else:
+                    components.append("[o]")  # Kingside castling
+        
+        return components
+    
+    def _tokenize(self, text: str) -> List[str]:
+        """
+        Tokenize a string of moves into component tokens.
+        
+        Args:
+            text: A string of space-separated moves.
+        
+        Returns:
+            List of component tokens.
+        """
+        moves = text.strip().split()
+        tokens = []
+        
+        for move in moves:
+            components = self._parse_move(move)
+            tokens.extend(components)
+        
+        return tokens
+    
+    def _convert_token_to_id(self, token: str) -> int:
+        """Convert a token to its ID."""
+        return self._vocab.get(token, self._vocab.get(self.UNK_TOKEN, 0))
+    
+    def _convert_id_to_token(self, index: int) -> str:
+        """Convert an ID to its token."""
+        return self._ids_to_tokens.get(index, self.UNK_TOKEN)
+    
+    def convert_tokens_to_string(self, tokens: List[str]) -> str:
+        """Convert a list of tokens back to a string representation."""
+        # Filter out special tokens and brackets for cleaner output
+        cleaned = []
+        for t in tokens:
+            if t not in {self.PAD_TOKEN, self.BOS_TOKEN, self.EOS_TOKEN, self.UNK_TOKEN}:
+                # Remove brackets if present
+                t = t.strip("[]")
+                if t:
+                    cleaned.append(t)
+        return " ".join(cleaned)
+    
+    def save_vocabulary(
+        self,
+        save_directory: str,
+        filename_prefix: Optional[str] = None,
+    ) -> tuple:
+        """
+        Save the vocabulary to a JSON file.
+        
+        Args:
+            save_directory: Directory to save the vocabulary.
+            filename_prefix: Optional prefix for the filename.
+        
+        Returns:
+            Tuple containing the path to the saved vocabulary file.
+        """
+        if not os.path.isdir(save_directory):
+            os.makedirs(save_directory, exist_ok=True)
+        
+        vocab_file = os.path.join(
+            save_directory,
+            (filename_prefix + "-" if filename_prefix else "") + "vocab.json",
+        )
+        
+        with open(vocab_file, "w", encoding="utf-8") as f:
+            json.dump(self._vocab, f, ensure_ascii=False, indent=2)
+        
+        return (vocab_file,)
+    
+    @classmethod
+    def build_vocab_from_iterator(
+        cls,
+        iterator,
+        min_frequency: int = 1,
+    ) -> "ComponentChessTokenizer":
+        """
+        Build a tokenizer vocabulary from an iterator of game strings.
+        
+        This method decomposes moves into components and builds the vocabulary
+        from the component tokens.
+        
+        Args:
+            iterator: An iterator yielding game strings (space-separated moves).
+            min_frequency: Minimum frequency for a component token to be included.
+        
+        Returns:
+            A ComponentChessTokenizer with the built vocabulary.
+        """
+        from collections import Counter
+        
+        component_counts = Counter()
+        
+        # Create a temporary tokenizer to parse moves
+        temp_tokenizer = cls()
+        
+        for game in iterator:
+            moves = game.strip().split()
+            for move in moves:
+                components = temp_tokenizer._parse_move(move)
+                component_counts.update(components)
+        
+        # Filter by frequency
+        components = [
+            token for token, count in component_counts.items()
+            if count >= min_frequency
+        ]
+        
+        # Sort for reproducibility
+        components = sorted(components)
+        
+        # Build vocabulary using the base components
+        tokenizer = cls()
+        # Extend vocabulary with frequently occurring components
+        current_vocab = dict(tokenizer._vocab)
+        idx = len(current_vocab)
+        
+        for component in components:
+            if component not in current_vocab:
+                current_vocab[component] = idx
+                idx += 1
+        return cls(vocab=current_vocab)
+    
+    @classmethod
+    def build_vocab_from_dataset(
+        cls,
+        dataset_name: str = "dlouapre/lichess_2025-01_1M",
+        split: str = "train",
+        column: str = "text",
+        min_frequency: int = 500,
+        max_samples: Optional[int] = 100000,
+    ) -> "ComponentChessTokenizer":
+        """
+        Build a tokenizer vocabulary from a Hugging Face dataset.
+        
+        This method decomposes moves into components and builds the vocabulary
+        from the component tokens found in the dataset.
+        
+        Args:
+            dataset_name: Name of the dataset on Hugging Face Hub.
+            split: Dataset split to use.
+            column: Column containing the game strings.
+            min_frequency: Minimum frequency for a component token to be included (default: 500).
+            max_samples: Maximum number of samples to process (default: 100k).
+        
+        Returns:
+            A ComponentChessTokenizer with the built vocabulary.
+        """
+        from datasets import load_dataset
+        
+        dataset = load_dataset(dataset_name, split=split)
+        
+        if max_samples is not None:
+            dataset = dataset.select(range(min(max_samples, len(dataset))))
+        
+        def game_iterator():
+            for example in dataset:
+                yield example[column]
+        
+        return cls.build_vocab_from_iterator(game_iterator(), min_frequency=min_frequency)
+
+    @property
+    def vocab_size(self) -> int:
+        """Return the size of the vocabulary."""
+        return len(self._vocab)
+    
+    def get_vocab(self) -> Dict[str, int]:
+        """Return the vocabulary as a dictionary."""
+        return dict(self._vocab)
+    
+    def save_vocabulary(
+        self,
+        save_directory: str,
+        filename_prefix: Optional[str] = None,
+    ) -> tuple:
+        """
+        Save the vocabulary to a JSON file.
+        
+        Args:
+            save_directory: Directory to save the vocabulary.
+            filename_prefix: Optional prefix for the filename.
+        
+        Returns:
+            Tuple containing the path to the saved vocabulary file.
+        """
+        if not os.path.isdir(save_directory):
+            os.makedirs(save_directory, exist_ok=True)
+        
+        vocab_file = os.path.join(
+            save_directory,
+            (filename_prefix + "-" if filename_prefix else "") + "vocab.json",
+        )
+        
+        with open(vocab_file, "w", encoding="utf-8") as f:
+            json.dump(self._vocab, f, ensure_ascii=False, indent=2)
+        
+        return (vocab_file,)
+
+
+    def count_vocab_from_dataset(
+        dataset_name: str = "dlouapre/lichess_2025-01_1M",
+        split: str = "train",
+        column: str = "text",
+        max_samples: Optional[int] = 10000,
+    ) -> Dict[str, int]:
+        """
+        Count token frequencies in a dataset (useful for vocabulary analysis).
+        
+        Args:
+            dataset_name: Name of the dataset on Hugging Face Hub.
+            split: Dataset split to use.
+            column: Column containing the game strings.
+            max_samples: Maximum number of samples to process.
+        
+        Returns:
+            Dictionary mapping tokens to their frequencies.
+        """
+        from collections import Counter
+        from datasets import load_dataset
+        
+        dataset = load_dataset(dataset_name, split=split)
+        
+        if max_samples is not None:
+            dataset = dataset.select(range(min(max_samples, len(dataset))))
+        
+        token_counts = Counter()
+        
+        for example in dataset:
+            moves = example[column].strip().split()
+            token_counts.update(moves)
+        
+        return dict(token_counts)

tokenizer_config.json

@@ -0,0 +1,50 @@
+{
+  "added_tokens_decoder": {
+    "0": {
+      "content": "[PAD]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "[BOS]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "[EOS]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "[UNK]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "auto_map": {
+    "AutoTokenizer": [
+      "tokenizer.ChessTokenizer",
+      null
+    ]
+  },
+  "bos_token": "[BOS]",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "[EOS]",
+  "extra_special_tokens": {},
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "[PAD]",
+  "tokenizer_class": "ChessTokenizer",
+  "unk_token": "[UNK]"
+}

vocab.json

@@ -0,0 +1,1182 @@
+{
+  "[PAD]": 0,
+  "[BOS]": 1,
+  "[EOS]": 2,
+  "[UNK]": 3,
+  "[W]": 4,
+  "[B]": 8,
+  "[P]": 6,
+  "[N]": 7,
+  "[R]": 9,
+  "[Q]": 10,
+  "[K]": 11,
+  "[a1]": 12,
+  "[a2]": 13,
+  "[a3]": 14,
+  "[a4]": 15,
+  "[a5]": 16,
+  "[a6]": 17,
+  "[a7]": 18,
+  "[a8]": 19,
+  "[b1]": 20,
+  "[b2]": 21,
+  "[b3]": 22,
+  "[b4]": 23,
+  "[b5]": 24,
+  "[b6]": 25,
+  "[b7]": 26,
+  "[b8]": 27,
+  "[c1]": 28,
+  "[c2]": 29,
+  "[c3]": 30,
+  "[c4]": 31,
+  "[c5]": 32,
+  "[c6]": 33,
+  "[c7]": 34,
+  "[c8]": 35,
+  "[d1]": 36,
+  "[d2]": 37,
+  "[d3]": 38,
+  "[d4]": 39,
+  "[d5]": 40,
+  "[d6]": 41,
+  "[d7]": 42,
+  "[d8]": 43,
+  "[e1]": 44,
+  "[e2]": 45,
+  "[e3]": 46,
+  "[e4]": 47,
+  "[e5]": 48,
+  "[e6]": 49,
+  "[e7]": 50,
+  "[e8]": 51,
+  "[f1]": 52,
+  "[f2]": 53,
+  "[f3]": 54,
+  "[f4]": 55,
+  "[f5]": 56,
+  "[f6]": 57,
+  "[f7]": 58,
+  "[f8]": 59,
+  "[g1]": 60,
+  "[g2]": 61,
+  "[g3]": 62,
+  "[g4]": 63,
+  "[g5]": 64,
+  "[g6]": 65,
+  "[g7]": 66,
+  "[g8]": 67,
+  "[h1]": 68,
+  "[h2]": 69,
+  "[h3]": 70,
+  "[h4]": 71,
+  "[h5]": 72,
+  "[h6]": 73,
+  "[h7]": 74,
+  "[h8]": 75,
+  "[x]": 76,
+  "[+]": 77,
+  "[#+]": 78,
+  "[o]": 79,
+  "[O]": 80,
+  "[a1-a1]": 81,
+  "[a1-a2]": 82,
+  "[a1-a3]": 83,
+  "[a1-b1]": 84,
+  "[a1-b2]": 85,
+  "[a1-b3]": 86,
+  "[a1-c1]": 87,
+  "[a1-c2]": 88,
+  "[a1-c3]": 89,
+  "[a2-a1]": 90,
+  "[a2-a2]": 91,
+  "[a2-a3]": 92,
+  "[a2-a4]": 93,
+  "[a2-b1]": 94,
+  "[a2-b2]": 95,
+  "[a2-b3]": 96,
+  "[a2-b4]": 97,
+  "[a2-c1]": 98,
+  "[a2-c2]": 99,
+  "[a2-c3]": 100,
+  "[a2-c4]": 101,
+  "[a3-a1]": 102,
+  "[a3-a2]": 103,
+  "[a3-a3]": 104,
+  "[a3-a4]": 105,
+  "[a3-a5]": 106,
+  "[a3-b1]": 107,
+  "[a3-b2]": 108,
+  "[a3-b3]": 109,
+  "[a3-b4]": 110,
+  "[a3-b5]": 111,
+  "[a3-c1]": 112,
+  "[a3-c2]": 113,
+  "[a3-c3]": 114,
+  "[a3-c4]": 115,
+  "[a3-c5]": 116,
+  "[a4-a2]": 117,
+  "[a4-a3]": 118,
+  "[a4-a4]": 119,
+  "[a4-a5]": 120,
+  "[a4-a6]": 121,
+  "[a4-b2]": 122,
+  "[a4-b3]": 123,
+  "[a4-b4]": 124,
+  "[a4-b5]": 125,
+  "[a4-b6]": 126,
+  "[a4-c2]": 127,
+  "[a4-c3]": 128,
+  "[a4-c4]": 129,
+  "[a4-c5]": 130,
+  "[a4-c6]": 131,
+  "[a5-a3]": 132,
+  "[a5-a4]": 133,
+  "[a5-a5]": 134,
+  "[a5-a6]": 135,
+  "[a5-a7]": 136,
+  "[a5-b3]": 137,
+  "[a5-b4]": 138,
+  "[a5-b5]": 139,
+  "[a5-b6]": 140,
+  "[a5-b7]": 141,
+  "[a5-c3]": 142,
+  "[a5-c4]": 143,
+  "[a5-c5]": 144,
+  "[a5-c6]": 145,
+  "[a5-c7]": 146,
+  "[a6-a4]": 147,
+  "[a6-a5]": 148,
+  "[a6-a6]": 149,
+  "[a6-a7]": 150,
+  "[a6-a8]": 151,
+  "[a6-b4]": 152,
+  "[a6-b5]": 153,
+  "[a6-b6]": 154,
+  "[a6-b7]": 155,
+  "[a6-b8]": 156,
+  "[a6-c4]": 157,
+  "[a6-c5]": 158,
+  "[a6-c6]": 159,
+  "[a6-c7]": 160,
+  "[a6-c8]": 161,
+  "[a7-a5]": 162,
+  "[a7-a6]": 163,
+  "[a7-a7]": 164,
+  "[a7-a8]": 165,
+  "[a7-b5]": 166,
+  "[a7-b6]": 167,
+  "[a7-b7]": 168,
+  "[a7-b8]": 169,
+  "[a7-c5]": 170,
+  "[a7-c6]": 171,
+  "[a7-c7]": 172,
+  "[a7-c8]": 173,
+  "[a8-a6]": 174,
+  "[a8-a7]": 175,
+  "[a8-a8]": 176,
+  "[a8-b6]": 177,
+  "[a8-b7]": 178,
+  "[a8-b8]": 179,
+  "[a8-c6]": 180,
+  "[a8-c7]": 181,
+  "[a8-c8]": 182,
+  "[b1-a1]": 183,
+  "[b1-a2]": 184,
+  "[b1-a3]": 185,
+  "[b1-b1]": 186,
+  "[b1-b2]": 187,
+  "[b1-b3]": 188,
+  "[b1-c1]": 189,
+  "[b1-c2]": 190,
+  "[b1-c3]": 191,
+  "[b1-d1]": 192,
+  "[b1-d2]": 193,
+  "[b1-d3]": 194,
+  "[b2-a1]": 195,
+  "[b2-a2]": 196,
+  "[b2-a3]": 197,
+  "[b2-a4]": 198,
+  "[b2-b1]": 199,
+  "[b2-b2]": 200,
+  "[b2-b3]": 201,
+  "[b2-b4]": 202,
+  "[b2-c1]": 203,
+  "[b2-c2]": 204,
+  "[b2-c3]": 205,
+  "[b2-c4]": 206,
+  "[b2-d1]": 207,
+  "[b2-d2]": 208,
+  "[b2-d3]": 209,
+  "[b2-d4]": 210,
+  "[b3-a1]": 211,
+  "[b3-a2]": 212,
+  "[b3-a3]": 213,
+  "[b3-a4]": 214,
+  "[b3-a5]": 215,
+  "[b3-b1]": 216,
+  "[b3-b2]": 217,
+  "[b3-b3]": 218,
+  "[b3-b4]": 219,
+  "[b3-b5]": 220,
+  "[b3-c1]": 221,
+  "[b3-c2]": 222,
+  "[b3-c3]": 223,
+  "[b3-c4]": 224,
+  "[b3-c5]": 225,
+  "[b3-d1]": 226,
+  "[b3-d2]": 227,
+  "[b3-d3]": 228,
+  "[b3-d4]": 229,
+  "[b3-d5]": 230,
+  "[b4-a2]": 231,
+  "[b4-a3]": 232,
+  "[b4-a4]": 233,
+  "[b4-a5]": 234,
+  "[b4-a6]": 235,
+  "[b4-b2]": 236,
+  "[b4-b3]": 237,
+  "[b4-b4]": 238,
+  "[b4-b5]": 239,
+  "[b4-b6]": 240,
+  "[b4-c2]": 241,
+  "[b4-c3]": 242,
+  "[b4-c4]": 243,
+  "[b4-c5]": 244,
+  "[b4-c6]": 245,
+  "[b4-d2]": 246,
+  "[b4-d3]": 247,
+  "[b4-d4]": 248,
+  "[b4-d5]": 249,
+  "[b4-d6]": 250,
+  "[b5-a3]": 251,
+  "[b5-a4]": 252,
+  "[b5-a5]": 253,
+  "[b5-a6]": 254,
+  "[b5-a7]": 255,
+  "[b5-b3]": 256,
+  "[b5-b4]": 257,
+  "[b5-b5]": 258,
+  "[b5-b6]": 259,
+  "[b5-b7]": 260,
+  "[b5-c3]": 261,
+  "[b5-c4]": 262,
+  "[b5-c5]": 263,
+  "[b5-c6]": 264,
+  "[b5-c7]": 265,
+  "[b5-d3]": 266,
+  "[b5-d4]": 267,
+  "[b5-d5]": 268,
+  "[b5-d6]": 269,
+  "[b5-d7]": 270,
+  "[b6-a4]": 271,
+  "[b6-a5]": 272,
+  "[b6-a6]": 273,
+  "[b6-a7]": 274,
+  "[b6-a8]": 275,
+  "[b6-b4]": 276,
+  "[b6-b5]": 277,
+  "[b6-b6]": 278,
+  "[b6-b7]": 279,
+  "[b6-b8]": 280,
+  "[b6-c4]": 281,
+  "[b6-c5]": 282,
+  "[b6-c6]": 283,
+  "[b6-c7]": 284,
+  "[b6-c8]": 285,
+  "[b6-d4]": 286,
+  "[b6-d5]": 287,
+  "[b6-d6]": 288,
+  "[b6-d7]": 289,
+  "[b6-d8]": 290,
+  "[b7-a5]": 291,
+  "[b7-a6]": 292,
+  "[b7-a7]": 293,
+  "[b7-a8]": 294,
+  "[b7-b5]": 295,
+  "[b7-b6]": 296,
+  "[b7-b7]": 297,
+  "[b7-b8]": 298,
+  "[b7-c5]": 299,
+  "[b7-c6]": 300,
+  "[b7-c7]": 301,
+  "[b7-c8]": 302,
+  "[b7-d5]": 303,
+  "[b7-d6]": 304,
+  "[b7-d7]": 305,
+  "[b7-d8]": 306,
+  "[b8-a6]": 307,
+  "[b8-a7]": 308,
+  "[b8-a8]": 309,
+  "[b8-b6]": 310,
+  "[b8-b7]": 311,
+  "[b8-b8]": 312,
+  "[b8-c6]": 313,
+  "[b8-c7]": 314,
+  "[b8-c8]": 315,
+  "[b8-d6]": 316,
+  "[b8-d7]": 317,
+  "[b8-d8]": 318,
+  "[c1-a1]": 319,
+  "[c1-a2]": 320,
+  "[c1-a3]": 321,
+  "[c1-b1]": 322,
+  "[c1-b2]": 323,
+  "[c1-b3]": 324,
+  "[c1-c1]": 325,
+  "[c1-c2]": 326,
+  "[c1-c3]": 327,
+  "[c1-d1]": 328,
+  "[c1-d2]": 329,
+  "[c1-d3]": 330,
+  "[c1-e1]": 331,
+  "[c1-e2]": 332,
+  "[c1-e3]": 333,
+  "[c2-a1]": 334,
+  "[c2-a2]": 335,
+  "[c2-a3]": 336,
+  "[c2-a4]": 337,
+  "[c2-b1]": 338,
+  "[c2-b2]": 339,
+  "[c2-b3]": 340,
+  "[c2-b4]": 341,
+  "[c2-c1]": 342,
+  "[c2-c2]": 343,
+  "[c2-c3]": 344,
+  "[c2-c4]": 345,
+  "[c2-d1]": 346,
+  "[c2-d2]": 347,
+  "[c2-d3]": 348,
+  "[c2-d4]": 349,
+  "[c2-e1]": 350,
+  "[c2-e2]": 351,
+  "[c2-e3]": 352,
+  "[c2-e4]": 353,
+  "[c3-a1]": 354,
+  "[c3-a2]": 355,
+  "[c3-a3]": 356,
+  "[c3-a4]": 357,
+  "[c3-a5]": 358,
+  "[c3-b1]": 359,
+  "[c3-b2]": 360,
+  "[c3-b3]": 361,
+  "[c3-b4]": 362,
+  "[c3-b5]": 363,
+  "[c3-c1]": 364,
+  "[c3-c2]": 365,
+  "[c3-c3]": 366,
+  "[c3-c4]": 367,
+  "[c3-c5]": 368,
+  "[c3-d1]": 369,
+  "[c3-d2]": 370,
+  "[c3-d3]": 371,
+  "[c3-d4]": 372,
+  "[c3-d5]": 373,
+  "[c3-e1]": 374,
+  "[c3-e2]": 375,
+  "[c3-e3]": 376,
+  "[c3-e4]": 377,
+  "[c3-e5]": 378,
+  "[c4-a2]": 379,
+  "[c4-a3]": 380,
+  "[c4-a4]": 381,
+  "[c4-a5]": 382,
+  "[c4-a6]": 383,
+  "[c4-b2]": 384,
+  "[c4-b3]": 385,
+  "[c4-b4]": 386,
+  "[c4-b5]": 387,
+  "[c4-b6]": 388,
+  "[c4-c2]": 389,
+  "[c4-c3]": 390,
+  "[c4-c4]": 391,
+  "[c4-c5]": 392,
+  "[c4-c6]": 393,
+  "[c4-d2]": 394,
+  "[c4-d3]": 395,
+  "[c4-d4]": 396,
+  "[c4-d5]": 397,
+  "[c4-d6]": 398,
+  "[c4-e2]": 399,
+  "[c4-e3]": 400,
+  "[c4-e4]": 401,
+  "[c4-e5]": 402,
+  "[c4-e6]": 403,
+  "[c5-a3]": 404,
+  "[c5-a4]": 405,
+  "[c5-a5]": 406,
+  "[c5-a6]": 407,
+  "[c5-a7]": 408,
+  "[c5-b3]": 409,
+  "[c5-b4]": 410,
+  "[c5-b5]": 411,
+  "[c5-b6]": 412,
+  "[c5-b7]": 413,
+  "[c5-c3]": 414,
+  "[c5-c4]": 415,
+  "[c5-c5]": 416,
+  "[c5-c6]": 417,
+  "[c5-c7]": 418,
+  "[c5-d3]": 419,
+  "[c5-d4]": 420,
+  "[c5-d5]": 421,
+  "[c5-d6]": 422,
+  "[c5-d7]": 423,
+  "[c5-e3]": 424,
+  "[c5-e4]": 425,
+  "[c5-e5]": 426,
+  "[c5-e6]": 427,
+  "[c5-e7]": 428,
+  "[c6-a4]": 429,
+  "[c6-a5]": 430,
+  "[c6-a6]": 431,
+  "[c6-a7]": 432,
+  "[c6-a8]": 433,
+  "[c6-b4]": 434,
+  "[c6-b5]": 435,
+  "[c6-b6]": 436,
+  "[c6-b7]": 437,
+  "[c6-b8]": 438,
+  "[c6-c4]": 439,
+  "[c6-c5]": 440,
+  "[c6-c6]": 441,
+  "[c6-c7]": 442,
+  "[c6-c8]": 443,
+  "[c6-d4]": 444,
+  "[c6-d5]": 445,
+  "[c6-d6]": 446,
+  "[c6-d7]": 447,
+  "[c6-d8]": 448,
+  "[c6-e4]": 449,
+  "[c6-e5]": 450,
+  "[c6-e6]": 451,
+  "[c6-e7]": 452,
+  "[c6-e8]": 453,
+  "[c7-a5]": 454,
+  "[c7-a6]": 455,
+  "[c7-a7]": 456,
+  "[c7-a8]": 457,
+  "[c7-b5]": 458,
+  "[c7-b6]": 459,
+  "[c7-b7]": 460,
+  "[c7-b8]": 461,
+  "[c7-c5]": 462,
+  "[c7-c6]": 463,
+  "[c7-c7]": 464,
+  "[c7-c8]": 465,
+  "[c7-d5]": 466,
+  "[c7-d6]": 467,
+  "[c7-d7]": 468,
+  "[c7-d8]": 469,
+  "[c7-e5]": 470,
+  "[c7-e6]": 471,
+  "[c7-e7]": 472,
+  "[c7-e8]": 473,
+  "[c8-a6]": 474,
+  "[c8-a7]": 475,
+  "[c8-a8]": 476,
+  "[c8-b6]": 477,
+  "[c8-b7]": 478,
+  "[c8-b8]": 479,
+  "[c8-c6]": 480,
+  "[c8-c7]": 481,
+  "[c8-c8]": 482,
+  "[c8-d6]": 483,
+  "[c8-d7]": 484,
+  "[c8-d8]": 485,
+  "[c8-e6]": 486,
+  "[c8-e7]": 487,
+  "[c8-e8]": 488,
+  "[d1-b1]": 489,
+  "[d1-b2]": 490,
+  "[d1-b3]": 491,
+  "[d1-c1]": 492,
+  "[d1-c2]": 493,
+  "[d1-c3]": 494,
+  "[d1-d1]": 495,
+  "[d1-d2]": 496,
+  "[d1-d3]": 497,
+  "[d1-e1]": 498,
+  "[d1-e2]": 499,
+  "[d1-e3]": 500,
+  "[d1-f1]": 501,
+  "[d1-f2]": 502,
+  "[d1-f3]": 503,
+  "[d2-b1]": 504,
+  "[d2-b2]": 505,
+  "[d2-b3]": 506,
+  "[d2-b4]": 507,
+  "[d2-c1]": 508,
+  "[d2-c2]": 509,
+  "[d2-c3]": 510,
+  "[d2-c4]": 511,
+  "[d2-d1]": 512,
+  "[d2-d2]": 513,
+  "[d2-d3]": 514,
+  "[d2-d4]": 515,
+  "[d2-e1]": 516,
+  "[d2-e2]": 517,
+  "[d2-e3]": 518,
+  "[d2-e4]": 519,
+  "[d2-f1]": 520,
+  "[d2-f2]": 521,
+  "[d2-f3]": 522,
+  "[d2-f4]": 523,
+  "[d3-b1]": 524,
+  "[d3-b2]": 525,
+  "[d3-b3]": 526,
+  "[d3-b4]": 527,
+  "[d3-b5]": 528,
+  "[d3-c1]": 529,
+  "[d3-c2]": 530,
+  "[d3-c3]": 531,
+  "[d3-c4]": 532,
+  "[d3-c5]": 533,
+  "[d3-d1]": 534,
+  "[d3-d2]": 535,
+  "[d3-d3]": 536,
+  "[d3-d4]": 537,
+  "[d3-d5]": 538,
+  "[d3-e1]": 539,
+  "[d3-e2]": 540,
+  "[d3-e3]": 541,
+  "[d3-e4]": 542,
+  "[d3-e5]": 543,
+  "[d3-f1]": 544,
+  "[d3-f2]": 545,
+  "[d3-f3]": 546,
+  "[d3-f4]": 547,
+  "[d3-f5]": 548,
+  "[d4-b2]": 549,
+  "[d4-b3]": 550,
+  "[d4-b4]": 551,
+  "[d4-b5]": 552,
+  "[d4-b6]": 553,
+  "[d4-c2]": 554,
+  "[d4-c3]": 555,
+  "[d4-c4]": 556,
+  "[d4-c5]": 557,
+  "[d4-c6]": 558,
+  "[d4-d2]": 559,
+  "[d4-d3]": 560,
+  "[d4-d4]": 561,
+  "[d4-d5]": 562,
+  "[d4-d6]": 563,
+  "[d4-e2]": 564,
+  "[d4-e3]": 565,
+  "[d4-e4]": 566,
+  "[d4-e5]": 567,
+  "[d4-e6]": 568,
+  "[d4-f2]": 569,
+  "[d4-f3]": 570,
+  "[d4-f4]": 571,
+  "[d4-f5]": 572,
+  "[d4-f6]": 573,
+  "[d5-b3]": 574,
+  "[d5-b4]": 575,
+  "[d5-b5]": 576,
+  "[d5-b6]": 577,
+  "[d5-b7]": 578,
+  "[d5-c3]": 579,
+  "[d5-c4]": 580,
+  "[d5-c5]": 581,
+  "[d5-c6]": 582,
+  "[d5-c7]": 583,
+  "[d5-d3]": 584,
+  "[d5-d4]": 585,
+  "[d5-d5]": 586,
+  "[d5-d6]": 587,
+  "[d5-d7]": 588,
+  "[d5-e3]": 589,
+  "[d5-e4]": 590,
+  "[d5-e5]": 591,
+  "[d5-e6]": 592,
+  "[d5-e7]": 593,
+  "[d5-f3]": 594,
+  "[d5-f4]": 595,
+  "[d5-f5]": 596,
+  "[d5-f6]": 597,
+  "[d5-f7]": 598,
+  "[d6-b4]": 599,
+  "[d6-b5]": 600,
+  "[d6-b6]": 601,
+  "[d6-b7]": 602,
+  "[d6-b8]": 603,
+  "[d6-c4]": 604,
+  "[d6-c5]": 605,
+  "[d6-c6]": 606,
+  "[d6-c7]": 607,
+  "[d6-c8]": 608,
+  "[d6-d4]": 609,
+  "[d6-d5]": 610,
+  "[d6-d6]": 611,
+  "[d6-d7]": 612,
+  "[d6-d8]": 613,
+  "[d6-e4]": 614,
+  "[d6-e5]": 615,
+  "[d6-e6]": 616,
+  "[d6-e7]": 617,
+  "[d6-e8]": 618,
+  "[d6-f4]": 619,
+  "[d6-f5]": 620,
+  "[d6-f6]": 621,
+  "[d6-f7]": 622,
+  "[d6-f8]": 623,
+  "[d7-b5]": 624,
+  "[d7-b6]": 625,
+  "[d7-b7]": 626,
+  "[d7-b8]": 627,
+  "[d7-c5]": 628,
+  "[d7-c6]": 629,
+  "[d7-c7]": 630,
+  "[d7-c8]": 631,
+  "[d7-d5]": 632,
+  "[d7-d6]": 633,
+  "[d7-d7]": 634,
+  "[d7-d8]": 635,
+  "[d7-e5]": 636,
+  "[d7-e6]": 637,
+  "[d7-e7]": 638,
+  "[d7-e8]": 639,
+  "[d7-f5]": 640,
+  "[d7-f6]": 641,
+  "[d7-f7]": 642,
+  "[d7-f8]": 643,
+  "[d8-b6]": 644,
+  "[d8-b7]": 645,
+  "[d8-b8]": 646,
+  "[d8-c6]": 647,
+  "[d8-c7]": 648,
+  "[d8-c8]": 649,
+  "[d8-d6]": 650,
+  "[d8-d7]": 651,
+  "[d8-d8]": 652,
+  "[d8-e6]": 653,
+  "[d8-e7]": 654,
+  "[d8-e8]": 655,
+  "[d8-f6]": 656,
+  "[d8-f7]": 657,
+  "[d8-f8]": 658,
+  "[e1-c1]": 659,
+  "[e1-c2]": 660,
+  "[e1-c3]": 661,
+  "[e1-d1]": 662,
+  "[e1-d2]": 663,
+  "[e1-d3]": 664,
+  "[e1-e1]": 665,
+  "[e1-e2]": 666,
+  "[e1-e3]": 667,
+  "[e1-f1]": 668,
+  "[e1-f2]": 669,
+  "[e1-f3]": 670,
+  "[e1-g1]": 671,
+  "[e1-g2]": 672,
+  "[e1-g3]": 673,
+  "[e2-c1]": 674,
+  "[e2-c2]": 675,
+  "[e2-c3]": 676,
+  "[e2-c4]": 677,
+  "[e2-d1]": 678,
+  "[e2-d2]": 679,
+  "[e2-d3]": 680,
+  "[e2-d4]": 681,
+  "[e2-e1]": 682,
+  "[e2-e2]": 683,
+  "[e2-e3]": 684,
+  "[e2-e4]": 685,
+  "[e2-f1]": 686,
+  "[e2-f2]": 687,
+  "[e2-f3]": 688,
+  "[e2-f4]": 689,
+  "[e2-g1]": 690,
+  "[e2-g2]": 691,
+  "[e2-g3]": 692,
+  "[e2-g4]": 693,
+  "[e3-c1]": 694,
+  "[e3-c2]": 695,
+  "[e3-c3]": 696,
+  "[e3-c4]": 697,
+  "[e3-c5]": 698,
+  "[e3-d1]": 699,
+  "[e3-d2]": 700,
+  "[e3-d3]": 701,
+  "[e3-d4]": 702,
+  "[e3-d5]": 703,
+  "[e3-e1]": 704,
+  "[e3-e2]": 705,
+  "[e3-e3]": 706,
+  "[e3-e4]": 707,
+  "[e3-e5]": 708,
+  "[e3-f1]": 709,
+  "[e3-f2]": 710,
+  "[e3-f3]": 711,
+  "[e3-f4]": 712,
+  "[e3-f5]": 713,
+  "[e3-g1]": 714,
+  "[e3-g2]": 715,
+  "[e3-g3]": 716,
+  "[e3-g4]": 717,
+  "[e3-g5]": 718,
+  "[e4-c2]": 719,
+  "[e4-c3]": 720,
+  "[e4-c4]": 721,
+  "[e4-c5]": 722,
+  "[e4-c6]": 723,
+  "[e4-d2]": 724,
+  "[e4-d3]": 725,
+  "[e4-d4]": 726,
+  "[e4-d5]": 727,
+  "[e4-d6]": 728,
+  "[e4-e2]": 729,
+  "[e4-e3]": 730,
+  "[e4-e4]": 731,
+  "[e4-e5]": 732,
+  "[e4-e6]": 733,
+  "[e4-f2]": 734,
+  "[e4-f3]": 735,
+  "[e4-f4]": 736,
+  "[e4-f5]": 737,
+  "[e4-f6]": 738,
+  "[e4-g2]": 739,
+  "[e4-g3]": 740,
+  "[e4-g4]": 741,
+  "[e4-g5]": 742,
+  "[e4-g6]": 743,
+  "[e5-c3]": 744,
+  "[e5-c4]": 745,
+  "[e5-c5]": 746,
+  "[e5-c6]": 747,
+  "[e5-c7]": 748,
+  "[e5-d3]": 749,
+  "[e5-d4]": 750,
+  "[e5-d5]": 751,
+  "[e5-d6]": 752,
+  "[e5-d7]": 753,
+  "[e5-e3]": 754,
+  "[e5-e4]": 755,
+  "[e5-e5]": 756,
+  "[e5-e6]": 757,
+  "[e5-e7]": 758,
+  "[e5-f3]": 759,
+  "[e5-f4]": 760,
+  "[e5-f5]": 761,
+  "[e5-f6]": 762,
+  "[e5-f7]": 763,
+  "[e5-g3]": 764,
+  "[e5-g4]": 765,
+  "[e5-g5]": 766,
+  "[e5-g6]": 767,
+  "[e5-g7]": 768,
+  "[e6-c4]": 769,
+  "[e6-c5]": 770,
+  "[e6-c6]": 771,
+  "[e6-c7]": 772,
+  "[e6-c8]": 773,
+  "[e6-d4]": 774,
+  "[e6-d5]": 775,
+  "[e6-d6]": 776,
+  "[e6-d7]": 777,
+  "[e6-d8]": 778,
+  "[e6-e4]": 779,
+  "[e6-e5]": 780,
+  "[e6-e6]": 781,
+  "[e6-e7]": 782,
+  "[e6-e8]": 783,
+  "[e6-f4]": 784,
+  "[e6-f5]": 785,
+  "[e6-f6]": 786,
+  "[e6-f7]": 787,
+  "[e6-f8]": 788,
+  "[e6-g4]": 789,
+  "[e6-g5]": 790,
+  "[e6-g6]": 791,
+  "[e6-g7]": 792,
+  "[e6-g8]": 793,
+  "[e7-c5]": 794,
+  "[e7-c6]": 795,
+  "[e7-c7]": 796,
+  "[e7-c8]": 797,
+  "[e7-d5]": 798,
+  "[e7-d6]": 799,
+  "[e7-d7]": 800,
+  "[e7-d8]": 801,
+  "[e7-e5]": 802,
+  "[e7-e6]": 803,
+  "[e7-e7]": 804,
+  "[e7-e8]": 805,
+  "[e7-f5]": 806,
+  "[e7-f6]": 807,
+  "[e7-f7]": 808,
+  "[e7-f8]": 809,
+  "[e7-g5]": 810,
+  "[e7-g6]": 811,
+  "[e7-g7]": 812,
+  "[e7-g8]": 813,
+  "[e8-c6]": 814,
+  "[e8-c7]": 815,
+  "[e8-c8]": 816,
+  "[e8-d6]": 817,
+  "[e8-d7]": 818,
+  "[e8-d8]": 819,
+  "[e8-e6]": 820,
+  "[e8-e7]": 821,
+  "[e8-e8]": 822,
+  "[e8-f6]": 823,
+  "[e8-f7]": 824,
+  "[e8-f8]": 825,
+  "[e8-g6]": 826,
+  "[e8-g7]": 827,
+  "[e8-g8]": 828,
+  "[f1-d1]": 829,
+  "[f1-d2]": 830,
+  "[f1-d3]": 831,
+  "[f1-e1]": 832,
+  "[f1-e2]": 833,
+  "[f1-e3]": 834,
+  "[f1-f1]": 835,
+  "[f1-f2]": 836,
+  "[f1-f3]": 837,
+  "[f1-g1]": 838,
+  "[f1-g2]": 839,
+  "[f1-g3]": 840,
+  "[f1-h1]": 841,
+  "[f1-h2]": 842,
+  "[f1-h3]": 843,
+  "[f2-d1]": 844,
+  "[f2-d2]": 845,
+  "[f2-d3]": 846,
+  "[f2-d4]": 847,
+  "[f2-e1]": 848,
+  "[f2-e2]": 849,
+  "[f2-e3]": 850,
+  "[f2-e4]": 851,
+  "[f2-f1]": 852,
+  "[f2-f2]": 853,
+  "[f2-f3]": 854,
+  "[f2-f4]": 855,
+  "[f2-g1]": 856,
+  "[f2-g2]": 857,
+  "[f2-g3]": 858,
+  "[f2-g4]": 859,
+  "[f2-h1]": 860,
+  "[f2-h2]": 861,
+  "[f2-h3]": 862,
+  "[f2-h4]": 863,
+  "[f3-d1]": 864,
+  "[f3-d2]": 865,
+  "[f3-d3]": 866,
+  "[f3-d4]": 867,
+  "[f3-d5]": 868,
+  "[f3-e1]": 869,
+  "[f3-e2]": 870,
+  "[f3-e3]": 871,
+  "[f3-e4]": 872,
+  "[f3-e5]": 873,
+  "[f3-f1]": 874,
+  "[f3-f2]": 875,
+  "[f3-f3]": 876,
+  "[f3-f4]": 877,
+  "[f3-f5]": 878,
+  "[f3-g1]": 879,
+  "[f3-g2]": 880,
+  "[f3-g3]": 881,
+  "[f3-g4]": 882,
+  "[f3-g5]": 883,
+  "[f3-h1]": 884,
+  "[f3-h2]": 885,
+  "[f3-h3]": 886,
+  "[f3-h4]": 887,
+  "[f3-h5]": 888,
+  "[f4-d2]": 889,
+  "[f4-d3]": 890,
+  "[f4-d4]": 891,
+  "[f4-d5]": 892,
+  "[f4-d6]": 893,
+  "[f4-e2]": 894,
+  "[f4-e3]": 895,
+  "[f4-e4]": 896,
+  "[f4-e5]": 897,
+  "[f4-e6]": 898,
+  "[f4-f2]": 899,
+  "[f4-f3]": 900,
+  "[f4-f4]": 901,
+  "[f4-f5]": 902,
+  "[f4-f6]": 903,
+  "[f4-g2]": 904,
+  "[f4-g3]": 905,
+  "[f4-g4]": 906,
+  "[f4-g5]": 907,
+  "[f4-g6]": 908,
+  "[f4-h2]": 909,
+  "[f4-h3]": 910,
+  "[f4-h4]": 911,
+  "[f4-h5]": 912,
+  "[f4-h6]": 913,
+  "[f5-d3]": 914,
+  "[f5-d4]": 915,
+  "[f5-d5]": 916,
+  "[f5-d6]": 917,
+  "[f5-d7]": 918,
+  "[f5-e3]": 919,
+  "[f5-e4]": 920,
+  "[f5-e5]": 921,
+  "[f5-e6]": 922,
+  "[f5-e7]": 923,
+  "[f5-f3]": 924,
+  "[f5-f4]": 925,
+  "[f5-f5]": 926,
+  "[f5-f6]": 927,
+  "[f5-f7]": 928,
+  "[f5-g3]": 929,
+  "[f5-g4]": 930,
+  "[f5-g5]": 931,
+  "[f5-g6]": 932,
+  "[f5-g7]": 933,
+  "[f5-h3]": 934,
+  "[f5-h4]": 935,
+  "[f5-h5]": 936,
+  "[f5-h6]": 937,
+  "[f5-h7]": 938,
+  "[f6-d4]": 939,
+  "[f6-d5]": 940,
+  "[f6-d6]": 941,
+  "[f6-d7]": 942,
+  "[f6-d8]": 943,
+  "[f6-e4]": 944,
+  "[f6-e5]": 945,
+  "[f6-e6]": 946,
+  "[f6-e7]": 947,
+  "[f6-e8]": 948,
+  "[f6-f4]": 949,
+  "[f6-f5]": 950,
+  "[f6-f6]": 951,
+  "[f6-f7]": 952,
+  "[f6-f8]": 953,
+  "[f6-g4]": 954,
+  "[f6-g5]": 955,
+  "[f6-g6]": 956,
+  "[f6-g7]": 957,
+  "[f6-g8]": 958,
+  "[f6-h4]": 959,
+  "[f6-h5]": 960,
+  "[f6-h6]": 961,
+  "[f6-h7]": 962,
+  "[f6-h8]": 963,
+  "[f7-d5]": 964,
+  "[f7-d6]": 965,
+  "[f7-d7]": 966,
+  "[f7-d8]": 967,
+  "[f7-e5]": 968,
+  "[f7-e6]": 969,
+  "[f7-e7]": 970,
+  "[f7-e8]": 971,
+  "[f7-f5]": 972,
+  "[f7-f6]": 973,
+  "[f7-f7]": 974,
+  "[f7-f8]": 975,
+  "[f7-g5]": 976,
+  "[f7-g6]": 977,
+  "[f7-g7]": 978,
+  "[f7-g8]": 979,
+  "[f7-h5]": 980,
+  "[f7-h6]": 981,
+  "[f7-h7]": 982,
+  "[f7-h8]": 983,
+  "[f8-d6]": 984,
+  "[f8-d7]": 985,
+  "[f8-d8]": 986,
+  "[f8-e6]": 987,
+  "[f8-e7]": 988,
+  "[f8-e8]": 989,
+  "[f8-f6]": 990,
+  "[f8-f7]": 991,
+  "[f8-f8]": 992,
+  "[f8-g6]": 993,
+  "[f8-g7]": 994,
+  "[f8-g8]": 995,
+  "[f8-h6]": 996,
+  "[f8-h7]": 997,
+  "[f8-h8]": 998,
+  "[g1-e1]": 999,
+  "[g1-e2]": 1000,
+  "[g1-e3]": 1001,
+  "[g1-f1]": 1002,
+  "[g1-f2]": 1003,
+  "[g1-f3]": 1004,
+  "[g1-g1]": 1005,
+  "[g1-g2]": 1006,
+  "[g1-g3]": 1007,
+  "[g1-h1]": 1008,
+  "[g1-h2]": 1009,
+  "[g1-h3]": 1010,
+  "[g2-e1]": 1011,
+  "[g2-e2]": 1012,
+  "[g2-e3]": 1013,
+  "[g2-e4]": 1014,
+  "[g2-f1]": 1015,
+  "[g2-f2]": 1016,
+  "[g2-f3]": 1017,
+  "[g2-f4]": 1018,
+  "[g2-g1]": 1019,
+  "[g2-g2]": 1020,
+  "[g2-g3]": 1021,
+  "[g2-g4]": 1022,
+  "[g2-h1]": 1023,
+  "[g2-h2]": 1024,
+  "[g2-h3]": 1025,
+  "[g2-h4]": 1026,
+  "[g3-e1]": 1027,
+  "[g3-e2]": 1028,
+  "[g3-e3]": 1029,
+  "[g3-e4]": 1030,
+  "[g3-e5]": 1031,
+  "[g3-f1]": 1032,
+  "[g3-f2]": 1033,
+  "[g3-f3]": 1034,
+  "[g3-f4]": 1035,
+  "[g3-f5]": 1036,
+  "[g3-g1]": 1037,
+  "[g3-g2]": 1038,
+  "[g3-g3]": 1039,
+  "[g3-g4]": 1040,
+  "[g3-g5]": 1041,
+  "[g3-h1]": 1042,
+  "[g3-h2]": 1043,
+  "[g3-h3]": 1044,
+  "[g3-h4]": 1045,
+  "[g3-h5]": 1046,
+  "[g4-e2]": 1047,
+  "[g4-e3]": 1048,
+  "[g4-e4]": 1049,
+  "[g4-e5]": 1050,
+  "[g4-e6]": 1051,
+  "[g4-f2]": 1052,
+  "[g4-f3]": 1053,
+  "[g4-f4]": 1054,
+  "[g4-f5]": 1055,
+  "[g4-f6]": 1056,
+  "[g4-g2]": 1057,
+  "[g4-g3]": 1058,
+  "[g4-g4]": 1059,
+  "[g4-g5]": 1060,
+  "[g4-g6]": 1061,
+  "[g4-h2]": 1062,
+  "[g4-h3]": 1063,
+  "[g4-h4]": 1064,
+  "[g4-h5]": 1065,
+  "[g4-h6]": 1066,
+  "[g5-e3]": 1067,
+  "[g5-e4]": 1068,
+  "[g5-e5]": 1069,
+  "[g5-e6]": 1070,
+  "[g5-e7]": 1071,
+  "[g5-f3]": 1072,
+  "[g5-f4]": 1073,
+  "[g5-f5]": 1074,
+  "[g5-f6]": 1075,
+  "[g5-f7]": 1076,
+  "[g5-g3]": 1077,
+  "[g5-g4]": 1078,
+  "[g5-g5]": 1079,
+  "[g5-g6]": 1080,
+  "[g5-g7]": 1081,
+  "[g5-h3]": 1082,
+  "[g5-h4]": 1083,
+  "[g5-h5]": 1084,
+  "[g5-h6]": 1085,
+  "[g5-h7]": 1086,
+  "[g6-e4]": 1087,
+  "[g6-e5]": 1088,
+  "[g6-e6]": 1089,
+  "[g6-e7]": 1090,
+  "[g6-e8]": 1091,
+  "[g6-f4]": 1092,
+  "[g6-f5]": 1093,
+  "[g6-f6]": 1094,
+  "[g6-f7]": 1095,
+  "[g6-f8]": 1096,
+  "[g6-g4]": 1097,
+  "[g6-g5]": 1098,
+  "[g6-g6]": 1099,
+  "[g6-g7]": 1100,
+  "[g6-g8]": 1101,
+  "[g6-h4]": 1102,
+  "[g6-h5]": 1103,
+  "[g6-h6]": 1104,
+  "[g6-h7]": 1105,
+  "[g6-h8]": 1106,
+  "[g7-e5]": 1107,
+  "[g7-e6]": 1108,
+  "[g7-e7]": 1109,
+  "[g7-e8]": 1110,
+  "[g7-f5]": 1111,
+  "[g7-f6]": 1112,
+  "[g7-f7]": 1113,
+  "[g7-f8]": 1114,
+  "[g7-g5]": 1115,
+  "[g7-g6]": 1116,
+  "[g7-g7]": 1117,
+  "[g7-g8]": 1118,
+  "[g7-h5]": 1119,
+  "[g7-h6]": 1120,
+  "[g7-h7]": 1121,
+  "[g7-h8]": 1122,
+  "[g8-e6]": 1123,
+  "[g8-e7]": 1124,
+  "[g8-e8]": 1125,
+  "[g8-f6]": 1126,
+  "[g8-f7]": 1127,
+  "[g8-f8]": 1128,
+  "[g8-g6]": 1129,
+  "[g8-g7]": 1130,
+  "[g8-g8]": 1131,
+  "[g8-h6]": 1132,
+  "[g8-h7]": 1133,
+  "[g8-h8]": 1134,
+  "[h1-f1]": 1135,
+  "[h1-f2]": 1136,
+  "[h1-f3]": 1137,
+  "[h1-g1]": 1138,
+  "[h1-g2]": 1139,
+  "[h1-g3]": 1140,
+  "[h1-h1]": 1141,
+  "[h1-h2]": 1142,
+  "[h1-h3]": 1143,
+  "[h2-f1]": 1144,
+  "[h2-f2]": 1145,
+  "[h2-f3]": 1146,
+  "[h2-f4]": 1147,
+  "[h2-g1]": 1148,
+  "[h2-g2]": 1149,
+  "[h2-g3]": 1150,
+  "[h2-g4]": 1151,
+  "[h2-h1]": 1152,
+  "[h2-h2]": 1153,
+  "[h2-h3]": 1154,
+  "[h2-h4]": 1155,
+  "[h3-f1]": 1156,
+  "[h3-f2]": 1157,
+  "[h3-f3]": 1158,
+  "[h3-f4]": 1159,
+  "[h3-f5]": 1160,
+  "[h3-g1]": 1161,
+  "[h3-g2]": 1162,
+  "[h3-g3]": 1163,
+  "[h3-g4]": 1164,
+  "[h3-g5]": 1165,
+  "[h3-h1]": 1166,
+  "[h3-h2]": 1167,
+  "[h3-h3]": 1168,
+  "[h3-h4]": 1169,
+  "[h3-h5]": 1170,
+  "[h4-f2]": 1171,
+  "[h4-f3]": 1172,
+  "[h4-f4]": 1173,
+  "[h4-f5]": 1174,
+  "[h4-f6]": 1175,
+  "[h4-g2]": 1176,
+  "[h4-g3]": 1177,
+  "[h4-g4]": 1178,
+  "[h4-g5]": 1179,
+  "[h4-g6]": 1180
+}