Delete tokenizer.py

tokenizer.py

@@ -1,121 +0,0 @@
-from __future__ import annotations
-import json
-import os
-from typing import Dict, List, Optional
-from transformers import PreTrainedTokenizer
-
-class ChessTokenizer(PreTrainedTokenizer):
-    """
-    符合评估脚本要求的 Chess Tokenizer。
-    1. 词表大小为 144 (4 special + 12 pieces + 64 from_sq + 64 to_sq)。
-    2. Decode 结果为紧凑格式（如 "WPe2e4"），确保 evaluate.py 的切片 [2:4] 和 [4:6] 正确。
-    3. 区分起始格和目标格语义。
-    """
-
-    model_input_names = ["input_ids", "attention_mask"]
-    vocab_files_names = {"vocab_file": "vocab.json"}
-
-    PAD_TOKEN = "[PAD]"
-    BOS_TOKEN = "[BOS]"
-    EOS_TOKEN = "[EOS]"
-    UNK_TOKEN = "[UNK]"
-
-    def __init__(self, vocab_file: Optional[str] = None, vocab: Optional[Dict[str, int]] = None, **kwargs):
-        special_tokens = [self.PAD_TOKEN, self.BOS_TOKEN, self.EOS_TOKEN, self.UNK_TOKEN]
-        
-        # 必须使用大写，以匹配 evaluate.py 生成的棋谱
-        self.colors_pieces = [f'{c}{p}' for c in ['W','B'] for p in ['P','N','B','R','Q','K']]  # 12个
-        self.squares = [f'{f}{r}' for r in '12345678' for f in 'abcdefgh']  # 64个
-
-        if vocab is not None:
-            self._vocab = vocab
-        elif vocab_file is not None and os.path.exists(vocab_file):
-            with open(vocab_file, "r", encoding="utf-8") as f:
-                self._vocab = json.load(f)
-        else:
-            # 构建 144 大小的词表
-            self._vocab = {t: i for i, t in enumerate(special_tokens)} # 0-3
-            
-            # 4-15: Piece tokens
-            for cp in self.colors_pieces:
-                self._vocab[cp] = len(self._vocab)
-            
-            # 16-79: From Square tokens (内部带后缀防止重名)
-            for sq in self.squares:
-                self._vocab[f"{sq}_f"] = len(self._vocab)
-            
-            # 80-143: To Square tokens
-            for sq in self.squares:
-                self._vocab[f"{sq}_t"] = len(self._vocab)
-
-        self._ids_to_tokens = {v: k for k, v in self._vocab.items()}
-
-        super().__init__(
-            pad_token=self.PAD_TOKEN,
-            bos_token=self.BOS_TOKEN,
-            eos_token=self.EOS_TOKEN,
-            unk_token=self.UNK_TOKEN,
-            **kwargs,
-        )
-
-    @property
-    def vocab_size(self) -> int:
-        return len(self._vocab)
-
-    def get_vocab(self) -> Dict[str, int]:
-        return dict(self._vocab)
-
-    def _tokenize(self, text: str) -> List[str]:
-        """将 WPe2e4 拆分为三个 token"""
-        tokens = []
-        # 处理可能的空格分隔（如历史棋谱）
-        moves = text.strip().split()
-        for move in moves:
-            # 过滤特殊 token 字符串
-            if move in [self.PAD_TOKEN, self.BOS_TOKEN, self.EOS_TOKEN, self.UNK_TOKEN]:
-                tokens.append(move)
-                continue
-            
-            if len(move) >= 6:
-                cp = move[:2]         # 例如 "WP"
-                from_sq = move[2:4] + "_f" # 例如 "e2_f"
-                to_sq = move[4:6] + "_t"   # 例如 "e4_t"
-                tokens.extend([cp, from_sq, to_sq])
-        return tokens
-
-    def _convert_token_to_id(self, token: str) -> int:
-        return self._vocab.get(token, self._vocab[self.UNK_TOKEN])
-
-    def _convert_id_to_token(self, index: int) -> str:
-        token = self._ids_to_tokens.get(index, self.UNK_TOKEN)
-        # 关键：在 decode 时去掉内部后缀，还原为 "e2", "e4"
-        return token.replace("_f", "").replace("_t", "")
-
-    def convert_tokens_to_string(self, tokens: List[str]) -> str:
-        """
-        将 token 列表合并。
-        evaluate.py 要求输出如 "WPe2e4"，因此这里不加空格。
-        """
-        # 过滤特殊 token，只保留棋步内容
-        clean_tokens = [t for t in tokens if t not in self.all_special_tokens]
-        return "".join(clean_tokens)
-
-    def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> tuple:
-        if not os.path.isdir(save_directory):
-            os.makedirs(save_directory, exist_ok=True)
-        vocab_file = os.path.join(
-            save_directory,
-            (filename_prefix + "-" if filename_prefix else "") + "vocab.json"
-        )
-        with open(vocab_file, "w", encoding="utf-8") as f:
-            json.dump(self._vocab, f, ensure_ascii=False, indent=2)
-        return (vocab_file,)
-
-    @classmethod
-    def from_pretrained(cls, pretrained_model_name_or_path, **kwargs) -> "ChessTokenizer":
-        vocab_file = os.path.join(pretrained_model_name_or_path, "vocab.json")
-        if not os.path.exists(vocab_file):
-            return cls() # 如果没有文件则初始化默认的
-        with open(vocab_file, "r", encoding="utf-8") as f:
-            vocab = json.load(f)
-        return cls(vocab=vocab, **kwargs)