Delete tokenizer.py

tokenizer.py

@@ -1,121 +0,0 @@
-from __future__ import annotations
-import json
-import os
-from typing import Dict, List, Optional
-from transformers import PreTrainedTokenizer
-import torch
-
-class ChessTokenizer(PreTrainedTokenizer):
-    """
-    vocab size: 149 (4 special + 12 pieces + 64 from_sq + 64 to_sq + 5 suffix)
-    """
-
-    model_input_names = ["input_ids", "attention_mask"]
-    vocab_files_names = {"vocab_file": "vocab.json"}
-
-    PAD_TOKEN = "[PAD]"
-    BOS_TOKEN = "[BOS]"
-    EOS_TOKEN = "[EOS]"
-    UNK_TOKEN = "[UNK]"
-
-    def __init__(self, vocab_file: Optional[str] = None, vocab: Optional[Dict[str, int]] = None, **kwargs):
-        special_tokens = [self.PAD_TOKEN, self.BOS_TOKEN, self.EOS_TOKEN, self.UNK_TOKEN]
-        
-        self.colors_pieces = [f'{c}{p}' for c in ['W','B'] for p in ['P','N','B','R','Q','K']]
-        self.squares = [f'{f}{r}' for r in '12345678' for f in 'abcdefgh']
-        self.suffixes = ["(x)", "(+)", "(+*)", "(o)", "(O)"]
-
-        if vocab is not None:
-            self._vocab = vocab
-        elif vocab_file is not None and os.path.exists(vocab_file):
-            with open(vocab_file, "r", encoding="utf-8") as f:
-                self._vocab = json.load(f)
-        else:
-            self._vocab = {t: i for i, t in enumerate(special_tokens)}
-            for cp in self.colors_pieces:
-                self._vocab[cp] = len(self._vocab)
-            for sq in self.squares:
-                self._vocab[f"{sq}_f"] = len(self._vocab)
-            for sq in self.squares:
-                self._vocab[f"{sq}_t"] = len(self._vocab)
-            for suf in self.suffixes:
-                self._vocab[suf] = len(self._vocab)
-
-        self._ids_to_tokens = {v: k for k, v in self._vocab.items()}
-
-        super().__init__(
-            pad_token=self.PAD_TOKEN,
-            bos_token=self.BOS_TOKEN,
-            eos_token=self.EOS_TOKEN,
-            unk_token=self.UNK_TOKEN,
-            **kwargs,
-        )
-
-    @property
-    def vocab_size(self) -> int:
-        return len(self._vocab)
-
-    def get_vocab(self) -> Dict[str, int]:
-        return dict(self._vocab)
-
-    def _tokenize(self, text: str) -> List[str]:
-        """Piece(2) + From(2) + To(2) + Suffix(?)"""
-        tokens = []
-        moves = text.strip().split()
-        
-        for move in moves:
-            if move in [self.PAD_TOKEN, self.BOS_TOKEN, self.EOS_TOKEN, self.UNK_TOKEN]:
-                tokens.append(move)
-                continue
-            
-            if len(move) >= 6:
-                tokens.append(move[:2])          # Piece (e.g., WP)
-                tokens.append(f"{move[2:4]}_f")  # From (e.g., e2_f)
-                tokens.append(f"{move[4:6]}_t")  # To (e.g., e4_t)
-                
-                if len(move) > 6:
-                    suffix = move[6:]
-                    if suffix in self.suffixes:
-                        tokens.append(suffix)
-        return tokens
-
-    def _convert_token_to_id(self, token: str) -> int:
-        return self._vocab.get(token, self._vocab.get(self.UNK_TOKEN))
-    def _convert_id_to_token(self, index: int) -> str:
-        token = self._ids_to_tokens.get(index, self.UNK_TOKEN)
-        if token in [self.PAD_TOKEN, self.BOS_TOKEN, self.EOS_TOKEN, self.UNK_TOKEN]:
-            return ""
-        if token in self.suffixes:
-            return token
-        return token.replace("_f", "").replace("_t", "")
-
-    def convert_tokens_to_string(self, tokens: List[str]) -> str:
-        return "".join([t for t in tokens if t])
-
-    def decode(self, token_ids, skip_special_tokens=True, **kwargs) -> str:
-        if hasattr(token_ids, "tolist"):
-            ids = token_ids.tolist()
-        elif isinstance(token_ids, (int, torch.LongTensor, torch.IntTensor)):
-            ids = [int(token_ids)] if isinstance(token_ids, int) else token_ids.tolist()
-        else:
-            ids = token_ids
-
-        tokens = [self._convert_id_to_token(i) for i in ids]
-        return self.convert_tokens_to_string(tokens)
-
-    def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> tuple:
-        if not os.path.isdir(save_directory):
-            os.makedirs(save_directory, exist_ok=True)
-        vocab_file = os.path.join(save_directory, (filename_prefix + "-" if filename_prefix else "") + "vocab.json")
-        with open(vocab_file, "w", encoding="utf-8") as f:
-            json.dump(self._vocab, f, ensure_ascii=False, indent=2)
-        return (vocab_file,)
-
-    @classmethod
-    def from_pretrained(cls, pretrained_model_name_or_path, **kwargs) -> "ChessTokenizer":
-        vocab_file = os.path.join(pretrained_model_name_or_path, "vocab.json")
-        if not os.path.exists(vocab_file):
-            return cls()
-        with open(vocab_file, "r", encoding="utf-8") as f:
-            vocab = json.load(f)
-        return cls(vocab=vocab, **kwargs)