| from transformers import PreTrainedTokenizerFast | |
| class ChessTokenizer(PreTrainedTokenizerFast): | |
| def __init__(self, tokenizer_file=None, **kwargs): | |
| super().__init__(tokenizer_file=tokenizer_file, **kwargs) | |
| self.pad_token = "[PAD]" | |
| self.bos_token = "[BOS]" | |
| self.eos_token = "[EOS]" | |
| self.unk_token = "[UNK]" | |