tokenizer.py

from transformers import PreTrainedTokenizerFast

class ChessTokenizer(PreTrainedTokenizerFast):
    def __init__(self, tokenizer_file=None, **kwargs):
        super().__init__(tokenizer_file=tokenizer_file, **kwargs)
        self.pad_token = "[PAD]"
        self.bos_token = "[BOS]"
        self.eos_token = "[EOS]"
        self.unk_token = "[UNK]"