tokenizer_config.json

{
  "tokenizer_class": "LogicTokenizerHF",
  "auto_map": {
    "AutoTokenizer": [
      "tokenization_nanogentzen.LogicTokenizerHF",
      "tokenization_nanogentzen.LogicTokenizerHF"
    ]
  },
  "pad_token": "<PAD>",
  "unk_token": "<UNK>",
  "cls_token": "<CLS>",
  "sep_token": "<SEP>",
  "eos_token": "<EOS>",
  "model_max_length": 256
}