tokenizer_config.json

{
  "vocab_target": 6000,
  "max_merge_steps": 200000,
  "special_tokens": [
    "<PAD>",
    "<UNK>",
    "<BOS>",
    "<EOS>"
  ],
  "placeholders": {
    "URL": "<URL>",
    "USER": "<USER>"
  },
  "protect_regex": {
    "url": "https?://\\S+",
    "user": "@[A-Za-z0-9_]+",
    "time": "\\b\\d{1,2}:\\d{2}\\b",
    "number": "\\b\\d+(?:\\.\\d+)?\\b",
    "emote": "(?:(?:[A-Za-z]+){2,}[A-Z]{2,}|[A-Za-z]+(?:LOL|KEKW|LUL|OMEGA)\\b)"
  },
  "version": 1
}