custom-tokenizer.json

{
  "version": "1.0",
  "truncation": null,
  "padding": null,
  "added_tokens": [],
  "normalizer": null,
  "pre_tokenizer": {
    "type": "Punctuation",
    "behavior": "Isolated"
  },
  "post_processor": null,
  "decoder": null,
  "model": {
    "type": "BPE",
    "dropout": null,
    "unk_token": null,
    "continuing_subword_prefix": null,
    "end_of_word_suffix": null,
    "fuse_unk": false,
    "byte_fallback": false,
    "ignore_merges": false,
    "vocab": {
      "\n": 0,
      " ": 1,
      "!": 2,
      ".": 3,
      "B": 4,
      "I": 5,
      "T": 6,
      "a": 7,
      "d": 8,
      "e": 9,
      "f": 10,
      "g": 11,
      "i": 12,
      "k": 13,
      "l": 14,
      "m": 15,
      "n": 16,
      "o": 17,
      "r": 18,
      "s": 19,
      "t": 20,
      "u": 21,
      "v": 22,
      "w": 23,
      "y": 24,
      "z": 25,
      " a": 26,
      "er": 27,
      "ers": 28,
      " l": 29,
      " o": 30,
      " t": 31,
      " y": 32,
      "Bu": 33,
      "I l": 34,
      "To": 35,
      "an": 36,
      "az": 37,
      "d y": 38,
      "en": 39,
      "e a": 40,
      "e t": 41,
      "fo": 42,
      "il": 43,
      "in": 44,
      "iz": 45,
      "ken": 46,
      "mers": 47,
      "maz": 48,
      "ou": 49,
      "ov": 50,
      "r o": 51,
      "ran": 52,
      "re a": 53,
      "rmers": 54,
      "sfo": 55,
      "wn": 56,
      " are a": 57,
      "ers are a": 58,
      "Buil": 59,
      "I lov": 60,
      "Token": 61,
      "d you": 62,
      "e tran": 63,
      "ing": 64,
      "izers are a": 65,
      "mazing": 66,
      "r own": 67,
      "sformers": 68,
      "Build you": 69,
      "I love tran": 70,
      "Tokenizers are a": 71,
      "Build your own": 72,
      "I love transformers": 73,
      "Tokenizers are amazing": 74
    },
    "merges": [
      [
        " ",
        "a"
      ],
      [
        "e",
        "r"
      ],
      [
        "er",
        "s"
      ],
      [
        " ",
        "l"
      ],
      [
        " ",
        "o"
      ],
      [
        " ",
        "t"
      ],
      [
        " ",
        "y"
      ],
      [
        "B",
        "u"
      ],
      [
        "I",
        " l"
      ],
      [
        "T",
        "o"
      ],
      [
        "a",
        "n"
      ],
      [
        "a",
        "z"
      ],
      [
        "d",
        " y"
      ],
      [
        "e",
        "n"
      ],
      [
        "e",
        " a"
      ],
      [
        "e",
        " t"
      ],
      [
        "f",
        "o"
      ],
      [
        "i",
        "l"
      ],
      [
        "i",
        "n"
      ],
      [
        "i",
        "z"
      ],
      [
        "k",
        "en"
      ],
      [
        "m",
        "ers"
      ],
      [
        "m",
        "az"
      ],
      [
        "o",
        "u"
      ],
      [
        "o",
        "v"
      ],
      [
        "r",
        " o"
      ],
      [
        "r",
        "an"
      ],
      [
        "r",
        "e a"
      ],
      [
        "r",
        "mers"
      ],
      [
        "s",
        "fo"
      ],
      [
        "w",
        "n"
      ],
      [
        " a",
        "re a"
      ],
      [
        "ers",
        " are a"
      ],
      [
        "Bu",
        "il"
      ],
      [
        "I l",
        "ov"
      ],
      [
        "To",
        "ken"
      ],
      [
        "d y",
        "ou"
      ],
      [
        "e t",
        "ran"
      ],
      [
        "in",
        "g"
      ],
      [
        "iz",
        "ers are a"
      ],
      [
        "maz",
        "ing"
      ],
      [
        "r o",
        "wn"
      ],
      [
        "sfo",
        "rmers"
      ],
      [
        "Buil",
        "d you"
      ],
      [
        "I lov",
        "e tran"
      ],
      [
        "Token",
        "izers are a"
      ],
      [
        "Build you",
        "r own"
      ],
      [
        "I love tran",
        "sformers"
      ],
      [
        "Tokenizers are a",
        "mazing"
      ]
    ]
  }
}