tokenizer.json

{
  "version": "1.0",
  "truncation": null,
  "padding": null,
  "added_tokens": [
    {
      "id": 63,
      "content": "<unk>",
      "single_word": false,
      "lstrip": false,
      "rstrip": false,
      "normalized": false,
      "special": true
    },
    {
      "id": 64,
      "content": "<sep>",
      "single_word": false,
      "lstrip": false,
      "rstrip": false,
      "normalized": false,
      "special": true
    },
    {
      "id": 65,
      "content": "<pad>",
      "single_word": false,
      "lstrip": false,
      "rstrip": false,
      "normalized": false,
      "special": true
    },
    {
      "id": 66,
      "content": "<cls>",
      "single_word": false,
      "lstrip": false,
      "rstrip": false,
      "normalized": false,
      "special": true
    },
    {
      "id": 67,
      "content": "<mask>",
      "single_word": false,
      "lstrip": false,
      "rstrip": false,
      "normalized": false,
      "special": true
    }
  ],
  "normalizer": null,
  "pre_tokenizer": {
    "type": "Whitespace"
  },
  "post_processor": {
    "type": "TemplateProcessing",
    "single": [
      {
        "Sequence": {
          "id": "A",
          "type_id": 0
        }
      }
    ],
    "pair": [
      {
        "Sequence": {
          "id": "A",
          "type_id": 0
        }
      },
      {
        "Sequence": {
          "id": "B",
          "type_id": 1
        }
      }
    ],
    "special_tokens": {}
  },
  "decoder": null,
  "model": {
    "type": "BPE",
    "dropout": null,
    "unk_token": null,
    "continuing_subword_prefix": null,
    "end_of_word_suffix": null,
    "fuse_unk": false,
    "byte_fallback": false,
    "ignore_merges": false,
    "vocab": {
      "!": 0,
      "B": 1,
      "I": 2,
      "L": 3,
      "T": 4,
      "a": 5,
      "d": 6,
      "e": 7,
      "f": 8,
      "g": 9,
      "i": 10,
      "k": 11,
      "l": 12,
      "m": 13,
      "n": 14,
      "o": 15,
      "r": 16,
      "s": 17,
      "t": 18,
      "u": 19,
      "v": 20,
      "w": 21,
      "y": 22,
      "z": 23,
      "er": 24,
      "en": 25,
      "iz": 26,
      "ken": 27,
      "oken": 28,
      "izer": 29,
      "okenizer": 30,
      "Bu": 31,
      "Lo": 32,
      "Tr": 33,
      "Tokenizer": 34,
      "am": 35,
      "an": 36,
      "ar": 37,
      "az": 38,
      "fo": 39,
      "il": 40,
      "in": 41,
      "mer": 42,
      "ou": 43,
      "ow": 44,
      "rmer": 45,
      "sfo": 46,
      "tokenizer": 47,
      "ve": 48,
      "you": 49,
      "Buil": 50,
      "Love": 51,
      "Tran": 52,
      "Tokenizers": 53,
      "amaz": 54,
      "are": 55,
      "ing": 56,
      "own": 57,
      "sformer": 58,
      "your": 59,
      "Build": 60,
      "Transformer": 61,
      "amazing": 62
    },
    "merges": [
      [
        "e",
        "r"
      ],
      [
        "e",
        "n"
      ],
      [
        "i",
        "z"
      ],
      [
        "k",
        "en"
      ],
      [
        "o",
        "ken"
      ],
      [
        "iz",
        "er"
      ],
      [
        "oken",
        "izer"
      ],
      [
        "B",
        "u"
      ],
      [
        "L",
        "o"
      ],
      [
        "T",
        "r"
      ],
      [
        "T",
        "okenizer"
      ],
      [
        "a",
        "m"
      ],
      [
        "a",
        "n"
      ],
      [
        "a",
        "r"
      ],
      [
        "a",
        "z"
      ],
      [
        "f",
        "o"
      ],
      [
        "i",
        "l"
      ],
      [
        "i",
        "n"
      ],
      [
        "m",
        "er"
      ],
      [
        "o",
        "u"
      ],
      [
        "o",
        "w"
      ],
      [
        "r",
        "mer"
      ],
      [
        "s",
        "fo"
      ],
      [
        "t",
        "okenizer"
      ],
      [
        "v",
        "e"
      ],
      [
        "y",
        "ou"
      ],
      [
        "Bu",
        "il"
      ],
      [
        "Lo",
        "ve"
      ],
      [
        "Tr",
        "an"
      ],
      [
        "Tokenizer",
        "s"
      ],
      [
        "am",
        "az"
      ],
      [
        "ar",
        "e"
      ],
      [
        "in",
        "g"
      ],
      [
        "ow",
        "n"
      ],
      [
        "sfo",
        "rmer"
      ],
      [
        "you",
        "r"
      ],
      [
        "Buil",
        "d"
      ],
      [
        "Tran",
        "sformer"
      ],
      [
        "amaz",
        "ing"
      ]
    ]
  }
}