{
  "version": "1.0",
  "truncation": {
    "max_length": 65536,
    "strategy": "longest_first",
    "direction": "right"
  },
  "padding": {
    "strategy": "right",
    "pad_id": 0,
    "pad_token": "<pad>"
  },
  "added_tokens": [
    {"id": 0, "content": "<pad>", "single_word": false, "special": true},
    {"id": 1, "content": "<bos>", "single_word": false, "special": true},
    {"id": 2, "content": "<eos>", "single_word": false, "special": true}
  ],
  "normalizer": {
    "type": "NFKC"
  },
  "pre_tokenizer": {
    "type": "Whitespace"
  },
  "post_processor": {
    "type": "TemplateProcessing",
    "single": "<bos> $A <eos>",
    "pair": "<bos> $A <eos> $B <eos>",
    "special_tokens": {
      "<bos>": 1,
      "<eos>": 2
    }
  },
  "decoder": {
    "type": "WordPiece",
    "prefix": "##"
  },
  "model": {
    "type": "BPE",
    "vocab": {
      "<pad>": 0,
      "<bos>": 1,
      "<eos>": 2,
      "the": 3,
      "of": 4,
      "to": 5,
      "and": 6,
      "I": 7,
      "you": 8
    },
    "merges": [
      "t h",
      "th e",
      "a n",
      "a nd",
      "i n",
      "i ng"
    ]
  }
}