tokenizers/tokenizer-latin-1.json

{
  "version": "1.0",
  "truncation": null,
  "padding": null,
  "added_tokens": [
    {
      "id": 0,
      "content": "[STOP]",
      "single_word": false,
      "lstrip": false,
      "rstrip": false,
      "normalized": false,
      "special": true
    },
    {
      "id": 1,
      "content": "[UNK]",
      "single_word": false,
      "lstrip": false,
      "rstrip": false,
      "normalized": false,
      "special": true
    },
    {
      "id": 2,
      "content": "[SPACE]",
      "single_word": false,
      "lstrip": false,
      "rstrip": false,
      "normalized": false,
      "special": true
    }
  ],
  "normalizer": null,
  "pre_tokenizer": {
    "type": "Whitespace"
  },
  "post_processor": null,
  "decoder": null,
  "model": {
    "type": "BPE",
    "dropout": null,
    "unk_token": "[UNK]",
    "continuing_subword_prefix": null,
    "end_of_word_suffix": null,
    "fuse_unk": false,
    "byte_fallback": false,
    "vocab": {
      "[STOP]": 0,
      "[UNK]": 1,
      "[SPACE]": 2,
	"!":3,
	"'":4,
	"(":5,
	")":6,
	",":7,
	"-":8,
	".":9,
	"/":10,
	":":11,
	";":12,
	"?":13,
	"a":14,
	"b":15,
	"c":16,
	"d":17,
	"e":18,
	"f":19,
	"g":20,
	"h":21,
	"i":22,
	"j":23,
	"k":24,
	"l":25,
	"m":26,
	"n":27,
	"o":28,
	"p":29,
	"q":30,
	"r":31,
	"s":32,
	"t":33,
	"u":34,
	"v":35,
	  "w":36,
      "x":37,
	  "y":38,
	  "z":39,
"¿": 40,
"À": 41,
"Á": 42,
"Â": 43,
"Ã": 44,
"Ä": 45,
"Å": 46,
"Æ": 47,
"Ç": 48,
"È": 49,
"É": 50,
"Ê": 51,
"Ë": 52,
"Ì": 53,
"Í": 54,
"Î": 55,
"Ï": 56,
"Ð": 57,
"Ñ": 58,
"Ò": 59,
"Ó": 60,
"Ô": 61,
"Õ": 62,
"Ö": 63,
"Ø": 64,
"Ù": 65,
"Ú": 66,
"Û": 67,
"Ü": 68,
"Ý": 69,
"Þ": 70,
"ß": 71,
"à": 72,
"á": 73,
"â": 74,
"ã": 75,
"ä": 76,
"å": 77,
"æ": 78,
"ç": 79,
"è": 80,
"é": 81,
"ê": 82,
"ë": 83,
"ì": 84,
"í": 85,
"î": 86,
"ï": 87,
"ð": 88,
"ñ": 89,
"ò": 90,
"ó": 91,
"ô": 92,
"õ": 93,
"ö": 94,
"ø": 95,
"ù": 96,
"ú": 97,
"û": 98,
"ü": 99,
"ý": 100,
"þ": 101,
"ÿ": 102,
"¡": 103
    },
    "merges": []
  }
}