Upload folder using huggingface_hub

README.md

@@ -46,4 +46,4 @@ tokens = tokenizer.encode("Hello, world!")
 ## Sample Encoding
 | Text | Tokens | Token IDs |
 |------|--------|-----------|
-| `123450019 mod 67` | `12, 34, 50, 0, 19,  , mod,  , 67` | `20, 42, 58, 7, 27, 6, 4, 6, 75` |
+| `1234500119 mod 67` | `12, 34, 50, 0, 1, 19,  , mod,  , 67` | `19, 41, 57, 7, 8, 26, 6, 4, 6, 74` |

merges.txt

@@ -88,3 +88,4 @@
 ['9', '6']
 ['9', '7']
 ['9', '8']
+['9', '9']

tokenizer.json

@@ -65,996 +65,6 @@
       "rstrip": false,
       "normalized": false,
       "special": true
-    },
-    {
-      "id": 7,
-      "content": "0",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": true,
-      "special": false
-    },
-    {
-      "id": 8,
-      "content": "1",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": true,
-      "special": false
-    },
-    {
-      "id": 9,
-      "content": "2",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": true,
-      "special": false
-    },
-    {
-      "id": 10,
-      "content": "3",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": true,
-      "special": false
-    },
-    {
-      "id": 11,
-      "content": "4",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": true,
-      "special": false
-    },
-    {
-      "id": 12,
-      "content": "5",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": true,
-      "special": false
-    },
-    {
-      "id": 13,
-      "content": "6",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": true,
-      "special": false
-    },
-    {
-      "id": 14,
-      "content": "7",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": true,
-      "special": false
-    },
-    {
-      "id": 15,
-      "content": "8",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": true,
-      "special": false
-    },
-    {
-      "id": 16,
-      "content": "9",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": true,
-      "special": false
-    },
-    {
-      "id": 18,
-      "content": "10",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": true,
-      "special": false
-    },
-    {
-      "id": 19,
-      "content": "11",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": true,
-      "special": false
-    },
-    {
-      "id": 20,
-      "content": "12",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": true,
-      "special": false
-    },
-    {
-      "id": 21,
-      "content": "13",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": true,
-      "special": false
-    },
-    {
-      "id": 22,
-      "content": "14",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": true,
-      "special": false
-    },
-    {
-      "id": 23,
-      "content": "15",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": true,
-      "special": false
-    },
-    {
-      "id": 24,
-      "content": "16",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": true,
-      "special": false
-    },
-    {
-      "id": 25,
-      "content": "17",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": true,
-      "special": false
-    },
-    {
-      "id": 26,
-      "content": "18",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": true,
-      "special": false
-    },
-    {
-      "id": 27,
-      "content": "19",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": true,
-      "special": false
-    },
-    {
-      "id": 28,
-      "content": "20",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": true,
-      "special": false
-    },
-    {
-      "id": 29,
-      "content": "21",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": true,
-      "special": false
-    },
-    {
-      "id": 30,
-      "content": "22",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": true,
-      "special": false
-    },
-    {
-      "id": 31,
-      "content": "23",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": true,
-      "special": false
-    },
-    {
-      "id": 32,
-      "content": "24",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": true,
-      "special": false
-    },
-    {
-      "id": 33,
-      "content": "25",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": true,
-      "special": false
-    },
-    {
-      "id": 34,
-      "content": "26",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": true,
-      "special": false
-    },
-    {
-      "id": 35,
-      "content": "27",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": true,
-      "special": false
-    },
-    {
-      "id": 36,
-      "content": "28",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": true,
-      "special": false
-    },
-    {
-      "id": 37,
-      "content": "29",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": true,
-      "special": false
-    },
-    {
-      "id": 38,
-      "content": "30",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": true,
-      "special": false
-    },
-    {
-      "id": 39,
-      "content": "31",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": true,
-      "special": false
-    },
-    {
-      "id": 40,
-      "content": "32",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": true,
-      "special": false
-    },
-    {
-      "id": 41,
-      "content": "33",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": true,
-      "special": false
-    },
-    {
-      "id": 42,
-      "content": "34",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": true,
-      "special": false
-    },
-    {
-      "id": 43,
-      "content": "35",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": true,
-      "special": false
-    },
-    {
-      "id": 44,
-      "content": "36",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": true,
-      "special": false
-    },
-    {
-      "id": 45,
-      "content": "37",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": true,
-      "special": false
-    },
-    {
-      "id": 46,
-      "content": "38",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": true,
-      "special": false
-    },
-    {
-      "id": 47,
-      "content": "39",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": true,
-      "special": false
-    },
-    {
-      "id": 48,
-      "content": "40",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": true,
-      "special": false
-    },
-    {
-      "id": 49,
-      "content": "41",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": true,
-      "special": false
-    },
-    {
-      "id": 50,
-      "content": "42",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": true,
-      "special": false
-    },
-    {
-      "id": 51,
-      "content": "43",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": true,
-      "special": false
-    },
-    {
-      "id": 52,
-      "content": "44",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": true,
-      "special": false
-    },
-    {
-      "id": 53,
-      "content": "45",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": true,
-      "special": false
-    },
-    {
-      "id": 54,
-      "content": "46",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": true,
-      "special": false
-    },
-    {
-      "id": 55,
-      "content": "47",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": true,
-      "special": false
-    },
-    {
-      "id": 56,
-      "content": "48",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": true,
-      "special": false
-    },
-    {
-      "id": 57,
-      "content": "49",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": true,
-      "special": false
-    },
-    {
-      "id": 58,
-      "content": "50",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": true,
-      "special": false
-    },
-    {
-      "id": 59,
-      "content": "51",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": true,
-      "special": false
-    },
-    {
-      "id": 60,
-      "content": "52",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": true,
-      "special": false
-    },
-    {
-      "id": 61,
-      "content": "53",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": true,
-      "special": false
-    },
-    {
-      "id": 62,
-      "content": "54",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": true,
-      "special": false
-    },
-    {
-      "id": 63,
-      "content": "55",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": true,
-      "special": false
-    },
-    {
-      "id": 64,
-      "content": "56",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": true,
-      "special": false
-    },
-    {
-      "id": 65,
-      "content": "57",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": true,
-      "special": false
-    },
-    {
-      "id": 66,
-      "content": "58",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": true,
-      "special": false
-    },
-    {
-      "id": 67,
-      "content": "59",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": true,
-      "special": false
-    },
-    {
-      "id": 68,
-      "content": "60",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": true,
-      "special": false
-    },
-    {
-      "id": 69,
-      "content": "61",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": true,
-      "special": false
-    },
-    {
-      "id": 70,
-      "content": "62",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": true,
-      "special": false
-    },
-    {
-      "id": 71,
-      "content": "63",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": true,
-      "special": false
-    },
-    {
-      "id": 72,
-      "content": "64",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": true,
-      "special": false
-    },
-    {
-      "id": 73,
-      "content": "65",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": true,
-      "special": false
-    },
-    {
-      "id": 74,
-      "content": "66",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": true,
-      "special": false
-    },
-    {
-      "id": 75,
-      "content": "67",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": true,
-      "special": false
-    },
-    {
-      "id": 76,
-      "content": "68",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": true,
-      "special": false
-    },
-    {
-      "id": 77,
-      "content": "69",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": true,
-      "special": false
-    },
-    {
-      "id": 78,
-      "content": "70",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": true,
-      "special": false
-    },
-    {
-      "id": 79,
-      "content": "71",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": true,
-      "special": false
-    },
-    {
-      "id": 80,
-      "content": "72",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": true,
-      "special": false
-    },
-    {
-      "id": 81,
-      "content": "73",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": true,
-      "special": false
-    },
-    {
-      "id": 82,
-      "content": "74",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": true,
-      "special": false
-    },
-    {
-      "id": 83,
-      "content": "75",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": true,
-      "special": false
-    },
-    {
-      "id": 84,
-      "content": "76",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": true,
-      "special": false
-    },
-    {
-      "id": 85,
-      "content": "77",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": true,
-      "special": false
-    },
-    {
-      "id": 86,
-      "content": "78",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": true,
-      "special": false
-    },
-    {
-      "id": 87,
-      "content": "79",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": true,
-      "special": false
-    },
-    {
-      "id": 88,
-      "content": "80",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": true,
-      "special": false
-    },
-    {
-      "id": 89,
-      "content": "81",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": true,
-      "special": false
-    },
-    {
-      "id": 90,
-      "content": "82",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": true,
-      "special": false
-    },
-    {
-      "id": 91,
-      "content": "83",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": true,
-      "special": false
-    },
-    {
-      "id": 92,
-      "content": "84",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": true,
-      "special": false
-    },
-    {
-      "id": 93,
-      "content": "85",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": true,
-      "special": false
-    },
-    {
-      "id": 94,
-      "content": "86",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": true,
-      "special": false
-    },
-    {
-      "id": 95,
-      "content": "87",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": true,
-      "special": false
-    },
-    {
-      "id": 96,
-      "content": "88",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": true,
-      "special": false
-    },
-    {
-      "id": 97,
-      "content": "89",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": true,
-      "special": false
-    },
-    {
-      "id": 98,
-      "content": "90",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": true,
-      "special": false
-    },
-    {
-      "id": 99,
-      "content": "91",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": true,
-      "special": false
-    },
-    {
-      "id": 100,
-      "content": "92",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": true,
-      "special": false
-    },
-    {
-      "id": 101,
-      "content": "93",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": true,
-      "special": false
-    },
-    {
-      "id": 102,
-      "content": "94",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": true,
-      "special": false
-    },
-    {
-      "id": 103,
-      "content": "95",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": true,
-      "special": false
-    },
-    {
-      "id": 104,
-      "content": "96",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": true,
-      "special": false
-    },
-    {
-      "id": 105,
-      "content": "97",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": true,
-      "special": false
-    },
-    {
-      "id": 106,
-      "content": "98",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": true,
-      "special": false
-    },
-    {
-      "id": 107,
-      "content": " 0",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": true,
-      "special": false
-    },
-    {
-      "id": 108,
-      "content": " 1",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": true,
-      "special": false
-    },
-    {
-      "id": 109,
-      "content": " 2",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": true,
-      "special": false
-    },
-    {
-      "id": 110,
-      "content": " 3",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": true,
-      "special": false
-    },
-    {
-      "id": 111,
-      "content": " 4",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": true,
-      "special": false
-    },
-    {
-      "id": 112,
-      "content": " 5",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": true,
-      "special": false
-    },
-    {
-      "id": 113,
-      "content": " 6",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": true,
-      "special": false
-    },
-    {
-      "id": 114,
-      "content": " 7",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": true,
-      "special": false
-    },
-    {
-      "id": 115,
-      "content": " 8",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": true,
-      "special": false
-    },
-    {
-      "id": 116,
-      "content": " 9",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": true,
-      "special": false
-    },
-    {
-      "id": 117,
-      "content": "99",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": true,
-      "special": false
     }
   ],
   "normalizer": null,
@@ -1111,96 +121,96 @@
       "7": 14,
       "8": 15,
       "9": 16,
-      "Ġ": 17,
-      "10": 18,
-      "11": 19,
-      "12": 20,
-      "13": 21,
-      "14": 22,
-      "15": 23,
-      "16": 24,
-      "17": 25,
-      "18": 26,
-      "19": 27,
-      "20": 28,
-      "21": 29,
-      "22": 30,
-      "23": 31,
-      "24": 32,
-      "25": 33,
-      "26": 34,
-      "27": 35,
-      "28": 36,
-      "29": 37,
-      "30": 38,
-      "31": 39,
-      "32": 40,
-      "33": 41,
-      "34": 42,
-      "35": 43,
-      "36": 44,
-      "37": 45,
-      "38": 46,
-      "39": 47,
-      "40": 48,
-      "41": 49,
-      "42": 50,
-      "43": 51,
-      "44": 52,
-      "45": 53,
-      "46": 54,
-      "47": 55,
-      "48": 56,
-      "49": 57,
-      "50": 58,
-      "51": 59,
-      "52": 60,
-      "53": 61,
-      "54": 62,
-      "55": 63,
-      "56": 64,
-      "57": 65,
-      "58": 66,
-      "59": 67,
-      "60": 68,
-      "61": 69,
-      "62": 70,
-      "63": 71,
-      "64": 72,
-      "65": 73,
-      "66": 74,
-      "67": 75,
-      "68": 76,
-      "69": 77,
-      "70": 78,
-      "71": 79,
-      "72": 80,
-      "73": 81,
-      "74": 82,
-      "75": 83,
-      "76": 84,
-      "77": 85,
-      "78": 86,
-      "79": 87,
-      "80": 88,
-      "81": 89,
-      "82": 90,
-      "83": 91,
-      "84": 92,
-      "85": 93,
-      "86": 94,
-      "87": 95,
-      "88": 96,
-      "89": 97,
-      "90": 98,
-      "91": 99,
-      "92": 100,
-      "93": 101,
-      "94": 102,
-      "95": 103,
-      "96": 104,
-      "97": 105,
-      "98": 106
+      "10": 17,
+      "11": 18,
+      "12": 19,
+      "13": 20,
+      "14": 21,
+      "15": 22,
+      "16": 23,
+      "17": 24,
+      "18": 25,
+      "19": 26,
+      "20": 27,
+      "21": 28,
+      "22": 29,
+      "23": 30,
+      "24": 31,
+      "25": 32,
+      "26": 33,
+      "27": 34,
+      "28": 35,
+      "29": 36,
+      "30": 37,
+      "31": 38,
+      "32": 39,
+      "33": 40,
+      "34": 41,
+      "35": 42,
+      "36": 43,
+      "37": 44,
+      "38": 45,
+      "39": 46,
+      "40": 47,
+      "41": 48,
+      "42": 49,
+      "43": 50,
+      "44": 51,
+      "45": 52,
+      "46": 53,
+      "47": 54,
+      "48": 55,
+      "49": 56,
+      "50": 57,
+      "51": 58,
+      "52": 59,
+      "53": 60,
+      "54": 61,
+      "55": 62,
+      "56": 63,
+      "57": 64,
+      "58": 65,
+      "59": 66,
+      "60": 67,
+      "61": 68,
+      "62": 69,
+      "63": 70,
+      "64": 71,
+      "65": 72,
+      "66": 73,
+      "67": 74,
+      "68": 75,
+      "69": 76,
+      "70": 77,
+      "71": 78,
+      "72": 79,
+      "73": 80,
+      "74": 81,
+      "75": 82,
+      "76": 83,
+      "77": 84,
+      "78": 85,
+      "79": 86,
+      "80": 87,
+      "81": 88,
+      "82": 89,
+      "83": 90,
+      "84": 91,
+      "85": 92,
+      "86": 93,
+      "87": 94,
+      "88": 95,
+      "89": 96,
+      "90": 97,
+      "91": 98,
+      "92": 99,
+      "93": 100,
+      "94": 101,
+      "95": 102,
+      "96": 103,
+      "97": 104,
+      "98": 105,
+      "99": 106
     },
     "merges": [
       [
@@ -1558,6 +568,10 @@
       [
         "9",
         "8"
+      ],
+      [
+        "9",
+        "9"
       ]
     ]
   }

tokenizer_config.json

@@ -55,886 +55,6 @@
       "rstrip": false,
       "single_word": false,
       "special": true
-    },
-    "7": {
-      "content": "0",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "8": {
-      "content": "1",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "9": {
-      "content": "2",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "10": {
-      "content": "3",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "11": {
-      "content": "4",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "12": {
-      "content": "5",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "13": {
-      "content": "6",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "14": {
-      "content": "7",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "15": {
-      "content": "8",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "16": {
-      "content": "9",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "18": {
-      "content": "10",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "19": {
-      "content": "11",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "20": {
-      "content": "12",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "21": {
-      "content": "13",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "22": {
-      "content": "14",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "23": {
-      "content": "15",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "24": {
-      "content": "16",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "25": {
-      "content": "17",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "26": {
-      "content": "18",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "27": {
-      "content": "19",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "28": {
-      "content": "20",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "29": {
-      "content": "21",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "30": {
-      "content": "22",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "31": {
-      "content": "23",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "32": {
-      "content": "24",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "33": {
-      "content": "25",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "34": {
-      "content": "26",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "35": {
-      "content": "27",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "36": {
-      "content": "28",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "37": {
-      "content": "29",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "38": {
-      "content": "30",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "39": {
-      "content": "31",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "40": {
-      "content": "32",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "41": {
-      "content": "33",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "42": {
-      "content": "34",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "43": {
-      "content": "35",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "44": {
-      "content": "36",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "45": {
-      "content": "37",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "46": {
-      "content": "38",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "47": {
-      "content": "39",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "48": {
-      "content": "40",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "49": {
-      "content": "41",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "50": {
-      "content": "42",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "51": {
-      "content": "43",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "52": {
-      "content": "44",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "53": {
-      "content": "45",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "54": {
-      "content": "46",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "55": {
-      "content": "47",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "56": {
-      "content": "48",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "57": {
-      "content": "49",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "58": {
-      "content": "50",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "59": {
-      "content": "51",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "60": {
-      "content": "52",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "61": {
-      "content": "53",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "62": {
-      "content": "54",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "63": {
-      "content": "55",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "64": {
-      "content": "56",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "65": {
-      "content": "57",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "66": {
-      "content": "58",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "67": {
-      "content": "59",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "68": {
-      "content": "60",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "69": {
-      "content": "61",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "70": {
-      "content": "62",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "71": {
-      "content": "63",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "72": {
-      "content": "64",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "73": {
-      "content": "65",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "74": {
-      "content": "66",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "75": {
-      "content": "67",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "76": {
-      "content": "68",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "77": {
-      "content": "69",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "78": {
-      "content": "70",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "79": {
-      "content": "71",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "80": {
-      "content": "72",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "81": {
-      "content": "73",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "82": {
-      "content": "74",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "83": {
-      "content": "75",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "84": {
-      "content": "76",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "85": {
-      "content": "77",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "86": {
-      "content": "78",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "87": {
-      "content": "79",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "88": {
-      "content": "80",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "89": {
-      "content": "81",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "90": {
-      "content": "82",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "91": {
-      "content": "83",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "92": {
-      "content": "84",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "93": {
-      "content": "85",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "94": {
-      "content": "86",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "95": {
-      "content": "87",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "96": {
-      "content": "88",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "97": {
-      "content": "89",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "98": {
-      "content": "90",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "99": {
-      "content": "91",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "100": {
-      "content": "92",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "101": {
-      "content": "93",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "102": {
-      "content": "94",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "103": {
-      "content": "95",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "104": {
-      "content": "96",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "105": {
-      "content": "97",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "106": {
-      "content": "98",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "107": {
-      "content": " 0",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "108": {
-      "content": " 1",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "109": {
-      "content": " 2",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "110": {
-      "content": " 3",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "111": {
-      "content": " 4",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "112": {
-      "content": " 5",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "113": {
-      "content": " 6",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "114": {
-      "content": " 7",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "115": {
-      "content": " 8",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "116": {
-      "content": " 9",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "117": {
-      "content": "99",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
     }
   },
   "additional_special_tokens": [

vocab.json

@@ -1,109 +1,109 @@
 {
-  "40": 48,
-  "37": 45,
-  "53": 61,
+  "17": 24,
+  "43": 50,
+  "16": 23,
+  "57": 64,
+  "9": 16,
+  "31": 38,
+  "</s>": 2,
+  "44": 51,
+  "26": 33,
+  "56": 63,
+  "21": 28,
+  "66": 73,
+  "48": 55,
   "5": 12,
-  "29": 37,
-  "91": 99,
-  "68": 76,
-  "55": 63,
-  "75": 83,
-  "97": 105,
-  "4": 11,
-  "47": 55,
+  "=": 5,
   "3": 10,
+  "94": 101,
+  "46": 53,
+  "71": 78,
+  "81": 88,
+  "61": 68,
+  "42": 49,
+  "4": 11,
+  "41": 48,
+  "65": 72,
+  "62": 69,
+  "<unk>": 0,
   "<s>": 1,
-  "87": 95,
-  "88": 96,
+  "11": 18,
+  "0": 7,
+  "36": 43,
+  "73": 80,
+  "30": 37,
+  "40": 47,
+  "mod": 4,
+  "22": 29,
+  "72": 79,
+  "87": 94,
+  "88": 95,
+  "76": 83,
+  "35": 42,
+  "99": 106,
   "7": 14,
-  "20": 28,
-  "Ġ": 17,
-  "45": 53,
-  "57": 65,
-  "49": 57,
-  "76": 84,
-  "19": 27,
-  "69": 77,
-  "23": 31,
-  "41": 49,
-  "8": 15,
-  "89": 97,
-  "17": 25,
-  "66": 74,
-  "65": 73,
-  "18": 26,
-  "84": 92,
-  "51": 59,
-  "67": 75,
-  "21": 29,
-  "56": 64,
-  "38": 46,
-  "44": 52,
-  "28": 36,
-  "26": 34,
-  "92": 100,
-  "72": 80,
-  "31": 39,
-  "90": 98,
-  "52": 60,
-  "39": 47,
-  "50": 58,
-  "81": 89,
-  "94": 102,
-  "13": 21,
-  "25": 33,
-  "9": 16,
-  "6": 13,
-  "80": 88,
-  "98": 106,
+  "34": 41,
+  "58": 65,
+  "23": 30,
+  "64": 71,
+  "25": 32,
+  "60": 67,
   " ": 6,
-  "48": 56,
-  "33": 41,
-  "46": 54,
-  "71": 79,
-  "12": 20,
-  "96": 104,
-  "10": 18,
-  "59": 67,
-  "34": 42,
-  "74": 82,
-  "58": 66,
-  "27": 35,
-  "70": 78,
-  "73": 81,
-  "54": 62,
-  "=": 5,
-  "83": 91,
-  "mod": 4,
-  "82": 90,
-  "60": 68,
-  "2": 9,
-  "95": 103,
-  "93": 101,
-  "77": 85,
-  "79": 87,
-  "63": 71,
-  "85": 93,
-  "0": 7,
-  "30": 38,
-  "35": 43,
-  "42": 50,
-  "22": 30,
-  "<unk>": 0,
-  "64": 72,
+  "14": 21,
+  "45": 52,
+  "32": 39,
   "1": 8,
+  "12": 19,
+  "50": 57,
+  "47": 54,
+  "67": 74,
+  "74": 81,
+  "18": 25,
+  "49": 56,
+  "85": 92,
+  "90": 97,
+  "2": 9,
+  "6": 13,
+  "15": 22,
+  "82": 89,
+  "27": 34,
+  "69": 76,
+  "79": 86,
+  "39": 46,
+  "97": 104,
+  "53": 60,
+  "8": 15,
+  "33": 40,
+  "13": 20,
+  "52": 59,
+  "96": 103,
+  "10": 17,
+  "37": 44,
+  "93": 100,
+  "89": 96,
   "<pad>": 3,
-  "61": 69,
-  "16": 24,
-  "62": 70,
-  "15": 23,
-  "32": 40,
-  "86": 94,
-  "24": 32,
-  "11": 19,
-  "78": 86,
-  "</s>": 2,
-  "43": 51,
-  "14": 22,
-  "36": 44
+  "80": 87,
+  "24": 31,
+  "28": 35,
+  "68": 75,
+  "38": 45,
+  "55": 62,
+  "84": 91,
+  "63": 70,
+  "19": 26,
+  "86": 93,
+  "91": 98,
+  "51": 58,
+  "20": 27,
+  "54": 61,
+  "75": 82,
+  "29": 36,
+  "77": 84,
+  "59": 66,
+  "98": 105,
+  "83": 90,
+  "95": 102,
+  "78": 85,
+  "70": 77,
+  "92": 99
 }