Upload tokenizer

special_tokens_map.json

@@ -0,0 +1,4 @@
+{
+  "eos_token": "<end>",
+  "pad_token": "<pad>"
+}

tokenizer.json

@@ -0,0 +1,823 @@
+{
+  "version": "1.0",
+  "truncation": null,
+  "padding": null,
+  "added_tokens": [
+    {
+      "id": 0,
+      "content": "<start>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 1,
+      "content": "<end>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 2,
+      "content": "<pad>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    }
+  ],
+  "normalizer": null,
+  "pre_tokenizer": {
+    "type": "ByteLevel",
+    "add_prefix_space": false,
+    "trim_offsets": true,
+    "use_regex": true
+  },
+  "post_processor": {
+    "type": "ByteLevel",
+    "add_prefix_space": true,
+    "trim_offsets": false,
+    "use_regex": true
+  },
+  "decoder": {
+    "type": "ByteLevel",
+    "add_prefix_space": true,
+    "trim_offsets": true,
+    "use_regex": true
+  },
+  "model": {
+    "type": "BPE",
+    "dropout": null,
+    "unk_token": null,
+    "continuing_subword_prefix": null,
+    "end_of_word_suffix": null,
+    "fuse_unk": false,
+    "byte_fallback": false,
+    "vocab": {
+      "<start>": 0,
+      "<end>": 1,
+      "<pad>": 2,
+      "\"": 3,
+      "'": 4,
+      ",": 5,
+      "-": 6,
+      ".": 7,
+      "<": 8,
+      ">": 9,
+      "A": 10,
+      "B": 11,
+      "C": 12,
+      "D": 13,
+      "G": 14,
+      "H": 15,
+      "I": 16,
+      "M": 17,
+      "N": 18,
+      "O": 19,
+      "S": 20,
+      "a": 21,
+      "b": 22,
+      "c": 23,
+      "d": 24,
+      "e": 25,
+      "f": 26,
+      "g": 27,
+      "h": 28,
+      "i": 29,
+      "j": 30,
+      "k": 31,
+      "l": 32,
+      "m": 33,
+      "n": 34,
+      "o": 35,
+      "p": 36,
+      "r": 37,
+      "s": 38,
+      "t": 39,
+      "u": 40,
+      "w": 41,
+      "x": 42,
+      "y": 43,
+      "²": 44,
+      "Ì": 45,
+      "Ġ": 46,
+      "st": 47,
+      "nd": 48,
+      "ar": 49,
+      "end": 50,
+      "Ġ<": 51,
+      "star": 52,
+      "start": 53,
+      "hl": 54,
+      "̲": 55,
+      "wi": 56,
+      "ii": 57,
+      "Ġg": 58,
+      "aa": 59,
+      "oo": 60,
+      "Ġn": 61,
+      "Ġwi": 62,
+      "Ġ'": 63,
+      "Ġii": 64,
+      "an": 65,
+      "Ġy": 66,
+      "Ġl": 67,
+      "Ii": 68,
+      "ĠIi": 69,
+      "oohl": 70,
+      "ee": 71,
+      "im": 72,
+      "Ġwil": 73,
+      "Ġh": 74,
+      "whl": 75,
+      "Ġhl": 76,
+      "ag": 77,
+      "dii": 78,
+      "nii": 79,
+      "ts": 80,
+      "xwi": 81,
+      "Ġd": 82,
+      "Ġha": 83,
+      "uu": 84,
+      "Ġnee": 85,
+      "xs": 86,
+      "Ġyu": 87,
+      "Ġa": 88,
+      "ip": 89,
+      "kwhl": 90,
+      "wihl": 91,
+      "gi": 92,
+      "Ġk": 93,
+      "xw": 94,
+      "'m": 95,
+      "Ġxs": 96,
+      "Ġdim": 97,
+      "Ġneedii": 98,
+      "igi": 99,
+      "Ġb": 100,
+      "Ġligi": 101,
+      "Ġwili": 102,
+      "di": 103,
+      "Ġj": 104,
+      "Ġp": 105,
+      "Ġt": 106,
+      "Ġwihl": 107,
+      "sxwi": 108,
+      "Ġs": 109,
+      "Ġya": 110,
+      "in": 111,
+      "Ġhlaa": 112,
+      "Ġna": 113,
+      "Ġan": 114,
+      "ax": 115,
+      "ay": 116,
+      "ahl": 117,
+      "oot": 118,
+      "ni": 119,
+      "ol": 120,
+      "Ġyukwhl": 121,
+      "Ġnii": 122,
+      "Ġnaa": 123,
+      "Ġwilp": 124,
+      "ipe": 125,
+      "Ġpipe": 126,
+      "uxw": 127,
+      "tshl": 128,
+      "Ġyatshl": 129,
+      "ĠS": 130,
+      "na": 131,
+      "hli": 132,
+      "Ġaa": 133,
+      "Ġneediit": 134,
+      "Ġ\"": 135,
+      "̲'": 136,
+      "il": 137,
+      "Ġw": 138,
+      "Ġyee": 139,
+      "Ġloot": 140,
+      "at": 141,
+      "ck": 142,
+      "hol": 143,
+      "ka": 144,
+      "lhl": 145,
+      "ock": 146,
+      "tock": 147,
+      "ya": 148,
+      "wil": 149,
+      "Ġgya": 150,
+      "Ġiin": 151,
+      "Ġluu": 152,
+      "uuhl": 153,
+      "ĠStock": 154,
+      "holm": 155,
+      "ĠStockholm": 156,
+      "ad": 157,
+      "ls": 158,
+      "xu": 159,
+      "Ġts": 160,
+      "hla": 161,
+      "Ġwina": 162,
+      "Ġhlg": 163,
+      "Ġhahla": 164,
+      "uut": 165,
+      "Ġbag": 166,
+      "ayt": 167,
+      "Ġwag": 168,
+      "lsdi": 169,
+      "as": 170,
+      "ok": 171,
+      "Ġhe": 172,
+      "diit": 173,
+      "ain": 174,
+      "wit": 175,
+      "Ġxsa": 176,
+      "Ġxsi": 177,
+      "Ġja": 178,
+      "nit": 179,
+      "xhl": 180,
+      "xwhl": 181,
+      "iihli": 182,
+      "Ġgiihli": 183,
+      "Ġlax": 184,
+      "ak": 185,
+      "̲.": 186,
+      "eek": 187,
+      "Ġap": 188,
+      "Ġxseek": 189,
+      "Ġji": 190,
+      "Ġaats": 191,
+      "hahl": 192,
+      "un": 193,
+      "waa": 194,
+      "oos": 195,
+      "Ġanhahl": 196,
+      "Ġanhahla": 197,
+      "ĠA": 198,
+      "Ġneet": 199,
+      "Ġam": 200,
+      "akwhl": 201,
+      "Ġak": 202,
+      "--": 203,
+      "Can": 204,
+      "Dim": 205,
+      "bi": 206,
+      "da": 207,
+      "fl": 208,
+      "gwaa": 209,
+      "isxwi": 210,
+      "ika": 211,
+      "ja": 212,
+      "kst": 213,
+      "lt": 214,
+      "lst": 215,
+      "nag": 216,
+      "pja": 217,
+      "rain": 218,
+      "sii": 219,
+      "ska": 220,
+      "sgwaa": 221,
+      "upja": 222,
+      "yt": 223,
+      "Ġag": 224,
+      "ĠCan": 225,
+      "Ġfl": 226,
+      "Ġisxwi": 227,
+      "Ġupja": 228,
+      "ndoos": 229,
+      "Ġgi": 230,
+      "Ġgwil": 231,
+      "Ġguuhl": 232,
+      "aahli": 233,
+      "oodi": 234,
+      "Ġno": 235,
+      "anhl": 236,
+      "anwil": 237,
+      "anuut": 238,
+      "anska": 239,
+      "Ġlip": 240,
+      "imil": 241,
+      "niig": 242,
+      "niisgwaa": 243,
+      "Ġyuwi": 244,
+      "Ġandoos": 245,
+      "gihl": 246,
+      "Ġky": 247,
+      "dilhl": 248,
+      "Ġpol": 249,
+      "Ġtun": 250,
+      "Ġtrain": 251,
+      "Ġsgihl": 252,
+      "Ġsdilhl": 253,
+      "Ġyalt": 254,
+      "insxwi": 255,
+      "Ġnakst": 256,
+      "Ġant": 257,
+      "Ġansii": 258,
+      "ayoo": 259,
+      "uxwt": 260,
+      "Ġaam": 261,
+      "adanska": 262,
+      "Ġhlgu": 263,
+      "Ġxsawi": 264,
+      "Ġjabi": 265,
+      "nagwit": 266,
+      "Ġagwihl": 267,
+      "ĠCanadanska": 268,
+      "Ġflika": 269,
+      "Ġgwila": 270,
+      "aahlihl": 271,
+      "anwilat": 272,
+      "anuutxw": 273,
+      "Ġandoosda": 274,
+      "Ġpole": 275,
+      "Ġyaltxu": 276,
+      "Ġansiip": 277,
+      "Hl": 278,
+      "Nii": 279,
+      "Oo": 280,
+      "nim": 281,
+      "wahl": 282,
+      "yhl": 283,
+      "ĠHl": 284,
+      "ĠNii": 285,
+      "wii": 286,
+      "Ġguxw": 287,
+      "Ġguut": 288,
+      "aaxhl": 289,
+      "Ġyuxwhl": 290,
+      "Ġkw": 291,
+      "Ġbas": 292,
+      "inhl": 293,
+      "ootxwi": 294,
+      "nisxwi": 295,
+      "uxwsxwi": 296,
+      "ilx": 297,
+      "adaaxhl": 298,
+      "Ġhlguxwsxwi": 299,
+      "Ġbagu": 300,
+      "asinhl": 301,
+      "Ġamhl": 302,
+      "ĠHlaa": 303,
+      "Ġguxws": 304,
+      "Ġbasax": 305,
+      ".\"": 306,
+      "daa": 307,
+      "ix": 308,
+      "idaa": 309,
+      "loohl": 310,
+      "phl": 311,
+      "pain": 312,
+      "sx": 313,
+      "yim": 314,
+      "Ġxhl": 315,
+      "aahl": 316,
+      "aasx": 317,
+      "aayim": 318,
+      "ook": 319,
+      "Ġhlag": 320,
+      "Ġhlidaa": 321,
+      "xwit": 322,
+      "Ġdok": 323,
+      "Ġdaayim": 324,
+      "Ġyuxw": 325,
+      "Ġaloohl": 326,
+      "Ġbax": 327,
+      "Ġbaasx": 328,
+      "Ġligit": 329,
+      "Ġjok": 330,
+      "Ġsg": 331,
+      "Ġsi": 332,
+      "ĠSpain": 333,
+      "nakwhl": 334,
+      "Ġhehl": 335,
+      "Ġhediit": 336,
+      "diithl": 337,
+      "witxwit": 338,
+      "Ġjaphl": 339,
+      "nithl": 340,
+      "ytxwhl": 341,
+      "Ġxhlii": 342,
+      "Ġdaayimaahl": 343,
+      "Ġyuxwdiithl": 344,
+      "Ġbaasxi": 345,
+      "Nakwhl": 346,
+      "gwi": 347,
+      "ukwhl": 348,
+      "yukwhl": 349,
+      "ĠAk": 350,
+      "ĠAgwi": 351,
+      "ĠAgwiyukwhl": 352,
+      "BM": 353,
+      "De": 354,
+      "Gi": 355,
+      "IBM": 356,
+      "aw": 357,
+      "ail": 358,
+      "ce": 359,
+      "ff": 360,
+      "gee": 361,
+      "it": 362,
+      "iwaa": 363,
+      "ice": 364,
+      "jit": 365,
+      "ljit": 366,
+      "mar": 367,
+      "mail": 368,
+      "nmar": 369,
+      "oxs": 370,
+      "off": 371,
+      "si": 372,
+      "wan": 373,
+      "way": 374,
+      "yo": 375,
+      "ĠDe": 376,
+      "ĠGi": 377,
+      "ĠIBM": 378,
+      "Ġmail": 379,
+      "Ġoff": 380,
+      "niiwan": 381,
+      "niiyo": 382,
+      "xsiwaa": 383,
+      "Ġsaw": 384,
+      "nix": 385,
+      "Ġwok": 386,
+      "atdiit": 387,
+      "̲.\"": 388,
+      "oosun": 389,
+      "ĠAp": 390,
+      "Ġamxsiwaa": 391,
+      "Ġaks": 392,
+      "geenix": 393,
+      "nmark": 394,
+      "oxsxw": 395,
+      "wayi": 396,
+      "ĠDenmark": 397,
+      "ĠGigeenix": 398,
+      "Ġoffice": 399,
+      "Ġsawatdiit": 400,
+      "ytxw": 401
+    },
+    "merges": [
+      "s t",
+      "n d",
+      "a r",
+      "e nd",
+      "Ġ <",
+      "st ar",
+      "star t",
+      "h l",
+      "Ì ²",
+      "w i",
+      "i i",
+      "Ġ g",
+      "a a",
+      "o o",
+      "Ġ n",
+      "Ġ wi",
+      "Ġ '",
+      "Ġ ii",
+      "a n",
+      "Ġ y",
+      "Ġ l",
+      "I i",
+      "Ġ Ii",
+      "oo hl",
+      "e e",
+      "i m",
+      "Ġwi l",
+      "Ġ h",
+      "w hl",
+      "Ġ hl",
+      "a g",
+      "d ii",
+      "n ii",
+      "t s",
+      "x wi",
+      "Ġ d",
+      "Ġh a",
+      "u u",
+      "Ġn ee",
+      "x s",
+      "Ġy u",
+      "Ġ a",
+      "i p",
+      "k whl",
+      "wi hl",
+      "g i",
+      "Ġ k",
+      "x w",
+      "' m",
+      "Ġ xs",
+      "Ġd im",
+      "Ġnee dii",
+      "i gi",
+      "Ġ b",
+      "Ġl igi",
+      "Ġwil i",
+      "d i",
+      "Ġ j",
+      "Ġ p",
+      "Ġ t",
+      "Ġwi hl",
+      "s xwi",
+      "Ġ s",
+      "Ġy a",
+      "i n",
+      "Ġhl aa",
+      "Ġn a",
+      "Ġ an",
+      "a x",
+      "a y",
+      "a hl",
+      "oo t",
+      "n i",
+      "o l",
+      "Ġyu kwhl",
+      "Ġn ii",
+      "Ġn aa",
+      "Ġwil p",
+      "ip e",
+      "Ġp ipe",
+      "u xw",
+      "ts hl",
+      "Ġya tshl",
+      "Ġ S",
+      "n a",
+      "hl i",
+      "Ġ aa",
+      "Ġneedii t",
+      "Ġ \"",
+      "̲ '",
+      "i l",
+      "Ġ w",
+      "Ġy ee",
+      "Ġl oot",
+      "a t",
+      "c k",
+      "h ol",
+      "k a",
+      "l hl",
+      "o ck",
+      "t ock",
+      "y a",
+      "wi l",
+      "Ġg ya",
+      "Ġii n",
+      "Ġl uu",
+      "uu hl",
+      "ĠS tock",
+      "hol m",
+      "ĠStock holm",
+      "a d",
+      "l s",
+      "x u",
+      "Ġ ts",
+      "hl a",
+      "Ġwi na",
+      "Ġhl g",
+      "Ġha hla",
+      "uu t",
+      "Ġb ag",
+      "ay t",
+      "Ġw ag",
+      "ls di",
+      "a s",
+      "o k",
+      "Ġh e",
+      "dii t",
+      "a in",
+      "wi t",
+      "Ġxs a",
+      "Ġxs i",
+      "Ġj a",
+      "ni t",
+      "x hl",
+      "x whl",
+      "ii hli",
+      "Ġg iihli",
+      "Ġl ax",
+      "a k",
+      "̲ .",
+      "ee k",
+      "Ġa p",
+      "Ġxs eek",
+      "Ġj i",
+      "Ġaa ts",
+      "h ahl",
+      "u n",
+      "w aa",
+      "oo s",
+      "Ġan hahl",
+      "Ġanhahl a",
+      "Ġ A",
+      "Ġnee t",
+      "Ġa m",
+      "a kwhl",
+      "Ġa k",
+      "- -",
+      "C an",
+      "D im",
+      "b i",
+      "d a",
+      "f l",
+      "g waa",
+      "i sxwi",
+      "i ka",
+      "j a",
+      "k st",
+      "l t",
+      "l st",
+      "n ag",
+      "p ja",
+      "r ain",
+      "s ii",
+      "s ka",
+      "s gwaa",
+      "u pja",
+      "y t",
+      "Ġ ag",
+      "Ġ Can",
+      "Ġ fl",
+      "Ġ isxwi",
+      "Ġ upja",
+      "nd oos",
+      "Ġg i",
+      "Ġg wil",
+      "Ġg uuhl",
+      "aa hli",
+      "oo di",
+      "Ġn o",
+      "an hl",
+      "an wil",
+      "an uut",
+      "an ska",
+      "Ġl ip",
+      "im il",
+      "nii g",
+      "nii sgwaa",
+      "Ġyu wi",
+      "Ġa ndoos",
+      "gi hl",
+      "Ġk y",
+      "di lhl",
+      "Ġp ol",
+      "Ġt un",
+      "Ġt rain",
+      "Ġs gihl",
+      "Ġs dilhl",
+      "Ġya lt",
+      "in sxwi",
+      "Ġna kst",
+      "Ġan t",
+      "Ġan sii",
+      "ay oo",
+      "uxw t",
+      "Ġaa m",
+      "ad anska",
+      "Ġhlg u",
+      "Ġxsa wi",
+      "Ġja bi",
+      "nag wit",
+      "Ġag wihl",
+      "ĠCan adanska",
+      "Ġfl ika",
+      "Ġgwil a",
+      "aahli hl",
+      "anwil at",
+      "anuut xw",
+      "Ġandoos da",
+      "Ġpol e",
+      "Ġyalt xu",
+      "Ġansii p",
+      "H l",
+      "N ii",
+      "O o",
+      "n im",
+      "w ahl",
+      "y hl",
+      "Ġ Hl",
+      "Ġ Nii",
+      "wi i",
+      "Ġg uxw",
+      "Ġg uut",
+      "aa xhl",
+      "Ġyu xwhl",
+      "Ġk w",
+      "Ġb as",
+      "in hl",
+      "oot xwi",
+      "ni sxwi",
+      "uxw sxwi",
+      "il x",
+      "ad aaxhl",
+      "Ġhlg uxwsxwi",
+      "Ġbag u",
+      "as inhl",
+      "Ġam hl",
+      "ĠHl aa",
+      "Ġguxw s",
+      "Ġbas ax",
+      ". \"",
+      "d aa",
+      "i x",
+      "i daa",
+      "l oohl",
+      "p hl",
+      "p ain",
+      "s x",
+      "y im",
+      "Ġ xhl",
+      "aa hl",
+      "aa sx",
+      "aa yim",
+      "oo k",
+      "Ġhl ag",
+      "Ġhl idaa",
+      "xwi t",
+      "Ġd ok",
+      "Ġd aayim",
+      "Ġyu xw",
+      "Ġa loohl",
+      "Ġb ax",
+      "Ġb aasx",
+      "Ġligi t",
+      "Ġj ok",
+      "Ġs g",
+      "Ġs i",
+      "ĠS pain",
+      "na kwhl",
+      "Ġhe hl",
+      "Ġhe diit",
+      "diit hl",
+      "wit xwit",
+      "Ġja phl",
+      "nit hl",
+      "yt xwhl",
+      "Ġxhl ii",
+      "Ġdaayim aahl",
+      "Ġyuxw diithl",
+      "Ġbaasx i",
+      "N akwhl",
+      "g wi",
+      "u kwhl",
+      "y ukwhl",
+      "ĠA k",
+      "ĠA gwi",
+      "ĠAgwi yukwhl",
+      "B M",
+      "D e",
+      "G i",
+      "I BM",
+      "a w",
+      "a il",
+      "c e",
+      "f f",
+      "g ee",
+      "i t",
+      "i waa",
+      "i ce",
+      "j it",
+      "l jit",
+      "m ar",
+      "m ail",
+      "n mar",
+      "o xs",
+      "o ff",
+      "s i",
+      "w an",
+      "w ay",
+      "y o",
+      "Ġ De",
+      "Ġ Gi",
+      "Ġ IBM",
+      "Ġ mail",
+      "Ġ off",
+      "nii wan",
+      "nii yo",
+      "xs iwaa",
+      "Ġs aw",
+      "ni x",
+      "Ġw ok",
+      "at diit",
+      "̲. \"",
+      "oos un",
+      "ĠA p",
+      "Ġam xsiwaa",
+      "Ġak s",
+      "gee nix",
+      "nmar k",
+      "oxs xw",
+      "way i",
+      "ĠDe nmark",
+      "ĠGi geenix",
+      "Ġoff ice",
+      "Ġsaw atdiit",
+      "yt xw"
+    ]
+  }
+}

tokenizer_config.json

@@ -0,0 +1,8 @@
+{
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "<end>",
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<pad>",
+  "sos_token": "<start>",
+  "tokenizer_class": "PreTrainedTokenizerFast"
+}