serratanis
/

hf_tokenizer

Transformers

Model card Files Files and versions

xet

Community

serratanis commited on Jul 14

Commit

8658ecb

verified ·

1 Parent(s): 4576dbd

Upload tokenizer

Browse files

Files changed (1) hide show

tokenizer.json +109 -157

tokenizer.json CHANGED Viewed

@@ -55,226 +55,178 @@
     "ignore_merges": false,
     "vocab": {
       "<unk>": 0,
-      "'": 1,
-      ".": 2,
-      "a": 3,
-      "b": 4,
-      "c": 5,
-      "d": 6,
-      "e": 7,
-      "f": 8,
-      "g": 9,
-      "h": 10,
-      "i": 11,
-      "k": 12,
-      "l": 13,
-      "m": 14,
-      "n": 15,
-      "o": 16,
-      "p": 17,
-      "r": 18,
-      "s": 19,
-      "t": 20,
-      "u": 21,
-      "w": 22,
-      "x": 23,
-      "y": 24,
-      "he": 25,
-      "the": 26,
-      "an": 27,
-      "od": 28,
-      "is": 29,
-      "te": 30,
-      "ar": 31,
-      "el": 32,
-      "le": 33,
-      "mod": 34,
-      "model": 35,
-      "at": 36,
-      "ge": 37,
-      "mp": 38,
-      "and": 39,
-      "xt": 40,
-      "text": 41,
-      "ld": 42,
-      "ll": 43,
-      "or": 44,
-      "ou": 45,
-      "wor": 46,
-      "hell": 47,
-      "world": 48,
-      "hello": 49,
-      "age": 50,
-      "gu": 51,
-      "lan": 52,
-      "ro": 53,
-      "arn": 54,
-      "learn": 55,
-      "guage": 56,
-      "language": 57,
-      "ad": 58,
-      "go": 59,
-      "in": 60,
-      "imp": 61,
-      "oo": 62,
-      "simp": 63
     },
     "merges": [
       [
-        "h",
-        "e"
-      ],
-      [
-        "t",
-        "he"
       ],
       [
-        "a",
         "n"
       ],
       [
-        "o",
-        "d"
       ],
       [
-        "i",
-        "s"
       ],
       [
         "t",
-        "e"
       ],
       [
         "a",
-        "r"
       ],
       [
-        "e",
         "l"
       ],
       [
-        "l",
-        "e"
       ],
       [
-        "m",
-        "od"
       ],
       [
-        "mod",
-        "el"
       ],
       [
         "a",
-        "t"
       ],
       [
-        "g",
-        "e"
       ],
       [
-        "m",
-        "p"
       ],
       [
-        "an",
-        "d"
       ],
       [
-        "x",
-        "t"
       ],
       [
-        "te",
-        "xt"
       ],
       [
-        "l",
         "d"
       ],
       [
-        "l",
         "l"
       ],
       [
-        "o",
-        "r"
-      ],
-      [
-        "o",
-        "u"
-      ],
-      [
-        "w",
-        "or"
-      ],
-      [
-        "he",
-        "ll"
-      ],
-      [
-        "wor",
-        "ld"
       ],
       [
-        "hell",
-        "o"
       ],
       [
-        "a",
-        "ge"
       ],
       [
         "g",
-        "u"
       ],
       [
         "l",
-        "an"
-      ],
-      [
-        "r",
-        "o"
-      ],
-      [
-        "ar",
-        "n"
-      ],
-      [
-        "le",
-        "arn"
-      ],
-      [
-        "gu",
-        "age"
-      ],
-      [
-        "lan",
-        "guage"
-      ],
-      [
-        "a",
-        "d"
       ],
       [
-        "g",
-        "o"
       ],
       [
-        "i",
         "n"
       ],
       [
-        "i",
-        "mp"
-      ],
-      [
-        "o",
-        "o"
-      ],
-      [
-        "s",
-        "imp"
       ]
     ]
   }

     "ignore_merges": false,
     "vocab": {
       "<unk>": 0,
+      ".": 1,
+      "A": 2,
+      "B": 3,
+      "D": 4,
+      "E": 5,
+      "L": 6,
+      "M": 7,
+      "N": 8,
+      "P": 9,
+      "T": 10,
+      "U": 11,
+      "a": 12,
+      "b": 13,
+      "c": 14,
+      "d": 15,
+      "e": 16,
+      "f": 17,
+      "g": 18,
+      "h": 19,
+      "i": 20,
+      "k": 21,
+      "l": 22,
+      "m": 23,
+      "n": 24,
+      "o": 25,
+      "p": 26,
+      "q": 27,
+      "r": 28,
+      "s": 29,
+      "t": 30,
+      "u": 31,
+      "v": 32,
+      "w": 33,
+      "x": 34,
+      "y": 35,
+      "z": 36,
+      "in": 37,
+      "en": 38,
+      "er": 39,
+      "on": 40,
+      "th": 41,
+      "at": 42,
+      "ing": 43,
+      "al": 44,
+      "an": 45,
+      "es": 46,
+      "to": 47,
+      "ar": 48,
+      "ro": 49,
+      "et": 50,
+      "ic": 51,
+      "ken": 52,
+      "the": 53,
+      "ed": 54,
+      "el": 55,
+      "ion": 56,
+      "ter": 57,
+      "token": 58,
+      "ge": 59,
+      "le": 60,
+      "or": 61,
+      "un": 62,
+      "ur": 63
     },
     "merges": [
       [
+        "i",
+        "n"
       ],
       [
+        "e",
         "n"
       ],
       [
+        "e",
+        "r"
       ],
       [
+        "o",
+        "n"
       ],
       [
         "t",
+        "h"
       ],
       [
         "a",
+        "t"
       ],
       [
+        "in",
+        "g"
+      ],
+      [
+        "a",
         "l"
       ],
       [
+        "a",
+        "n"
       ],
       [
+        "e",
+        "s"
       ],
       [
+        "t",
+        "o"
       ],
       [
         "a",
+        "r"
       ],
       [
+        "r",
+        "o"
       ],
       [
+        "e",
+        "t"
       ],
       [
+        "i",
+        "c"
       ],
       [
+        "k",
+        "en"
       ],
       [
+        "th",
+        "e"
       ],
       [
+        "e",
         "d"
       ],
       [
+        "e",
         "l"
       ],
       [
+        "i",
+        "on"
       ],
       [
+        "t",
+        "er"
       ],
       [
+        "to",
+        "ken"
       ],
       [
         "g",
+        "e"
       ],
       [
         "l",
+        "e"
       ],
       [
+        "o",
+        "r"
       ],
       [
+        "u",
         "n"
       ],
       [
+        "u",
+        "r"
       ]
     ]
   }