Initial upload of character-level tokenizer

merges.txt

@@ -0,0 +1 @@
+# This is a character-level tokenizer. There are no merges.

special_tokens_map.json

@@ -0,0 +1,6 @@
+{
+  "unk_token": "<|unk|>",
+  "pad_token": "<|pad|>",
+  "bos_token": "<|bos|>",
+  "eos_token": "<|eos|>"
+}

tokenizer_config.json

@@ -0,0 +1,9 @@
+{
+  "model_type": "gpt2",
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "<|unk|>",
+  "pad_token": "<|pad|>",
+  "bos_token": "<|bos|>",
+  "eos_token": "<|eos|>",
+  "add_prefix_space": false
+}

vocab.json

@@ -0,0 +1,148 @@
+{
+  "<|unk|>": 0,
+  "<|pad|>": 1,
+  "<|bos|>": 2,
+  "<|eos|>": 3,
+  "!": 4,
+  "\"": 5,
+  "#": 6,
+  "$": 7,
+  "%": 8,
+  "&": 9,
+  "'": 10,
+  "(": 11,
+  ")": 12,
+  "*": 13,
+  "+": 14,
+  ",": 15,
+  "-": 16,
+  ".": 17,
+  "/": 18,
+  "0": 19,
+  "1": 20,
+  "2": 21,
+  "3": 22,
+  "4": 23,
+  "5": 24,
+  "6": 25,
+  "7": 26,
+  "8": 27,
+  "9": 28,
+  ":": 29,
+  ";": 30,
+  "<": 31,
+  "=": 32,
+  ">": 33,
+  "?": 34,
+  "@": 35,
+  "[": 36,
+  "\\": 37,
+  "]": 38,
+  "^": 39,
+  "_": 40,
+  "`": 41,
+  "a": 42,
+  "b": 43,
+  "c": 44,
+  "d": 45,
+  "e": 46,
+  "f": 47,
+  "g": 48,
+  "h": 49,
+  "i": 50,
+  "j": 51,
+  "k": 52,
+  "l": 53,
+  "m": 54,
+  "n": 55,
+  "o": 56,
+  "p": 57,
+  "q": 58,
+  "r": 59,
+  "s": 60,
+  "t": 61,
+  "u": 62,
+  "v": 63,
+  "w": 64,
+  "x": 65,
+  "y": 66,
+  "z": 67,
+  "{": 68,
+  "|": 69,
+  "}": 70,
+  "~": 71,
+  "ก": 72,
+  "ข": 73,
+  "ฃ": 74,
+  "ค": 75,
+  "ฅ": 76,
+  "ฆ": 77,
+  "ง": 78,
+  "จ": 79,
+  "ฉ": 80,
+  "ช": 81,
+  "ซ": 82,
+  "ฌ": 83,
+  "ญ": 84,
+  "ฎ": 85,
+  "ฏ": 86,
+  "ฐ": 87,
+  "ฑ": 88,
+  "ฒ": 89,
+  "ณ": 90,
+  "ด": 91,
+  "ต": 92,
+  "ถ": 93,
+  "ท": 94,
+  "ธ": 95,
+  "น": 96,
+  "บ": 97,
+  "ป": 98,
+  "ผ": 99,
+  "ฝ": 100,
+  "พ": 101,
+  "ฟ": 102,
+  "ภ": 103,
+  "ม": 104,
+  "ย": 105,
+  "ร": 106,
+  "ฤ": 107,
+  "ล": 108,
+  "ฦ": 109,
+  "ว": 110,
+  "ศ": 111,
+  "ษ": 112,
+  "ส": 113,
+  "ห": 114,
+  "ฬ": 115,
+  "อ": 116,
+  "ฮ": 117,
+  "ฯ": 118,
+  "ะ": 119,
+  "ั": 120,
+  "า": 121,
+  "ำ": 122,
+  "ิ": 123,
+  "ี": 124,
+  "ึ": 125,
+  "ื": 126,
+  "ุ": 127,
+  "ู": 128,
+  "ฺ": 129,
+  "฿": 130,
+  "เ": 131,
+  "แ": 132,
+  "โ": 133,
+  "ใ": 134,
+  "ไ": 135,
+  "ๅ": 136,
+  "ๆ": 137,
+  "็": 138,
+  "่": 139,
+  "้": 140,
+  "๊": 141,
+  "๋": 142,
+  "์": 143,
+  "ํ": 144,
+  "•": 145
+}