Upload folder using huggingface_hub

special_tokens_map.json

@@ -0,0 +1,6 @@
+{
+  "bos_token": "<s>",
+  "eos_token": "</s>",
+  "pad_token": "<pad>",
+  "unk_token": "<unk>"
+}

tokenizer_config.json

@@ -0,0 +1,11 @@
+{
+  "tokenizer_class": "CharacterTokenizer",
+  "auto_map": {
+    "AutoTokenizer": "tokenizer.CharacterTokenizer"
+  },
+  "bos_token": "<s>",
+  "eos_token": "</s>",
+  "unk_token": "<unk>",
+  "pad_token": "<pad>",
+  "vocab_file": "vocab.json"
+}

vocab.json

@@ -0,0 +1,147 @@
+{
+  "<pad>": 0,
+  "<unk>": 1,
+  "<s>": 2,
+  "</s>": 3,
+  " ": 4,
+  "!": 5,
+  "\"": 6,
+  "#": 7,
+  "$": 8,
+  "%": 9,
+  "&": 10,
+  "'": 11,
+  "(": 12,
+  ")": 13,
+  "*": 14,
+  "+": 15,
+  ",": 16,
+  "-": 17,
+  ".": 18,
+  "/": 19,
+  "0": 20,
+  "1": 21,
+  "2": 22,
+  "3": 23,
+  "4": 24,
+  "5": 25,
+  "6": 26,
+  "7": 27,
+  "8": 28,
+  "9": 29,
+  ":": 30,
+  ";": 31,
+  "<": 32,
+  "=": 33,
+  ">": 34,
+  "?": 35,
+  "@": 36,
+  "A": 37,
+  "B": 38,
+  "C": 39,
+  "D": 40,
+  "E": 41,
+  "F": 42,
+  "G": 43,
+  "H": 44,
+  "I": 45,
+  "J": 46,
+  "K": 47,
+  "L": 48,
+  "M": 49,
+  "N": 50,
+  "O": 51,
+  "P": 52,
+  "Q": 53,
+  "R": 54,
+  "S": 55,
+  "T": 56,
+  "U": 57,
+  "V": 58,
+  "W": 59,
+  "X": 60,
+  "Y": 61,
+  "Z": 62,
+  "[": 63,
+  "\\": 64,
+  "]": 65,
+  "^": 66,
+  "_": 67,
+  "`": 68,
+  "a": 69,
+  "b": 70,
+  "c": 71,
+  "d": 72,
+  "e": 73,
+  "f": 74,
+  "g": 75,
+  "h": 76,
+  "i": 77,
+  "j": 78,
+  "k": 79,
+  "l": 80,
+  "m": 81,
+  "n": 82,
+  "o": 83,
+  "p": 84,
+  "q": 85,
+  "r": 86,
+  "s": 87,
+  "t": 88,
+  "u": 89,
+  "v": 90,
+  "w": 91,
+  "x": 92,
+  "y": 93,
+  "z": 94,
+  "{": 95,
+  "|": 96,
+  "}": 97,
+  "~": 98,
+  "£": 99,
+  "©": 100,
+  "®": 101,
+  "°": 102,
+  "à": 103,
+  "á": 104,
+  "ä": 105,
+  "å": 106,
+  "ç": 107,
+  "è": 108,
+  "é": 109,
+  "ê": 110,
+  "ë": 111,
+  "î": 112,
+  "ï": 113,
+  "ñ": 114,
+  "ó": 115,
+  "ô": 116,
+  "ö": 117,
+  "ø": 118,
+  "û": 119,
+  "ü": 120,
+  "ƍ": 121,
+  "̄": 122,
+  "Δ": 123,
+  "α": 124,
+  "β": 125,
+  "γ": 126,
+  "η": 127,
+  "θ": 128,
+  "κ": 129,
+  "λ": 130,
+  "μ": 131,
+  "ν": 132,
+  "ξ": 133,
+  "π": 134,
+  "σ": 135,
+  "τ": 136,
+  "φ": 137,
+  "ψ": 138,
+  "ω": 139,
+  "•": 140,
+  "′": 141,
+  "€": 142,
+  "☐": 143,
+  "☒": 144
+}