Upload tokenizer

merges.txt

@@ -1,4 +1,4 @@
-#version: 0.2 - Trained by `huggingface/tokenizers`
+#version: 0.2
 Ġ Ð
 Ð ¾
 Ð µ

special_tokens_map.json

@@ -20,6 +20,7 @@
     "rstrip": false,
     "single_word": false
   },
+  "pad_token": "<|endoftext|>",
   "unk_token": {
     "content": "<|endoftext|>",
     "lstrip": false,

tokenizer.json

@@ -84,6 +84,7 @@
     "continuing_subword_prefix": "",
     "end_of_word_suffix": "",
     "fuse_unk": false,
+    "byte_fallback": false,
     "vocab": {
       "<pad>": 0,
       "<s>": 1,

tokenizer_config.json

@@ -20,6 +20,7 @@
   "errors": "replace",
   "model_max_length": 1000000000000000019884624838656,
   "pad_token": null,
+  "padding_side": "left",
   "special_tokens_map_file": null,
   "tokenizer_class": "GPT2Tokenizer",
   "unk_token": {