Upload tokenizer

tokenizer.json

@@ -155,12 +155,6 @@
           "id": "A",
           "type_id": 0
         }
-      },
-      {
-        "SpecialToken": {
-          "id": "<|endoftext|>",
-          "type_id": 0
-        }
       }
     ],
     "pair": [
@@ -170,36 +164,14 @@
           "type_id": 0
         }
       },
-      {
-        "SpecialToken": {
-          "id": "<|endoftext|>",
-          "type_id": 0
-        }
-      },
       {
         "Sequence": {
           "id": "B",
           "type_id": 1
         }
-      },
-      {
-        "SpecialToken": {
-          "id": "<|endoftext|>",
-          "type_id": 1
-        }
       }
     ],
-    "special_tokens": {
-      "<|endoftext|>": {
-        "id": "<|endoftext|>",
-        "ids": [
-          32000
-        ],
-        "tokens": [
-          "<|endoftext|>"
-        ]
-      }
-    }
+    "special_tokens": {}
   },
   "decoder": {
     "type": "Sequence",

tokenizer_config.json

@@ -1,6 +1,6 @@
 {
   "add_bos_token": false,
-  "add_eos_token": true,
+  "add_eos_token": false,
   "add_prefix_space": null,
   "added_tokens_decoder": {
     "0": {
@@ -121,9 +121,9 @@
   "clean_up_tokenization_spaces": false,
   "eos_token": "<|endoftext|>",
   "legacy": false,
-  "model_max_length": 131072,
+  "model_max_length": 300,
   "pad_token": "<unk>",
-  "padding_side": "left",
+  "padding_side": "right",
   "sp_model_kwargs": {},
   "tokenizer_class": "LlamaTokenizer",
   "unk_token": "<unk>",