Upload tokenizer

added_tokens.json

@@ -0,0 +1,5 @@
+{
+  "<|bos|>": 52928,
+  "<|eos|>": 52929,
+  "<|pad|>": 52930
+}

special_tokens_map.json

@@ -1,20 +1,20 @@
 {
   "bos_token": {
-    "content": "<s>",
+    "content": "<|bos|>",
     "lstrip": false,
     "normalized": true,
     "rstrip": false,
     "single_word": false
   },
   "eos_token": {
-    "content": "</s>",
+    "content": "<|eos|>",
     "lstrip": false,
     "normalized": true,
     "rstrip": false,
     "single_word": false
   },
   "pad_token": {
-    "content": "<pad>",
+    "content": "<|pad|>",
     "lstrip": false,
     "normalized": true,
     "rstrip": false,

tokenizer.model

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9d3b73f0ec0077cc30378a54fa2053ebb3bde8ed7c54e11fee1609bf16cdad98
-size 828483
+oid sha256:68d82608100430846be804d586a586a80fc77d4f0eb06ee6d62133a47c9beb9f
+size 827607

tokenizer_config.json

@@ -1,9 +1,9 @@
 {
   "add_bos_token": true,
-  "add_eos_token": false,
+  "add_eos_token": true,
   "bos_token": {
     "__type": "AddedToken",
-    "content": "<s>",
+    "content": "<|bos|>",
     "lstrip": false,
     "normalized": true,
     "rstrip": false,
@@ -12,7 +12,7 @@
   "clean_up_tokenization_spaces": false,
   "eos_token": {
     "__type": "AddedToken",
-    "content": "</s>",
+    "content": "<|eos|>",
     "lstrip": false,
     "normalized": true,
     "rstrip": false,
@@ -21,7 +21,7 @@
   "model_max_length": 1000000000000000019884624838656,
   "pad_token": {
     "__type": "AddedToken",
-    "content": "<pad>",
+    "content": "<|pad|>",
     "lstrip": false,
     "normalized": true,
     "rstrip": false,