Upload folder using huggingface_hub

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b284fd57d27bb0a2a1c0964c39de34e9b9c35d23dc30bb2595629bd4aa482a47
+oid sha256:62ab482f1b2b08e746302264cab16536b7e335238fd69e80108859dbf36c902e
 size 2481263088

special_tokens_map.json

@@ -1,9 +1,4 @@
 {
-  "bos_token": "<|bos|>",
-  "eos_token": "<|eos|>",
-  "unk_token": "<|unk|>",
-  "pad_token": "<|pad|>",
-  "mask_token": "<|mask|>",
   "additional_special_tokens": [
     "<|reserved_265|>",
     "<|reserved_266|>",
@@ -28,5 +23,40 @@
     "<|reserved_285|>",
     "<|reserved_286|>",
     "<|reserved_287|>"
-  ]
-}
+  ],
+  "bos_token": {
+    "content": "<|bos|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|eos|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "mask_token": {
+    "content": "<|mask|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|pad|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<|unk|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer_config.json

@@ -1,45 +1,6 @@
 {
-  "tokenizer_class": "PreTrainedTokenizerFast",
-  "model_max_length": 1000000000000000019884624838656,
-  "padding_side": "right",
-  "truncation_side": "right",
-  "bos_token": "<|bos|>",
-  "eos_token": "<|eos|>",
-  "unk_token": "<|unk|>",
-  "pad_token": "<|pad|>",
-  "mask_token": "<|mask|>",
   "add_bos_token": false,
   "add_eos_token": false,
-  "clean_up_tokenization_spaces": true,
-  "legacy": false,
-  "name_or_path": "custom_autotokenizer_byte_tokenizer",
-  "trust_remote_code": false,
-  "use_fast": true,
-  "additional_special_tokens": [
-    "<|reserved_265|>",
-    "<|reserved_266|>",
-    "<|reserved_267|>",
-    "<|reserved_268|>",
-    "<|reserved_269|>",
-    "<|reserved_270|>",
-    "<|reserved_271|>",
-    "<|reserved_272|>",
-    "<|reserved_273|>",
-    "<|reserved_274|>",
-    "<|reserved_275|>",
-    "<|reserved_276|>",
-    "<|reserved_277|>",
-    "<|reserved_278|>",
-    "<|reserved_279|>",
-    "<|reserved_280|>",
-    "<|reserved_281|>",
-    "<|reserved_282|>",
-    "<|reserved_283|>",
-    "<|reserved_284|>",
-    "<|reserved_285|>",
-    "<|reserved_286|>",
-    "<|reserved_287|>"
-  ],
   "added_tokens_decoder": {
     "256": {
       "content": "<|pad|>",
@@ -297,5 +258,44 @@
       "single_word": false,
       "special": true
     }
-  }
-}
+  },
+  "additional_special_tokens": [
+    "<|reserved_265|>",
+    "<|reserved_266|>",
+    "<|reserved_267|>",
+    "<|reserved_268|>",
+    "<|reserved_269|>",
+    "<|reserved_270|>",
+    "<|reserved_271|>",
+    "<|reserved_272|>",
+    "<|reserved_273|>",
+    "<|reserved_274|>",
+    "<|reserved_275|>",
+    "<|reserved_276|>",
+    "<|reserved_277|>",
+    "<|reserved_278|>",
+    "<|reserved_279|>",
+    "<|reserved_280|>",
+    "<|reserved_281|>",
+    "<|reserved_282|>",
+    "<|reserved_283|>",
+    "<|reserved_284|>",
+    "<|reserved_285|>",
+    "<|reserved_286|>",
+    "<|reserved_287|>"
+  ],
+  "bos_token": "<|bos|>",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "<|eos|>",
+  "extra_special_tokens": {},
+  "legacy": false,
+  "mask_token": "<|mask|>",
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<|pad|>",
+  "padding_side": "right",
+  "tokenizer_class": "PreTrainedTokenizerFast",
+  "truncation_side": "right",
+  "trust_remote_code": false,
+  "unk_token": "<|unk|>",
+  "use_fast": true
+}