Upload tokenizer

tokenizer.json

@@ -1,6 +1,11 @@
 {
   "version": "1.0",
-  "truncation": null,
+  "truncation": {
+    "direction": "Right",
+    "max_length": 1024,
+    "strategy": "LongestFirst",
+    "stride": 0
+  },
   "padding": null,
   "added_tokens": [
     {
@@ -139,85 +144,51 @@
       "special": false
     }
   ],
-  "normalizer": {
-    "type": "Sequence",
-    "normalizers": []
-  },
+  "normalizer": null,
   "pre_tokenizer": {
+    "type": "Metaspace",
+    "replacement": "▁",
+    "prepend_scheme": "always",
+    "split": false
+  },
+  "post_processor": {
+    "type": "ByteLevel",
+    "add_prefix_space": true,
+    "trim_offsets": false,
+    "use_regex": true
+  },
+  "decoder": {
     "type": "Sequence",
-    "pretokenizers": [
-      {
-        "type": "Split",
-        "pattern": {
-          "Regex": "[\r\n]"
-        },
-        "behavior": "Isolated",
-        "invert": false
-      },
-      {
-        "type": "Split",
-        "pattern": {
-          "Regex": "\\s?[A-Za-zµÀ-ÖØ-öø-ƺƼ-ƿDŽ-ʓʕ-ʯͰ-ͳͶͷͻ-ͽͿΆΈ-ΊΌΎ-ΡΣ-ϵϷ-ҁҊ-ԯԱ-ՖႠ-ჅᎠ-Ᏽᏸ-ᏽᲐ-ᲺᲽ-Ჿᴀ-ᴫᵫ-ᵷᵹ-ᶚḀ-ἕἘ-Ἕἠ-ὅὈ-Ὅὐ-ὗὙὛὝὟ-ώᾀ-ᾴᾶ-ᾼιῂ-ῄῆ-ῌῐ-ΐῖ-Ίῠ-Ῥῲ-ῴῶ-ῼℂℇℊ-ℓℕℙ-ℝℤΩℨK-ℭℯ-ℴℹℼ-ℿⅅ-ⅉⅎↃↄⰀ-ⱻⱾ-ⳤⳫ-ⳮⳲⳳꙀ-ꙭꚀ-ꚛꜢ-ꝯꝱ-ꞇꞋ-ꞎꭰ-ꮿff-stﬓ-ﬗＡ-Ｚａ-ｚ𐐀-𐑏𐒰-𐓓𐓘-𐓻𐲀-𐲲𐳀-𐳲𑢠-𑣟𞤀-𞥃]+"
-        },
-        "behavior": "Isolated",
-        "invert": false
-      },
+    "decoders": [
       {
-        "type": "Split",
+        "type": "Replace",
         "pattern": {
-          "Regex": "\\s?[!-/:-~！-／：-～‘-‟　-。]+"
+          "String": "▁"
         },
-        "behavior": "Isolated",
-        "invert": false
+        "content": " "
       },
       {
-        "type": "Split",
-        "pattern": {
-          "Regex": "\\s+$"
-        },
-        "behavior": "Isolated",
-        "invert": false
+        "type": "ByteFallback"
       },
       {
-        "type": "Split",
-        "pattern": {
-          "Regex": "[一-龥ࠀ-一가-퟿]+"
-        },
-        "behavior": "Isolated",
-        "invert": false
+        "type": "Fuse"
       },
       {
-        "type": "Digits",
-        "individual_digits": true
-      },
-      {
-        "type": "ByteLevel",
-        "add_prefix_space": false,
-        "trim_offsets": true,
-        "use_regex": false
+        "type": "Strip",
+        "content": " ",
+        "start": 1,
+        "stop": 0
       }
     ]
   },
-  "post_processor": {
-    "type": "ByteLevel",
-    "add_prefix_space": true,
-    "trim_offsets": false,
-    "use_regex": true
-  },
-  "decoder": {
-    "type": "ByteLevel",
-    "add_prefix_space": true,
-    "trim_offsets": true,
-    "use_regex": true
-  },
   "model": {
     "type": "BPE",
     "dropout": null,
     "unk_token": null,
     "continuing_subword_prefix": null,
     "end_of_word_suffix": null,
-    "fuse_unk": false,
-    "byte_fallback": false,
+    "fuse_unk": true,
+    "byte_fallback": true,
     "ignore_merges": false,
     "vocab": {
       "!": 0,

tokenizer_config.json

@@ -1,13 +1,14 @@
 {
+  "add_prefix_space": null,
   "backend": "tokenizers",
   "bos_token": "<｜begin▁of▁sentence｜>",
   "clean_up_tokenization_spaces": false,
   "eos_token": "<｜end▁of▁sentence｜>",
   "is_local": false,
-  "legacy": true,
   "model_max_length": 4096,
   "pad_token": "<｜end▁of▁sentence｜>",
   "sp_model_kwargs": {},
-  "tokenizer_class": "TokenizersBackend",
-  "unk_token": null
+  "tokenizer_class": "LlamaTokenizer",
+  "unk_token": null,
+  "use_default_system_prompt": false
 }