Sync LoRA adapter: tokenizer.json

tokenizer.json

@@ -1,6 +1,11 @@
 {
   "version": "1.0",
-  "truncation": null,
+  "truncation": {
+    "direction": "Right",
+    "max_length": 512,
+    "strategy": "LongestFirst",
+    "stride": 0
+  },
   "padding": null,
   "added_tokens": [
     {
@@ -159,21 +164,37 @@
   ],
   "normalizer": null,
   "pre_tokenizer": {
-    "type": "Sequence",
-    "pretokenizers": [
+    "type": "ByteLevel",
+    "add_prefix_space": false,
+    "trim_offsets": true,
+    "use_regex": true
+  },
+  "post_processor": {
+    "type": "TemplateProcessing",
+    "single": [
+      {
+        "Sequence": {
+          "id": "A",
+          "type_id": 0
+        }
+      }
+    ],
+    "pair": [
       {
-        "type": "Digits",
-        "individual_digits": true
+        "Sequence": {
+          "id": "A",
+          "type_id": 0
+        }
       },
       {
-        "type": "ByteLevel",
-        "add_prefix_space": false,
-        "trim_offsets": true,
-        "use_regex": true
+        "Sequence": {
+          "id": "B",
+          "type_id": 1
+        }
       }
-    ]
+    ],
+    "special_tokens": {}
   },
-  "post_processor": null,
   "decoder": {
     "type": "ByteLevel",
     "add_prefix_space": true,
@@ -184,8 +205,8 @@
     "type": "BPE",
     "dropout": null,
     "unk_token": null,
-    "continuing_subword_prefix": null,
-    "end_of_word_suffix": null,
+    "continuing_subword_prefix": "",
+    "end_of_word_suffix": "",
     "fuse_unk": false,
     "byte_fallback": false,
     "ignore_merges": false,