davda54
/

wiki-retrieval-patch-base

Model card Files Files and versions

davda54 commited on Nov 22, 2023

Commit

e7828e2

·

1 Parent(s): 13f1ca3

Upload tokenizer.json

Files changed (1) hide show

tokenizer.json +2 -2

tokenizer.json CHANGED Viewed

@@ -55,7 +55,7 @@
     "pretokenizers": [
       {
         "type": "ByteLevel",
-        "add_prefix_space": true,
         "trim_offsets": true,
         "use_regex": true
       },
@@ -142,7 +142,7 @@
   },
   "decoder": {
     "type": "ByteLevel",
-    "add_prefix_space": true,
     "trim_offsets": true,
     "use_regex": true
   },

     "pretokenizers": [
       {
         "type": "ByteLevel",
+        "add_prefix_space": false,
         "trim_offsets": true,
         "use_regex": true
       },
   },
   "decoder": {
     "type": "ByteLevel",
+    "add_prefix_space": false,
     "trim_offsets": true,
     "use_regex": true
   },