Upload tokenizer

Files changed (2) hide show

tokenizer.json CHANGED Viewed

The diff for this file is too large to render. See raw diff

tokenizer_config.json CHANGED Viewed

@@ -315,7 +315,7 @@
     }
   },
   "bos_token": "<|endoftext|>",
-  "clean_up_tokenization_spaces": true,
   "eos_token": "<|endoftext|>",
   "model_max_length": 2048,
   "pad_token": "<|endoftext|>",

     }
   },
   "bos_token": "<|endoftext|>",
+  "clean_up_tokenization_spaces": false,
   "eos_token": "<|endoftext|>",
   "model_max_length": 2048,
   "pad_token": "<|endoftext|>",