Update tokenizer.py

tokenizer.py

@@ -251,6 +251,7 @@ class ParadigmTokenizerWrapper(PreTrainedTokenizerFast):
                   or os.path.dirname(getattr(self, "tokenizer_file", "")) or "."
         revision = kwargs.get("revision", None)
 
+        cfg = {"lowercase": True, "space_punct": True}
         ppath = _get_repo_file(repo_id_or_path, "paradigms.json", revision)
         self.paradigms, self.paradigms_meta = _load_paradigms_any(ppath)