Upload 3 files

config.json

@@ -1,5 +1,4 @@
 {
-  "_name_or_path": "BEREL_PyTorch_Model",
   "architectures": [
     "BertForMaskedLM"
   ],
@@ -19,7 +18,7 @@
   "pad_token_id": 0,
   "position_embedding_type": "absolute",
   "torch_dtype": "float32",
-  "transformers_version": "4.28.1",
+  "transformers_version": "4.12.0.dev0",
   "type_vocab_size": 2,
   "use_cache": true,
   "vocab_size": 128000

rabtokenizer.py

@@ -0,0 +1,31 @@
+from transformers import BertTokenizer, BasicTokenizer
+from transformers.tokenization_utils import _is_punctuation
+
+class OurBasicTokenizer(BasicTokenizer):
+    def _run_split_on_punc(self, text, never_split=None):
+        """Splits punctuation on a piece of text."""
+        if text in self.never_split or (never_split and text in never_split):
+            return [text]
+        chars = list(text)
+        i = 0
+        start_new_word = True
+        output = []
+        while i < len(chars):
+            char = chars[i]
+            if _is_punctuation(char) and char != "'" and not (char == '"' and i + 1 < len(chars) and not _is_punctuation(chars[i + 1])):
+                output.append([char])
+                start_new_word = True
+            else:
+                if start_new_word:
+                    output.append([])
+                start_new_word = False
+                output[-1].append(char)
+            i += 1
+
+        return ["".join(x) for x in output]
+
+
+def RabbinicTokenizer(tok):
+    tok.basic_tokenizer = OurBasicTokenizer(tok.basic_tokenizer.do_lower_case, tok.basic_tokenizer.never_split)
+    return tok
+