jh

tokenizer_config.json

@@ -1,17 +1,13 @@
-    
-{
-    "auto_map": {
-      "AutoTokenizer": [
-        "tokenizeConfig.OBITokenizer",
-        null
-      ]
-    },
-    "bos_token": "<s>",
-    "clean_up_tokenization_spaces": false,
-    "eos_token": "</s>",
-    "model_max_length": 1000000000000000019884624838656,
-    "pad_token": "</s>",
+
+  {
+    "pat_str": "'s|'t|'re|'ve|'m|'ll|'d| ?[\\p{L}]+| ?[\\p{N}]+| ?[^\\s\\p{L}\\p{N}]+|\\s+(?!\\S)|\\s+",
+
     "tokenizer_class": "OBITokenizer",
-    "unk_token": "<unk>"
-  }
-    
+    "auto_map": {
+        "AutoTokenizer": [
+          "tokenizeConfig.OBITokenizer",
+          null
+        ]
+      },
+    "vocab_size": 600
+}