Upload tokenizer

tokenizer.py

@@ -137,7 +137,9 @@ class STLTokenizer(PreTrainedTokenizer):
         Returns:
             List[int]: A list of corresponding token IDs.
         """
-        return [self.vocab.get(token, self.vocab[self.unk_token]) for token in tokens]
+        unk_token_str = str(self.unk_token)
+        unk_token_id = self.vocab.get(unk_token_str)
+        return [self.vocab.get(token, unk_token_id) for token in tokens]
 
     def convert_ids_to_tokens(self, ids: List[int]) -> List[str]:
         """

tokenizer_config.json

@@ -33,6 +33,12 @@
       "special": true
     }
   },
+  "auto_map": {
+    "AutoTokenizer": [
+      "tokenizer.STLTokenizer",
+      null
+    ]
+  },
   "bos_token": "/s",
   "clean_up_tokenization_spaces": false,
   "eos_token": "s",