commit files to HF hub

fasttext_jp_tokenizer.py

@@ -100,7 +100,10 @@ class FastTextJpTokenizer(MeCabTokenizer):
         Returns:
             int: ID
         """
-        return self.stoi[token]
+        id = self.stoi.get(token)
+        if id is not None:
+            return id
+        return self.stoi[self.unk_token]
 
     def _convert_id_to_token(self, index: int) -> str:
         """IDから単語