Upload SMILES tokenizer

gpt2_tokenizer.py

@@ -0,0 +1,25 @@
+"""GPT2 Tokenizer that loads our custom SMILES tokenizer."""
+
+import os
+import json
+import torch
+from transformers import PreTrainedTokenizer
+
+from .smiles_tokenizer import SmilesTokenizer, SmilesVocabulary
+
+class GPT2Tokenizer(PreTrainedTokenizer):
+    """
+    GPT2Tokenizer wrapper for our SMILES tokenizer.
+    This class exists only to make AutoTokenizer find our tokenizer.
+    """
+
+    def __init__(self, **kwargs):
+        from .tokenizer_class import HFSmilesTokenizer
+        self.tokenizer = HFSmilesTokenizer(**kwargs)
+        super().__init__(**kwargs)
+
+    def __getattr__(self, name):
+        return getattr(self.tokenizer, name)
+
+    def __call__(self, *args, **kwargs):
+        return self.tokenizer(*args, **kwargs)

tokenizer_config.json

@@ -1,6 +1,6 @@
 {
   "model_type": "gpt2",
-  "tokenizer_class": "HFSmilesTokenizer",
+  "tokenizer_class": "GPT2Tokenizer",
   "bos_token": "<go>",
   "eos_token": "</s>",
   "unk_token": "<unk>",