First commit

BpeTokenizer.py

@@ -0,0 +1,13 @@
+from tokenizers import Tokenizer
+
+def bpe_tokenizer(smiles_string):
+    # Load the tokenizer from the saved file
+    tokenizer = Tokenizer.from_file("bpe_tokenizer.json")
+
+    # Tokenize the SMILES string
+    encoded_output = tokenizer.encode(smiles_string)
+
+    # To get the tokenized output as text
+    tokens_text = encoded_output.tokens
+    
+    return tokens_text

app.py

@@ -0,0 +1,19 @@
+import gradio as gr
+from BpeTokenizer import bpe_tokenizer
+
+# def tem(name, num = 3):
+#     return name + num
+
+
+# iface = gr.Interface(fn=tem, inputs=["text", "text"], outputs="text")
+
+iface = gr.Interface(
+    fn = bpe_tokenizer,
+    inputs=[
+        gr.Textbox(label="SMILES"),         
+    ],
+    outputs="text"
+)
+
+iface.launch()
+

requirements.txt

@@ -0,0 +1 @@
+tokenizers

trainBpeTokenizer.py

@@ -0,0 +1,33 @@
+from tokenizers import Tokenizer
+from tokenizers.models import BPE
+from tokenizers.trainers import BpeTrainer
+from tokenizers.pre_tokenizers import ByteLevel
+from tokenizers.processors import TemplateProcessing
+
+# Initialize a tokenizer
+tokenizer = Tokenizer(BPE(unk_token="[UNK]"))
+
+# Use the byte level pre-tokenizer
+tokenizer.pre_tokenizer = ByteLevel()
+
+# Customize training with a BpeTrainer
+trainer = BpeTrainer(special_tokens=["[UNK]", "[CLS]", "[SEP]", "[PAD]", "[MASK]"])
+
+# Path to the file(s) for training the tokenizer
+files = ["/home/saicharan/Downloads/chembl.csv"]
+
+# Train the tokenizer
+tokenizer.train(files, trainer)
+
+# Optionally, you can customize the post-processing to add special tokens
+tokenizer.post_processor = TemplateProcessing(
+    single="[CLS] $A [SEP]",
+    pair="[CLS] $A [SEP] $B:1 [SEP]:1",
+    special_tokens=[
+        ("[CLS]", tokenizer.token_to_id("[CLS]")),
+        ("[SEP]", tokenizer.token_to_id("[SEP]")),
+    ],
+)
+
+# Save the tokenizer
+tokenizer.save("/home/saicharan/Downloads/chembl_bpe_tokenizer.json")