Adding other tokenizers

app.py

@@ -3,12 +3,20 @@ import random
 import gradio as gr
 from language_bpe import BPETokenizer
 
-tokenizer = BPETokenizer()
-tokenizer.load('models/english_5000.model')
+hinglish_tokenizer = BPETokenizer()
+hinglish_tokenizer.load('models/hinglish_5000.model')
 
-def inference(input_text):
-    encoding = tokenizer.encode_ordinary(input_text)
-    sentence = [tokenizer.decode([x]) for x in encoding]
+hindi_tokenizer = BPETokenizer()
+hindi_tokenizer.load('models/hindi_5000.model')
+
+english_tokenizer = BPETokenizer()
+english_tokenizer.load('models/english_5000.model')
+
+tokenizer_dict = {"Hinglish_5k": hinglish_tokenizer, "Hindi_5k": hindi_tokenizer, "English_5k": english_tokenizer}
+
+def inference(input_text, tokenizer):
+    encoding = tokenizer_dict[tokenizer].encode_ordinary(input_text)
+    sentence = [tokenizer_dict[tokenizer].decode([x]) for x in encoding]
     color_sentence = []
     color_encoding = []
     for word, encode in zip(sentence, encoding):
@@ -26,6 +34,7 @@ demo = gr.Interface(
     inference, 
     inputs = [
         gr.Textbox(label="Enter any sentence in Hindi, English or both language", type="text"),
+        gr.Dropdown(label="Tokenizer", choices=["Hinglish_5k", "Hindi_5k", "English_5k"], value="Hinglish_5k")
         ], 
     outputs = [
         gr.Label(label="Token count"),