Update app.py

app.py

@@ -1,26 +1,27 @@
 from transformers import AutoTokenizer
 import gradio as gr
 
+
 gpt2_tokenizer = AutoTokenizer.from_pretrained("gpt2")
 gptj_tokenizer = AutoTokenizer.from_pretrained("EleutherAI/gpt-j-6b")
 gpt_neox_tokenizer = AutoTokenizer.from_pretrained("EleutherAI/gpt-neox-20b")
 llama_tokenizer = AutoTokenizer.from_pretrained("hf-internal-testing/llama-tokenizer")
 
-tokenizers = {
-    "GPT-2": gpt2_tokenizer,
-    "GPT-J": gptj_tokenizer,
-    "GPT-NeoX": gpt_neox_tokenizer,
-    "LLaMa": llama_tokenizer
-}
+def tokenize(input_text):
+    gpt2_tokens = gpt2_tokenizer(input_text)["input_ids"]
+    gptj_tokens = gptj_tokenizer(input_text)["input_ids"]
+    gpt_neox_tokens = gpt_neox_tokenizer(input_text)["input_ids"]
+    llama_tokens = llama_tokenizer(input_text)["input_ids"]
 
-def tokenize(input_text, tokenizer_name):
-    tokenizer = tokenizers[tokenizer_name]
-    tokens = tokenizer(input_text)["input_ids"]
-    return f"Number of tokens for {tokenizer_name}: {len(tokens)}"
+    
+    
+    return f"""Number of tokens.
 
-textbox_input = gr.inputs.Textbox(lines=7)
-dropdown_tokenizer = gr.inputs.Dropdown(choices=list(tokenizers.keys()), default="LLaMa")
-output_text = "text"
+    GPT-2: {len(gpt2_tokens)}
+    GPT-J: {len(gptj_tokens)}
+    GPT-NeoX: {len(gpt_neox_tokens)}
+    LLaMa: {len(llama_tokens)}
+    """
 
-iface = gr.Interface(fn=tokenize, inputs=[textbox_input, dropdown_tokenizer], outputs=output_text)
-iface.launch()
+iface = gr.Interface(fn=tokenize, inputs=gr.inputs.Textbox(lines=7), outputs="text")
+iface.launch()