Update app.py

app.py

@@ -3,17 +3,21 @@ import gradio as gr
 
 
 gpt2_tokenizer = AutoTokenizer.from_pretrained("gpt2")
-gptj_tokenizer = AutoTokenizer.from_pretrained("EleutherAI/gpt-j-6b")
 gpt_neox_tokenizer = AutoTokenizer.from_pretrained("EleutherAI/gpt-neox-20b")
 llama_tokenizer = AutoTokenizer.from_pretrained("hf-internal-testing/llama-tokenizer")
+yi_tokenizer = AutoTokenizer.from_pretrained("01-ai/Yi-34B")
+yayi2_tokenizer = AutoTokenizer.from_pretrained("wenge-research/yayi2-30b")
+phi2_tokenizer = AutoTokenizer.from_pretrained("microsoft/phi-2")
 
 def tokenize(input_text):
     gpt2_tokens = gpt2_tokenizer(input_text)["input_ids"]
-    gptj_tokens = gptj_tokenizer(input_text)["input_ids"]
     gpt_neox_tokens = gpt_neox_tokenizer(input_text)["input_ids"]
     llama_tokens = llama_tokenizer(input_text)["input_ids"]
+    yi_tokens = llama_tokenizer(input_text)["input_ids"]
+    yayi2_tokens = yayi2_tokenizer(input_text)["input_ids"]
+    phi2_tokens = phi2_tokenizer(input_text)["input_ids"]
 
-    return f"GPT-2: {len(gpt2_tokens)}\nGPT-J: {len(gptj_tokens)}\nGPT-NeoX: {len(gpt_neox_tokens)}\nLLaMa: {len(llama_tokens)}"
+    return f"GPT-2/GPT-J: {len(gpt2_tokens)}\nGPT-NeoX: {len(gpt_neox_tokens)}\nLLaMa: {len(llama_tokens)}\nYi: {len(yi_tokens)}\nYayi-2: {len(yayi2_tokens)}\nPhi-2: {len(phi2_tokens)}"
 
 iface = gr.Interface(fn=tokenize, inputs=gr.Textbox(lines=7), outputs="text")
 iface.launch()