Update tokenizer list

app.py

@@ -9,21 +9,26 @@ from datasets import load_dataset
 from PIL import Image
 from transformers import AutoTokenizer
 
-tokenizers = [
-    "google/gemma-7b",
-    "meta-llama/Llama-2-7b",
-    "mistralai/Mistral-7B-v0.1",
-    "facebook/opt-2.7b",
-    "microsoft/phi-2",
-    "THUDM/chatglm3-6b",
-    "Qwen/Qwen1.5-7B-Chat",
-    "bigscience/bloom-560m",
-    "ise-uiuc/Magicoder-S-DS-6.7B",
-    "google/flan-t5-base",
-    "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
-    "google-bert/bert-base-uncased"
-]
+tokenizers = {
+    "bert": "google-bert/bert-base-uncased",
+    "bloom": "bigscience/bloom-560m",
+    "gemma": "fxmarty/tiny-random-GemmaForCausalLM",
+    "chatglm3": "THUDM/chatglm3-6b",
+    "falcon": "tiiuae/falcon-7b",
+    "gpt-neox": "EleutherAI/gpt-neox-20b",
+    "llama": "TinyLlama/TinyLlama-1.1B-Chat-v0.6",
+    "magicoder": "ise-uiuc/Magicoder-S-DS-6.7B",
+    "mistral": "mistralai/Mistral-7B-v0.1",
+    "opt": "facebook/opt-2.7b",
+    "phi-2": "microsoft/phi-2",
+    "pythia": "EleutherAI/pythia-1.4b-deduped",
+    "roberta": "FacebookAI/roberta-base",
+    "qwen": "Qwen/Qwen1.5-7B-Chat",
+    "starcoder": "bigcode/starcoder2-7b",
+    "t5": "google-t5/t5-base",
+}
 
+tokenizers = list(tokenizers.values())
 
 def plot_histogram(data):
     plt.hist(data)
@@ -86,11 +91,12 @@ demo = gr.Interface(
     examples=[
         ["mistralai/Mistral-7B-v0.1", "gsarti/flores_101", "eng", "dev", "sentence"],
         ["mistralai/Mistral-7B-v0.1", "Muennighoff/flores200", "eng_Latn", "dev", "sentence"],
-        ["mistralai/Mistral-7B-v0.1", "wikitext", "wikitext-2-v1", "validation", "text"],
         ["mistralai/Mistral-7B-v0.1", "hails/mmlu_no_train", "elementary_mathematics", "test", "question"],
-        ["mistralai/Mistral-7B-v0.1", "imdb", "", "test", "text"],
         ["mistralai/Mistral-7B-v0.1", "gsm8k", "main", "test", "question"],
         ["mistralai/Mistral-7B-v0.1", "locuslab/TOFU", "world_facts", "train", "question"],
+        ["mistralai/Mistral-7B-v0.1", "imdb", "", "test", "text"],
+        ["mistralai/Mistral-7B-v0.1", "wikitext", "wikitext-2-v1", "validation", "text"],
+        ["mistralai/Mistral-7B-v0.1", "zeroshot/twitter-financial-news-sentiment", "", "validation", "text"],
     ],
     cache_examples=True
 )