Update app.py

app.py

@@ -6,58 +6,21 @@ def formatarr(input):
 
 def tokenize(input_text):
     llama_tokens = llama_tokenizer(input_text, add_special_tokens=True)["input_ids"]
-
-    llama3_tokens = len(
-        llama3_tokenizer(input_text, add_special_tokens=True)["input_ids"]
-    )
-    mistral_tokens = len(
-        mistral_tokenizer(input_text, add_special_tokens=True)["input_ids"]
-    )
-    gpt2_tokens = len(
-        gpt2_tokenizer(input_text, add_special_tokens=True)["input_ids"]
-    )
-    gpt_neox_tokens = len(
-        gpt_neox_tokenizer(input_text, add_special_tokens=True)["input_ids"]
-    )
-    falcon_tokens = len(
-        falcon_tokenizer(input_text, add_special_tokens=True)["input_ids"]
-    )
-    phi2_tokens = len(
-        phi2_tokenizer(input_text, add_special_tokens=True)["input_ids"]
-    )
-    t5_tokens = len(
-        t5_tokenizer(input_text, add_special_tokens=True)["input_ids"]
-    )
-    gemma_tokens = len(
-        gemma_tokenizer(input_text, add_special_tokens=True)["input_ids"]
-    )
-    command_r_tokens = len(
-        command_r_tokenizer(input_text, add_special_tokens=True)["input_ids"]
-    )
-    qwen_tokens = len(
-        qwen_tokenizer(input_text, add_special_tokens=True)["input_ids"]
-    )
-    codeqwen_tokens = len(
-        codeqwen_tokenizer(input_text, add_special_tokens=True)["input_ids"]
-    )
+    llama3_tokens = llama3_tokenizer(input_text, add_special_tokens=True)["input_ids"]
+    mistral_tokens = mistral_tokenizer(input_text, add_special_tokens=True)["input_ids"]
+    gpt2_tokens = gpt2_tokenizer(input_text, add_special_tokens=True)["input_ids"]
+    gpt_neox_tokens = gpt_neox_tokenizer(input_text, add_special_tokens=True)["input_ids"]    
+    falcon_tokens = falcon_tokenizer(input_text, add_special_tokens=True)["input_ids"]    
+    phi2_tokens = phi2_tokenizer(input_text, add_special_tokens=True)["input_ids"]    
+    t5_tokens = t5_tokenizer(input_text, add_special_tokens=True)["input_ids"]    
+    gemma_tokens = gemma_tokenizer(input_text, add_special_tokens=True)["input_ids"]    
+    command_r_tokens = command_r_tokenizer(input_text, add_special_tokens=True)["input_ids"]    
+    qwen_tokens = qwen_tokenizer(input_text, add_special_tokens=True)["input_ids"]    
+    codeqwen_tokens = codeqwen_tokenizer(input_text, add_special_tokens=True)["input_ids"]
+    
 
     results = {
-        "LLaMa-1/LLaMa-2": len(llama_tokens),
-        "LLaMa-3": llama3_tokens,
-        "Mistral": mistral_tokens,
-        "GPT-2/GPT-J": gpt2_tokens,
-        "GPT-NeoX": gpt_neox_tokens,
-        "Falcon": falcon_tokens,
-        "Phi-1/Phi-2": phi2_tokens,
-        "T5": t5_tokens,
-        "Gemma": gemma_tokens,
-        "Command-R": command_r_tokens,
-        "Qwen/Qwen1.5": qwen_tokens,
-        "CodeQwen": codeqwen_tokens,
-    }
-
-    results2 = {
-        "LLaMa-1/LLaMa-2": formatarr(llama_tokens),
+        "LLaMa-1/LLaMa-2": llama_tokens,
         "LLaMa-3": llama3_tokens,
         "Mistral": mistral_tokens,
         "GPT-2/GPT-J": gpt2_tokens,
@@ -71,16 +34,10 @@ def tokenize(input_text):
         "CodeQwen": codeqwen_tokens,
     }
 
-    # Sort the results in descending order based on token length
-    sorted_results = sorted(results.items(), key=lambda x: x[1], reverse=True)
-
-    lens = "\n".join([f"{model}: {tokens}" for model, tokens in sorted_results])
-    toks = ""
-    
-    for model, tokens in results2.items():
-        toks += f"\n{model}: {tokens}" 
- 
-    return lens + "\n" + toks
+    toks = ""    
+    for model, tokens in results.items():
+        toks += f"\n{model}: {len(tokens)} = {formatarr(tokens)}"  
+    return toks
 
 
 if __name__ == "__main__":