Improvements

app.py

@@ -576,7 +576,13 @@ with gr.Blocks(
                             "tokenmonster",
                             "byt5",
                         ],
-                        value=["gpt-4", "llama-3", "gpt-2"],
+                        value=[
+                            "gpt-4",
+                            "llama-3",
+                            "gemma-2",
+                            "qwen2.5",
+                            "tokenmonster",
+                        ],
                         label="Select tokenizers to compare",
                     )
                     show_details = gr.Checkbox(
@@ -679,7 +685,7 @@ with gr.Blocks(
             )
 
             # Combine or show separately
-            combined_html = f"<h3>Normalized Text: {normalized_text}</h3>{norm_html}\n<h2>Original</h2>{orig_html}"
+            combined_html = f"<h3>Normalized ({norm_method}) Text: {normalized_text} </h3>{norm_html}\n<h2>Original</h2>{orig_html}"
 
             return (
                 orig_eff,

utils.py

@@ -110,9 +110,9 @@ def tokenize_with_tiktoken(text, model):
 
     return {
         "model": TOKENIZER_INFO[model]["name"],
-        "token_count": len(tokens),
+        "token_count": len(token_data),
         "tokens": token_data,
-        "compression_ratio": len(text) / len(tokens) if tokens else 0,
+        "compression_ratio": len(text) / len(token_data) if token_data else 0,
         "encoding": TOKENIZER_INFO[model]["encoding"],
         "vocab_size": TOKENIZER_INFO[model]["vocab_size"],
     }
@@ -187,9 +187,9 @@ def tokenize_with_hf(text, model):
 
         return {
             "model": TOKENIZER_INFO[model]["name"],
-            "token_count": len(token_ids),
+            "token_count": len(token_data),
             "tokens": token_data,
-            "compression_ratio": len(text) / len(token_ids) if token_ids else 0,
+            "compression_ratio": len(text) / len(token_data) if token_data else 0,
             "encoding": TOKENIZER_INFO[model]["encoding"],
             "vocab_size": TOKENIZER_INFO[model]["vocab_size"],
         }