Add Qwen3-4B-Base tokenizer

app.py

@@ -37,7 +37,7 @@ def read_svg_file(name: str) -> str:
 def get_model_icon(name: str) -> str:
     """Get the HTML for the model icon."""
     # Skip icons for collapsed models
-    if name in ["google/gemma-3-27b-it", "mistralai/Mistral-Nemo-Instruct-2407", "CohereLabs/aya-expanse-8b"]:
+    if name in ["google/gemma-3-27b-it", "mistralai/Mistral-Nemo-Instruct-2407", "CohereLabs/aya-expanse-8b", "Qwen/Qwen3-4B-Base"]:
         return ""
         
     svg_content = read_svg_file(name)
@@ -92,7 +92,7 @@ with gr.Blocks(title="Tokens matter.", theme=theme, css="""
             with gr.Row():
                 with gr.Column():
                     for name in all_tokenizer_names[:2]:
-                        if name in ["google/gemma-3-27b-it", "mistralai/Mistral-Nemo-Instruct-2407", "CohereLabs/aya-expanse-8b"]:
+                        if name in ["google/gemma-3-27b-it", "mistralai/Mistral-Nemo-Instruct-2407", "CohereLabs/aya-expanse-8b", "Qwen/Qwen3-4B-Base"]:
                             continue
                         display_name = utils.MODEL_DISPLAY_NAMES.get(name, name)
                         with gr.Group(elem_classes="tokenizer-panel"):
@@ -101,7 +101,7 @@ with gr.Blocks(title="Tokens matter.", theme=theme, css="""
                             main_output_boxes.append(box)
                 with gr.Column():
                     for name in all_tokenizer_names[2:4]:
-                        if name in ["google/gemma-3-27b-it", "mistralai/Mistral-Nemo-Instruct-2407", "CohereLabs/aya-expanse-8b"]:
+                        if name in ["google/gemma-3-27b-it", "mistralai/Mistral-Nemo-Instruct-2407", "CohereLabs/aya-expanse-8b", "Qwen/Qwen3-4B-Base"]:
                             continue
                         display_name = utils.MODEL_DISPLAY_NAMES.get(name, name)
                         with gr.Group(elem_classes="tokenizer-panel"):
@@ -113,7 +113,7 @@ with gr.Blocks(title="Tokens matter.", theme=theme, css="""
             more_models = gr.Accordion("More Models", open=False, visible=False)
             with more_models:
                 for name in all_tokenizer_names:
-                    if name in ["google/gemma-3-27b-it", "mistralai/Mistral-Nemo-Instruct-2407", "CohereLabs/aya-expanse-8b"]:
+                    if name in ["google/gemma-3-27b-it", "mistralai/Mistral-Nemo-Instruct-2407", "CohereLabs/aya-expanse-8b", "Qwen/Qwen3-4B-Base"]:
                         display_name = utils.MODEL_DISPLAY_NAMES.get(name, name)
                         with gr.Group(elem_classes="tokenizer-panel"):
                             gr.HTML(value=f'<div style="display: flex; align-items: center; margin-bottom: 8px;">{get_model_icon(name)}<span class="model-name"  style="font-weight: bold;">{display_name}</span></div>')

tests/test_utils.py

@@ -19,7 +19,7 @@ class TestTokenizerLoading(unittest.TestCase):
         tokenizers = load_tokenizers()
         
         # Check that we have the expected number of tokenizers
-        expected_count = 6  # 5 HF + 1 OpenAI
+        expected_count = 8  # 7 HF + 1 OpenAI
         self.assertEqual(len(tokenizers), expected_count)
         
         # Check that all expected tokenizers are present
@@ -29,6 +29,8 @@ class TestTokenizerLoading(unittest.TestCase):
             "ZurichNLP/swissbert",
             "mistralai/Mistral-Nemo-Instruct-2407",
             "google/gemma-3-27b-it",
+            "CohereLabs/aya-expanse-8b",
+            "Qwen/Qwen3-4B-Base",
             "gpt-4o"
         }
         self.assertEqual(set(tokenizers.keys()), expected_names)

utils.py

@@ -96,6 +96,7 @@ def load_tokenizers() -> Dict[str, object]:
         "google/gemma-3-27b-it",
         "mistralai/Mistral-Nemo-Instruct-2407",
         "CohereLabs/aya-expanse-8b",
+        "Qwen/Qwen3-4B-Base",
     ]
     for name in hf_names:
         tokenizer_name, tokenizer = load_hf_tokenizer(name)
@@ -111,7 +112,8 @@ MODEL_DISPLAY_NAMES = {
     "mistralai/Mistral-Nemo-Instruct-2407": "Mistral NeMo",
     "google/gemma-3-27b-it": "Gemma 3",
     "gpt-4o": "ChatGPT (gpt-4o)",
-    "CohereLabs/aya-expanse-8b": "Aya Expanse"
+    "CohereLabs/aya-expanse-8b": "Aya Expanse",
+    "Qwen/Qwen3-4B-Base": "Qwen3 4B Base",
 }
 
 def tokenize(s: str, tokenizer) -> List[str]:
@@ -156,7 +158,7 @@ def _tokenize_with_tokenizer(s: str, tokenizer) -> List[str]:
     """
     if hasattr(tokenizer, "tokenize"):
         encoded = tokenizer.encode(s, add_special_tokens=False)
-        if hasattr(tokenizer, "name_or_path") and any(name in tokenizer.name_or_path.lower() for name in ["llama", "deepseek", "mistral", "aya"]):
+        if hasattr(tokenizer, "name_or_path") and any(name in tokenizer.name_or_path.lower() for name in ["llama", "deepseek", "mistral", "aya", "qwen"]):
             tokens = [tokenizer.decode([token_id], skip_special_tokens=False) for token_id in encoded]
         else:
             tokens = tokenizer.convert_ids_to_tokens(encoded)