Update app.py

app.py

@@ -20,15 +20,21 @@ def visualize_tokens(text, tokenizer_name, show_token_ids):
     token_ids = encoded["input_ids"]
     tokens = tokenizer.convert_ids_to_tokens(token_ids)
 
-    # Generate a random pastel color
     def random_pastel():
         r = lambda: random.randint(100, 255)
         return f"rgb({r()},{r()},{r()})"
 
-    # Create HTML tokens with random colors and bigger size
+    # Helper to detect special tokens
+    def is_special_token(token):
+        return token.startswith('[') and token.endswith(']') or token.startswith('<') and token.endswith('>') or token in tokenizer.all_special_tokens
+
     html_tokens = []
     for idx, token in enumerate(tokens):
-        color = random_pastel()
+        if is_special_token(token):
+            color = "lightgray"  # special token color
+        else:
+            color = random_pastel()
+
         html_token = f"""
         <span style='
             display:inline-block;
@@ -50,6 +56,7 @@ def visualize_tokens(text, tokenizer_name, show_token_ids):
 
     return html_output, f"🔢 Token Count: {len(tokens)}"
 
+
 # Gradio app
 with gr.Blocks() as app:
     gr.Markdown("# 🚀 Tokenizer Playground (Tiktokenizer-Style)")