Create app.py

app.py

@@ -0,0 +1,49 @@
+import gradio as gr
+from transformers import AutoTokenizer
+import random
+import colorsys
+import html
+
+def get_distinct_colors(n):
+    colors = []
+    for i in range(n):
+        h = i / n
+        s = 0.6
+        v = 0.7
+        r, g, b = colorsys.hsv_to_rgb(h, s, v)
+        color = "#{:02x}{:02x}{:02x}".format(int(r*255), int(g*255), int(b*255))
+        colors.append(color)
+    return colors
+
+def tokenize_text(hf_model_id, text):
+    try:
+        tokenizer = AutoTokenizer.from_pretrained(hf_model_id)
+        tokens = tokenizer.tokenize(text)
+        token_count = len(tokens)
+        colors = get_distinct_colors(token_count)
+        colored_tokens = []
+        for i, token in enumerate(tokens):
+            display_token = token.replace('Ġ', '<space>')
+            display_token = html.escape(display_token)
+            colored_tokens.append(f'<span style="background-color: {colors[i]}; color: white; padding: 2px 4px; border-radius: 3px; margin: 2px; display: inline-block;">{display_token}</span>')
+        tokenized_text = "".join(colored_tokens)
+        return token_count, tokenized_text
+    except Exception as e:
+        return f"Error: {str(e)}", ""
+
+demo = gr.Interface(
+    fn=tokenize_text,
+    inputs=[
+        gr.Textbox(label="Hugging Face Model ID", placeholder="e.g., gpt2, bert-base-uncased", value="unsloth/gemma-3-27b-it"),
+        gr.Textbox(label="Text to Tokenize", lines=5, placeholder="Enter your text here...")
+    ],
+    outputs=[
+        gr.Number(label="Token Count"),
+        gr.HTML(label="Tokenized Text", container=True, show_label=True)
+    ],
+    title="HuggingFace Tokenizer",
+    description="Enter a HuggingFace model ID and text to see how it gets tokenized.",
+    allow_flagging="never"
+)
+
+demo.launch()