Update app.py

app.py

@@ -14,28 +14,22 @@ tokenizers = {
     "DeepSeek Tokenizer": AutoTokenizer.from_pretrained("deepseek-ai/DeepSeek-R1")
 }
 
-# Decode byte-level tokens back to UTF-8 for visual clarity (only for Tabularis models)
+# Decode byte-level tokens back to UTF-8 (for Tabularis only)
 def decode_byte_token(token):
     token_clean = token.replace("Ġ", "")
     try:
         byte_seq = bytes([ord(c) for c in token_clean])
         return byte_seq.decode("utf-8")
     except Exception:
-        return token  # fallback
+        return token_clean  # fallback
 
-# Token visualization
+# Token visualization function
 def visualize_tokens(text, tokenizer_name, show_token_ids):
     tokenizer = tokenizers[tokenizer_name]
     encoded = tokenizer(text, add_special_tokens=False, return_tensors=None)
     token_ids = encoded["input_ids"]
     tokens = tokenizer.convert_ids_to_tokens(token_ids)
 
-    # Decode byte-level tokens for your custom tokenizer
-    if tokenizer_name.startswith("Tabularis"):
-        readable_tokens = [decode_byte_token(t) for t in tokens]
-    else:
-        readable_tokens = tokens
-
     def random_pastel():
         r = lambda: random.randint(100, 255)
         return f"rgb({r()},{r()},{r()})"
@@ -48,10 +42,19 @@ def visualize_tokens(text, tokenizer_name, show_token_ids):
         )
 
     html_tokens = []
-    for token in readable_tokens:
+    for token in tokens:
+        visible_token = token
+
+        # Decode clean UTF-8 version for hover
+        if tokenizer_name.startswith("Tabularis"):
+            decoded = decode_byte_token(token)
+        else:
+            decoded = token
+
         color = "lightgray" if is_special_token(token) else random_pastel()
+
         html_token = f"""
-        <span style='
+        <span title="{decoded}" style='
             display:inline-block;
             margin:4px;
             padding:8px 12px;
@@ -60,7 +63,7 @@ def visualize_tokens(text, tokenizer_name, show_token_ids):
             font-size:18px;
             font-family:monospace;
             font-weight:bold;
-        '>{token}</span>
+        '>{visible_token}</span>
         """
         html_tokens.append(html_token)
 
@@ -69,7 +72,7 @@ def visualize_tokens(text, tokenizer_name, show_token_ids):
     if show_token_ids:
         html_output += "<br><br><b>Token IDs:</b><br>" + str(token_ids)
 
-    # Show decoded output using tokenizer.decode
+    # Decode full sentence
     try:
         decoded_output = tokenizer.decode(token_ids, skip_special_tokens=True)
     except Exception:
@@ -79,7 +82,7 @@ def visualize_tokens(text, tokenizer_name, show_token_ids):
 
 # Gradio app
 with gr.Blocks() as app:
-    gr.Markdown("# 🚀 German Tokenizers ")
+    gr.Markdown("# 🚀 German Tokenizers")
 
     with gr.Row():
         with gr.Column():