Update app.py

app.py

@@ -23,7 +23,6 @@ def decode_byte_token(token):
     except Exception:
         return token_clean  # fallback
 
-# Token visualization function
 def visualize_tokens(text, tokenizer_name, show_token_ids):
     tokenizer = tokenizers[tokenizer_name]
     encoded = tokenizer(text, add_special_tokens=False, return_tensors=None)
@@ -43,18 +42,27 @@ def visualize_tokens(text, tokenizer_name, show_token_ids):
 
     html_tokens = []
     for token in tokens:
-        visible_token = token
-
-        # Decode clean UTF-8 version for hover
-        if tokenizer_name.startswith("Tabularis"):
-            decoded = decode_byte_token(token)
-        else:
-            decoded = token
-
+        prefix = ""
+        token_body = token
+
+        # For your byte-level tokenizers: separate Ġ from body
+        if tokenizer_name.startswith("Tabularis") and token.startswith("Ġ"):
+            prefix = "Ġ"
+            token_body = token[1:]
+
+        # Try to decode only the body (not Ġ)
+        try:
+            byte_seq = bytes([ord(c) for c in token_body])
+            decoded = byte_seq.decode("utf-8")
+        except Exception:
+            decoded = token_body  # fallback
+
+        # Display = prefix (e.g. Ġ) + decoded characters
+        label = f"{prefix}{decoded}"
         color = "lightgray" if is_special_token(token) else random_pastel()
 
         html_token = f"""
-        <span title="{decoded}" style='
+        <span title="{token}" style='
             display:inline-block;
             margin:4px;
             padding:8px 12px;
@@ -63,7 +71,7 @@ def visualize_tokens(text, tokenizer_name, show_token_ids):
             font-size:18px;
             font-family:monospace;
             font-weight:bold;
-        '>{visible_token}</span>
+        '>{label}</span>
         """
         html_tokens.append(html_token)
 
@@ -80,6 +88,7 @@ def visualize_tokens(text, tokenizer_name, show_token_ids):
 
     return html_output, f"🔢 Token Count: {len(tokens)}", decoded_output
 
+
 # Gradio app
 with gr.Blocks() as app:
     gr.Markdown("# 🚀 German Tokenizers")