Update app.py

app.py

@@ -1,9 +1,8 @@
 import random
 import gradio as gr
-from transformers import AutoTokenizer
-from transformers import PreTrainedTokenizerFast
+from transformers import AutoTokenizer, PreTrainedTokenizerFast
 
-# Load all tokenizers (at startup)
+# Load all tokenizers at startup
 tokenizers = {
     "Tabularis German Tokenizer_whiteS": PreTrainedTokenizerFast(tokenizer_file="tokenizer.json"),
     "Tabularis German Tokenizer": PreTrainedTokenizerFast(tokenizer_file="tokenizer_BPE.json"),
@@ -15,28 +14,42 @@ tokenizers = {
     "DeepSeek Tokenizer": AutoTokenizer.from_pretrained("deepseek-ai/DeepSeek-R1")
 }
 
-# Fancy token visualization with random colors
+# Decode byte-level tokens back to UTF-8 for visual clarity (only for Tabularis models)
+def decode_byte_token(token):
+    token_clean = token.replace("Ġ", "")
+    try:
+        byte_seq = bytes([ord(c) for c in token_clean])
+        return byte_seq.decode("utf-8")
+    except Exception:
+        return token  # fallback
+
+# Token visualization
 def visualize_tokens(text, tokenizer_name, show_token_ids):
     tokenizer = tokenizers[tokenizer_name]
     encoded = tokenizer(text, add_special_tokens=False, return_tensors=None)
     token_ids = encoded["input_ids"]
     tokens = tokenizer.convert_ids_to_tokens(token_ids)
 
+    # Decode byte-level tokens for your custom tokenizer
+    if tokenizer_name.startswith("Tabularis"):
+        readable_tokens = [decode_byte_token(t) for t in tokens]
+    else:
+        readable_tokens = tokens
+
     def random_pastel():
         r = lambda: random.randint(100, 255)
         return f"rgb({r()},{r()},{r()})"
 
-    # Helper to detect special tokens
     def is_special_token(token):
-        return token.startswith('[') and token.endswith(']') or token.startswith('<') and token.endswith('>') or token in tokenizer.all_special_tokens
+        return (
+            token.startswith('[') and token.endswith(']')
+            or token.startswith('<') and token.endswith('>')
+            or token in tokenizer.all_special_tokens
+        )
 
     html_tokens = []
-    for idx, token in enumerate(tokens):
-        if is_special_token(token):
-            color = "lightgray"  # special token color
-        else:
-            color = random_pastel()
-
+    for token in readable_tokens:
+        color = "lightgray" if is_special_token(token) else random_pastel()
         html_token = f"""
         <span style='
             display:inline-block;
@@ -52,12 +65,17 @@ def visualize_tokens(text, tokenizer_name, show_token_ids):
         html_tokens.append(html_token)
 
     html_output = "".join(html_tokens)
-    
+
     if show_token_ids:
         html_output += "<br><br><b>Token IDs:</b><br>" + str(token_ids)
 
-    return html_output, f"🔢 Token Count: {len(tokens)}"
+    # Show decoded output using tokenizer.decode
+    try:
+        decoded_output = tokenizer.decode(token_ids, skip_special_tokens=True)
+    except Exception:
+        decoded_output = "[Could not decode using this tokenizer]"
 
+    return html_output, f"🔢 Token Count: {len(tokens)}", decoded_output
 
 # Gradio app
 with gr.Blocks() as app:
@@ -72,12 +90,13 @@ with gr.Blocks() as app:
         with gr.Column():
             html_output = gr.HTML(label="Tokens Visualized")
             token_count = gr.Label(label="Token Count")
+            decoded_output = gr.Textbox(label="Decoded Text", lines=3)
 
     tokenize_btn.click(
         visualize_tokens,
         inputs=[text_input, tokenizer_choice, show_ids],
-        outputs=[html_output, token_count]
+        outputs=[html_output, token_count, decoded_output]
     )
 
-# Launch
+# Launch the app
 app.launch()