Update app.py

app.py

@@ -1,5 +1,6 @@
 import random
 import gradio as gr
+import unicodedata
 from transformers import AutoTokenizer, PreTrainedTokenizerFast
 
 # Load all tokenizers at startup
@@ -14,15 +15,16 @@ tokenizers = {
     "DeepSeek Tokenizer": AutoTokenizer.from_pretrained("deepseek-ai/DeepSeek-R1")
 }
 
-# Decode byte-level tokens back to UTF-8 (for Tabularis only)
+#decode byte-level tokens back to UTF-8, normalized (fixes ß, ä, ö, ü)
 def decode_byte_token(token):
     token_clean = token.replace("Ġ", "")
     try:
         byte_seq = bytes([ord(c) for c in token_clean])
-        return byte_seq.decode("utf-8")
+        return unicodedata.normalize("NFC", byte_seq.decode("utf-8"))
     except Exception:
         return token_clean  # fallback
 
+
 def visualize_tokens(text, tokenizer_name, show_token_ids):
     tokenizer = tokenizers[tokenizer_name]
     encoded = tokenizer(text, add_special_tokens=False, return_tensors=None)
@@ -45,19 +47,19 @@ def visualize_tokens(text, tokenizer_name, show_token_ids):
         prefix = ""
         token_body = token
 
-        # For your byte-level tokenizers: separate Ġ from body
+        # Handle byte-level space marker
         if tokenizer_name.startswith("Tabularis") and token.startswith("Ġ"):
             prefix = "Ġ"
             token_body = token[1:]
 
-        # Try to decode only the body (not Ġ)
+        # Decode the token body only (not Ġ), with normalization
         try:
             byte_seq = bytes([ord(c) for c in token_body])
-            decoded = byte_seq.decode("utf-8")
+            decoded = unicodedata.normalize("NFC", byte_seq.decode("utf-8"))
         except Exception:
             decoded = token_body  # fallback
 
-        # Display = prefix (e.g. Ġ) + decoded characters
+        # Combine prefix with decoded token
         label = f"{prefix}{decoded}"
         color = "lightgray" if is_special_token(token) else random_pastel()
 
@@ -80,7 +82,7 @@ def visualize_tokens(text, tokenizer_name, show_token_ids):
     if show_token_ids:
         html_output += "<br><br><b>Token IDs:</b><br>" + str(token_ids)
 
-    # Decode full sentence
+    #decode full sentence
     try:
         decoded_output = tokenizer.decode(token_ids, skip_special_tokens=True)
     except Exception:
@@ -88,8 +90,6 @@ def visualize_tokens(text, tokenizer_name, show_token_ids):
 
     return html_output, f"🔢 Token Count: {len(tokens)}", decoded_output
 
-
-# Gradio app
 with gr.Blocks() as app:
     gr.Markdown("# 🚀 German Tokenizers")
 
@@ -110,5 +110,4 @@ with gr.Blocks() as app:
         outputs=[html_output, token_count, decoded_output]
     )
 
-# Launch the app
 app.launch()