Upload 2 files

app.py

@@ -0,0 +1,58 @@
+import gradio as gr
+from transformers import AutoTokenizer
+
+def inspect_tokenizer(hf_token, model_name):
+    try:
+        tok = AutoTokenizer.from_pretrained(
+            model_name,
+            token=hf_token if hf_token else None
+        )
+
+        info = []
+        info.append(f"pad: {repr(tok.pad_token)} {tok.pad_token_id}")
+        info.append(f"eos: {repr(tok.eos_token)} {tok.eos_token_id}")
+        info.append(f"bos: {repr(tok.bos_token)} {tok.bos_token_id}")
+
+        test = [
+            {"role": "user", "content": "hello"},
+            {"role": "assistant", "content": "hi there"}
+        ]
+
+        template = tok.apply_chat_template(
+            test,
+            tokenize=False,
+            add_generation_prompt=False
+        )
+
+        return "\n".join(info) + "\n\nChat template:\n" + repr(template)
+
+    except Exception as e:
+        return f"Error: {str(e)}"
+
+
+with gr.Blocks() as demo:
+    gr.Markdown("## Tokenizer Inspector")
+
+    hf_token = gr.Textbox(
+        label="HF Token (optional)",
+        placeholder="Enter your Hugging Face token if needed",
+        type="password"
+    )
+
+    model_name = gr.Textbox(
+        label="Model Name",
+        value="Qwen/Qwen3-1.7B",
+        placeholder="e.g. Qwen/Qwen3-1.7B"
+    )
+
+    run_btn = gr.Button("Inspect")
+
+    output = gr.Textbox(label="Output", lines=15)
+
+    run_btn.click(
+        fn=inspect_tokenizer,
+        inputs=[hf_token, model_name],
+        outputs=output
+    )
+
+demo.launch()

requirements

@@ -0,0 +1,4 @@
+transformers
+huggingface_hub
+sentencepiece
+gradio