app.py

import gradio as gr
from transformers import AutoTokenizer

def inspect_tokenizer(hf_token, model_name):
    try:
        tok = AutoTokenizer.from_pretrained(
            model_name,
            token=hf_token if hf_token else None
        )

        info = []
        info.append(f"pad: {repr(tok.pad_token)} {tok.pad_token_id}")
        info.append(f"eos: {repr(tok.eos_token)} {tok.eos_token_id}")
        info.append(f"bos: {repr(tok.bos_token)} {tok.bos_token_id}")

        test = [
            {"role": "user", "content": "hello"},
            {"role": "assistant", "content": "hi there"}
        ]

        template = tok.apply_chat_template(
            test,
            tokenize=False,
            add_generation_prompt=False
        )

        return "\n".join(info) + "\n\nChat template:\n" + repr(template)

    except Exception as e:
        return f"Error: {str(e)}"


with gr.Blocks() as demo:
    gr.Markdown("## Tokenizer Inspector")

    hf_token = gr.Textbox(
        label="HF Token (optional)",
        placeholder="Enter your Hugging Face token if needed",
        type="password"
    )

    model_name = gr.Textbox(
        label="Model Name",
        value="Qwen/Qwen3-1.7B",
        placeholder="e.g. Qwen/Qwen3-1.7B"
    )

    run_btn = gr.Button("Inspect")

    output = gr.Textbox(label="Output", lines=15)

    run_btn.click(
        fn=inspect_tokenizer,
        inputs=[hf_token, model_name],
        outputs=output
    )

demo.launch()