import gradio as gr from html import escape import spaces from tokenizer import tokenize CONTEXT_WINDOW = 4096 def create_token_visualization(token_data): html = """
""" for item in token_data: token = item["token"] token_id = item["token_id"] # Preserve spaces when displaying tokens in HTML display_token = escape(token).replace(" ", " ") html += f"""
{display_token}
ID: {token_id}
""" html += "
" return html def create_context_visualization(token_count): remaining = CONTEXT_WINDOW - token_count # Context window is a fixed limit for each inference request. # The tokens included in that request are checked against this limit. usage_percentage = (token_count / CONTEXT_WINDOW) * 100 if remaining < 0: status = "⚠️ Context window exceeded" else: status = "✅ Within context window" return f"""

Context Window

Limit: {CONTEXT_WINDOW} tokens

Used: {token_count} tokens

Remaining: {max(remaining, 0)} tokens

Usage: {usage_percentage:.2f}%

{status}

""" @spaces.GPU def process_text(text): # Treat empty or whitespace-only input as empty if not text.strip(): return ( "", 0, """

Context Window

No text entered.

""" ) token_data = [] token_ids, tokens, token_count = tokenize(text) for token_id, token in zip(token_ids, tokens): token_data.append({ "token": token, "token_id": token_id }) token_visualization = create_token_visualization(token_data) context_visualization = create_context_visualization(token_count) return ( token_visualization, token_count, context_visualization ) with gr.Blocks(title="Tokenizer Visualizer") as app: gr.Markdown( """ # Tokenizer Visualizer Understand how text is converted into tokens and token IDs. """ ) with gr.Row(): # LEFT SIDE — INPUT with gr.Column(): text_input = gr.Textbox( label="Enter text", placeholder="Type some text here...", lines=5 ) tokenize_button = gr.Button( "Tokenize", variant="primary" ) # RIGHT SIDE — OUTPUT with gr.Column(): token_visualization = gr.HTML( label="Token Visualization" ) token_count = gr.Number( label="Token Count", value=0, interactive=False ) context_visualization = gr.HTML( label="Context Window" ) tokenize_button.click( fn=process_text, inputs=text_input, outputs=[ token_visualization, token_count, context_visualization ] ) app.launch()