Pawan19012006's picture
Upload app.py with huggingface_hub
dd452b7 verified
Raw History Blame Contribute Delete
4.18 kB
import gradio as gr
from html import escape
import spaces
from tokenizer import tokenize
CONTEXT_WINDOW = 4096
def create_token_visualization(token_data):
html = """
<div style="
display: flex;
flex-wrap: wrap;
gap: 10px;
width: 100%;
box-sizing: border-box;
">
"""
for item in token_data:
token = item["token"]
token_id = item["token_id"]
# Preserve spaces when displaying tokens in HTML
display_token = escape(token).replace(" ", "&nbsp;")
html += f"""
<div style="
border: 1px solid #555;
border-radius: 8px;
padding: 12px 16px;
min-width: 100px;
text-align: center;
box-sizing: border-box;
">
<div style="
font-size: 16px;
font-weight: bold;
margin-bottom: 8px;
">
{display_token}
</div>
<div style="
font-size: 13px;
opacity: 0.8;
">
ID: {token_id}
</div>
</div>
"""
html += "</div>"
return html
def create_context_visualization(token_count):
remaining = CONTEXT_WINDOW - token_count
# Context window is a fixed limit for each inference request.
# The tokens included in that request are checked against this limit.
usage_percentage = (token_count / CONTEXT_WINDOW) * 100
if remaining < 0:
status = "⚠️ Context window exceeded"
else:
status = "✅ Within context window"
return f"""
<div style="
padding: 15px;
border: 1px solid #555;
border-radius: 10px;
">
<h3>Context Window</h3>
<p>
<strong>Limit:</strong> {CONTEXT_WINDOW} tokens
</p>
<p>
<strong>Used:</strong> {token_count} tokens
</p>
<p>
<strong>Remaining:</strong> {max(remaining, 0)} tokens
</p>
<p>
<strong>Usage:</strong> {usage_percentage:.2f}%
</p>
<p>{status}</p>
</div>
"""
@spaces.GPU
def process_text(text):
# Treat empty or whitespace-only input as empty
if not text.strip():
return (
"",
0,
"""
<div style="padding: 15px;">
<h3>Context Window</h3>
<p>No text entered.</p>
</div>
"""
)
token_data = []
token_ids, tokens, token_count = tokenize(text)
for token_id, token in zip(token_ids, tokens):
token_data.append({
"token": token,
"token_id": token_id
})
token_visualization = create_token_visualization(token_data)
context_visualization = create_context_visualization(token_count)
return (
token_visualization,
token_count,
context_visualization
)
with gr.Blocks(title="Tokenizer Visualizer") as app:
gr.Markdown(
"""
# Tokenizer Visualizer
Understand how text is converted into tokens and token IDs.
"""
)
with gr.Row():
# LEFT SIDE — INPUT
with gr.Column():
text_input = gr.Textbox(
label="Enter text",
placeholder="Type some text here...",
lines=5
)
tokenize_button = gr.Button(
"Tokenize",
variant="primary"
)
# RIGHT SIDE — OUTPUT
with gr.Column():
token_visualization = gr.HTML(
label="Token Visualization"
)
token_count = gr.Number(
label="Token Count",
value=0,
interactive=False
)
context_visualization = gr.HTML(
label="Context Window"
)
tokenize_button.click(
fn=process_text,
inputs=text_input,
outputs=[
token_visualization,
token_count,
context_visualization
]
)
app.launch()