Spaces:
Sleeping
Sleeping
Download app.py from Pawan19012006/tokenizer-visualizer: direct link, hf CLI and curl.
- Browser
- Download file 4.18 kB
-
https://huggingface.co/spaces/Pawan19012006/tokenizer-visualizer/resolve/main/app.py
- Command line
-
hf download hf://spaces/Pawan19012006/tokenizer-visualizer/app.py
-
curl -L -o app.py https://huggingface.co/spaces/Pawan19012006/tokenizer-visualizer/resolve/main/app.py
4.18 kB
| import gradio as gr | |
| from html import escape | |
| import spaces | |
| from tokenizer import tokenize | |
| CONTEXT_WINDOW = 4096 | |
| def create_token_visualization(token_data): | |
| html = """ | |
| <div style=" | |
| display: flex; | |
| flex-wrap: wrap; | |
| gap: 10px; | |
| width: 100%; | |
| box-sizing: border-box; | |
| "> | |
| """ | |
| for item in token_data: | |
| token = item["token"] | |
| token_id = item["token_id"] | |
| # Preserve spaces when displaying tokens in HTML | |
| display_token = escape(token).replace(" ", " ") | |
| html += f""" | |
| <div style=" | |
| border: 1px solid #555; | |
| border-radius: 8px; | |
| padding: 12px 16px; | |
| min-width: 100px; | |
| text-align: center; | |
| box-sizing: border-box; | |
| "> | |
| <div style=" | |
| font-size: 16px; | |
| font-weight: bold; | |
| margin-bottom: 8px; | |
| "> | |
| {display_token} | |
| </div> | |
| <div style=" | |
| font-size: 13px; | |
| opacity: 0.8; | |
| "> | |
| ID: {token_id} | |
| </div> | |
| </div> | |
| """ | |
| html += "</div>" | |
| return html | |
| def create_context_visualization(token_count): | |
| remaining = CONTEXT_WINDOW - token_count | |
| # Context window is a fixed limit for each inference request. | |
| # The tokens included in that request are checked against this limit. | |
| usage_percentage = (token_count / CONTEXT_WINDOW) * 100 | |
| if remaining < 0: | |
| status = "⚠️ Context window exceeded" | |
| else: | |
| status = "✅ Within context window" | |
| return f""" | |
| <div style=" | |
| padding: 15px; | |
| border: 1px solid #555; | |
| border-radius: 10px; | |
| "> | |
| <h3>Context Window</h3> | |
| <p> | |
| <strong>Limit:</strong> {CONTEXT_WINDOW} tokens | |
| </p> | |
| <p> | |
| <strong>Used:</strong> {token_count} tokens | |
| </p> | |
| <p> | |
| <strong>Remaining:</strong> {max(remaining, 0)} tokens | |
| </p> | |
| <p> | |
| <strong>Usage:</strong> {usage_percentage:.2f}% | |
| </p> | |
| <p>{status}</p> | |
| </div> | |
| """ | |
| def process_text(text): | |
| # Treat empty or whitespace-only input as empty | |
| if not text.strip(): | |
| return ( | |
| "", | |
| 0, | |
| """ | |
| <div style="padding: 15px;"> | |
| <h3>Context Window</h3> | |
| <p>No text entered.</p> | |
| </div> | |
| """ | |
| ) | |
| token_data = [] | |
| token_ids, tokens, token_count = tokenize(text) | |
| for token_id, token in zip(token_ids, tokens): | |
| token_data.append({ | |
| "token": token, | |
| "token_id": token_id | |
| }) | |
| token_visualization = create_token_visualization(token_data) | |
| context_visualization = create_context_visualization(token_count) | |
| return ( | |
| token_visualization, | |
| token_count, | |
| context_visualization | |
| ) | |
| with gr.Blocks(title="Tokenizer Visualizer") as app: | |
| gr.Markdown( | |
| """ | |
| # Tokenizer Visualizer | |
| Understand how text is converted into tokens and token IDs. | |
| """ | |
| ) | |
| with gr.Row(): | |
| # LEFT SIDE — INPUT | |
| with gr.Column(): | |
| text_input = gr.Textbox( | |
| label="Enter text", | |
| placeholder="Type some text here...", | |
| lines=5 | |
| ) | |
| tokenize_button = gr.Button( | |
| "Tokenize", | |
| variant="primary" | |
| ) | |
| # RIGHT SIDE — OUTPUT | |
| with gr.Column(): | |
| token_visualization = gr.HTML( | |
| label="Token Visualization" | |
| ) | |
| token_count = gr.Number( | |
| label="Token Count", | |
| value=0, | |
| interactive=False | |
| ) | |
| context_visualization = gr.HTML( | |
| label="Context Window" | |
| ) | |
| tokenize_button.click( | |
| fn=process_text, | |
| inputs=text_input, | |
| outputs=[ | |
| token_visualization, | |
| token_count, | |
| context_visualization | |
| ] | |
| ) | |
| app.launch() |