Spaces:
Sleeping
Sleeping
File size: 1,883 Bytes
034d4ef 3b5aa5b c413131 3b5aa5b 37af725 3b5aa5b c413131 3b5aa5b c413131 3b5aa5b c413131 3b5aa5b 034d4ef 3b5aa5b c413131 3b5aa5b c413131 3b5aa5b | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 | import sys
import io
import gradio
from transformers import AutoTokenizer
# List of supported models
MODELS = [
"zai-org/GLM-5",
"MiniMaxAI/MiniMax-M2.5",
"deepseek-ai/DeepSeek-V3.2",
"Qwen/Qwen3-14B",
"Qwen/Qwen3-235B-A22B",
"Qwen/Qwen3-30B-A3B",
"Qwen/Qwen3-32B"
]
# Cache tokenizers to avoid repeated downloads
tokenizer_cache = {}
def highlight_text(tokens):
return [
(token, str(i%9))
for i, token in enumerate(tokens)
]
def count_tokens(model_name, text_input):
# textbytes = text_input.encode('utf-8')
text = text_input
if not text.strip():
return 0, []
# Load tokenizer (with caching)
if model_name not in tokenizer_cache:
tokenizer_cache[model_name] = AutoTokenizer.from_pretrained(
model_name, trust_remote_code=True
)
tokenizer = tokenizer_cache[model_name]
token_ids = tokenizer.encode(text, add_special_tokens=False)
tokens = []
for tid in token_ids:
tokens.append(tokenizer.decode([tid]))
return len(token_ids), len(text), highlight_text(tokens)
gradio.Interface(
fn=count_tokens,
inputs=[
gradio.Dropdown(choices=MODELS, label="Select Model", value=MODELS[0]),
gradio.Textbox(lines=5, label="Input Text")
],
outputs=[
gradio.Number(label="Tokens"),
gradio.Number(label="Characters"),
# gradio.Textbox(label="Tokens")
gradio.HighlightedText(
label="Tokens",
show_inline_category=False,
color_map={
"1": "red", "2": "green", "3": "blue",
"4": "yellow", "5": "orange", "6": "purple",
"7": "pink", "8": "cyan", "9": "gray"
}
)
],
title="Model Tokenizer",
description="Select a model and input text to see token count and token list."
).launch()
|