File size: 8,329 Bytes
7765cd2 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 | #!/usr/bin/env python3
"""
tokenizer_inspector.py v3 — Inspect and compare Hugging Face tokenizers.
Accepts repo IDs (e.g. gpt2, HuggingFaceTB/SmolLM2-135M) or local paths to
tokenizer.json files. Uses the `tokenizers` library for accurate tokenization.
Usage:
python3 tokenizer_inspector.py gpt2
python3 tokenizer_inspector.py gpt2 HuggingFaceTB/SmolLM2-135M --compare
python3 tokenizer_inspector.py gpt2 --text "hello world"
python3 tokenizer_inspector.py gpt2 --show-tokens
What it reports:
- Vocab size, token type (BPE / SentencePiece / byte-level)
- Special tokens
- chars/token on a test corpus (higher = more compact)
- Vocab utilization: what fraction of the vocab does your data actually use?
- Sample of interesting tokens (longest, shortest, most common)
- In --compare mode: side-by-side comparison table
"""
import argparse
import json
import sys
from collections import Counter
from pathlib import Path
# Default test corpus: mixed English + code + numbers
DEFAULT_CORPUS = """\
The quick brown fox jumps over the lazy dog.
def fibonacci(n):
if n <= 1: return n
return fibonacci(n-1) + fibonacci(n-2)
print(fibonacci(10))
#!/usr/bin/env python3
import numpy as np
arr = np.zeros((1024, 1024), dtype=np.float32)
print(arr.sum())
The quick brown fox jumps over the lazy dog.
"""
def load_tokenizer(source: str):
"""Load a tokenizer from a HF repo ID or local path. Returns (Tokenizer, display_name, json_path_or_None)."""
from tokenizers import Tokenizer
# Try local path first
local = Path(source)
if local.exists() and local.is_file():
return Tokenizer.from_file(str(local)), local.name, str(local)
if local.exists() and local.is_dir():
for name in ("tokenizer.json", "tokenizer.model"):
p = local / name
if p.exists():
return Tokenizer.from_file(str(p)), source, str(p)
# Try Hugging Face
try:
from huggingface_hub import hf_hub_download
path = hf_hub_download(repo_id=source, filename="tokenizer.json")
return Tokenizer.from_file(path), source, path
except Exception:
pass
try:
from huggingface_hub import hf_hub_download
path = hf_hub_download(repo_id=source, filename="tokenizer.model")
return Tokenizer.from_file(path), source, None
except Exception as e:
print(f"ERROR: cannot load tokenizer for '{source}': {e}", file=sys.stderr)
sys.exit(1)
def get_special_tokens(json_path):
"""Read special/added tokens from the tokenizer JSON."""
if not json_path:
return []
try:
with open(json_path) as f:
data = json.load(f)
added = data.get("added_tokens", [])
return [t["content"] for t in added]
except Exception:
return []
def detect_token_type(tok) -> str:
"""Detect the tokenization scheme."""
s = tok.to_str()
head = s[:3000]
if "ByteLevel" in head:
return "byte-level BPE"
if "BPE" in head:
return "BPE"
if "Unigram" in head:
return "Unigram (SentencePiece)"
if "WordPiece" in head:
return "WordPiece"
if "WordLevel" in head:
return "WordLevel"
return "unknown"
def analyze(source: str, corpus: str, show_tokens: bool = False) -> dict:
"""Analyze one tokenizer and return a metrics dict."""
tok, name, json_path = load_tokenizer(source)
vocab_size = tok.get_vocab_size()
token_type = detect_token_type(tok)
special_tokens = get_special_tokens(json_path)
enc = tok.encode(corpus)
tokens = enc.tokens
n_tokens = len(tokens)
n_chars = len(corpus)
chars_per_token = n_chars / max(n_tokens, 1)
freq = Counter(tokens)
most_common = freq.most_common(10)
# Vocab utilization: how many distinct vocab tokens appear in the corpus?
distinct_used = set(tokens)
utilization = len(distinct_used) / max(vocab_size, 1)
# Longest / shortest real tokens (exclude special)
vocab = tok.get_vocab()
real = [t for t in vocab if t and not (t.startswith("<") and t.endswith(">"))]
longest = max(real, key=len) if real else ("", 0)
shortest = min((t for t in real if t), key=len, default="")
# Roundtrip check
decoded = tok.decode(enc.ids)
roundtrip_ok = decoded.strip() == corpus.strip()
result = {
"repo": name,
"vocab_size": vocab_size,
"token_type": token_type,
"n_special": len(special_tokens),
"special_sample": special_tokens[:5],
"corpus_chars": n_chars,
"corpus_tokens": n_tokens,
"chars_per_token": round(chars_per_token, 2),
"vocab_utilization": round(utilization, 4),
"distinct_tokens_used": len(distinct_used),
"most_common": most_common,
"longest_token": (longest[:40], len(longest)),
"shortest_token": (shortest[:40], len(shortest)),
"roundtrip_ok": roundtrip_ok,
}
if show_tokens:
result["sample_tokens"] = tokens[:30]
return result
def print_report(r: dict, show_tokens: bool = False):
print(f"\n{'='*60}")
print(f" {r['repo']}")
print(f"{'='*60}")
print(f" Vocab size: {r['vocab_size']:,}")
print(f" Token type: {r['token_type']}")
print(f" Special tokens: {r['n_special']} ({', '.join(r['special_sample'])}{'...' if r['n_special'] > 5 else ''})")
print(f" Corpus: {r['corpus_chars']} chars -> {r['corpus_tokens']} tokens")
print(f" Chars/token: {r['chars_per_token']}")
print(f" Vocab utilization: {r['vocab_utilization']:.2%} ({r['distinct_tokens_used']} distinct)")
print(f" Longest token: {r['longest_token'][0]!r} ({r['longest_token'][1]} chars)")
print(f" Shortest token: {r['shortest_token'][0]!r} ({r['shortest_token'][1]} chars)")
print(f" Roundtrip OK: {'yes' if r['roundtrip_ok'] else 'NO'}")
print(f"\n Top 10 tokens by frequency:")
for tok, cnt in r["most_common"]:
print(f" {repr(tok):>20s} x{cnt}")
if show_tokens and "sample_tokens" in r:
print(f"\n First 30 tokens:")
print(f" {r['sample_tokens']}")
print()
def print_comparison(results: list):
print(f"\n{'='*60}")
print(f" COMPARISON ({len(results)} tokenizers)")
print(f"{'='*60}")
names = [r["repo"][:18] for r in results]
print(f" {'Metric':<20s} " + " ".join(f"{n:>20s}" for n in names))
print(f" {'-'*20} " + " ".join(f"{'-'*20}" for _ in names))
rows = [
("Vocab size", lambda r: f"{r['vocab_size']:,}"),
("Token type", lambda r: r["token_type"]),
("Special tokens", lambda r: str(r["n_special"])),
("Chars/token", lambda r: f"{r['chars_per_token']}"),
("Corpus tokens", lambda r: str(r["corpus_tokens"])),
("Vocab util", lambda r: f"{r['vocab_utilization']:.2%}"),
("Longest tok", lambda r: str(r["longest_token"][1])),
("Roundtrip", lambda r: "ok" if r["roundtrip_ok"] else "FAIL"),
]
for label, fn in rows:
print(f" {label:<20s} " + " ".join(f"{fn(r):>20s}" for r in results))
print()
def main():
parser = argparse.ArgumentParser(description="Inspect and compare HF tokenizers")
parser.add_argument("sources", nargs="+", help="Repo IDs or paths to tokenizer.json")
parser.add_argument("--corpus", type=str, default=DEFAULT_CORPUS, help="Test corpus string")
parser.add_argument("--text", type=str, default=None, help="Short text to tokenize (overrides default corpus)")
parser.add_argument("--compare", action="store_true", help="Side-by-side comparison mode")
parser.add_argument("--show-tokens", action="store_true", help="Show first 30 tokens of the corpus")
parser.add_argument("--json", action="store_true", help="Output as JSON")
args = parser.parse_args()
corpus = args.text if args.text else args.corpus
results = [analyze(s, corpus, args.show_tokens) for s in args.sources]
if args.json:
print(json.dumps(results, indent=2))
elif args.compare and len(results) >= 2:
print_comparison(results)
for r in results:
print_report(r, args.show_tokens)
else:
for r in results:
print_report(r, args.show_tokens)
if __name__ == "__main__":
main() |