diff --git "a/index.html" "b/index.html" --- "a/index.html" +++ "b/index.html" @@ -1,1512 +1,1522 @@ - - - TokenViz - Universal Tokenizer Visualizer - - - - - + + - /* Info panel */ - .info-panel { - background: var(--bg-secondary); - border-radius: 10px; - padding: 1rem; - margin-top: 1rem; - font-size: 0.85rem; - color: var(--text-secondary); - border: 1px solid var(--border-color); - } + +
+
- .info-panel code { - background: var(--bg-primary); - padding: 0.15rem 0.35rem; - border-radius: 4px; - font-family: 'JetBrains Mono', monospace; - font-size: 0.8rem; - color: var(--accent); - } +
- /* Copy button */ - .copy-btn { - background: none; - border: 1px solid var(--border-color); - color: var(--text-muted); - border-radius: 6px; - padding: 0.4rem 0.75rem; - font-size: 0.8rem; - cursor: pointer; - transition: all 0.2s ease; - display: inline-flex; - align-items: center; - gap: 0.35rem; - } + +
+ +
+
+ + runs in-browser · no server · no GPU +
+
+
- .copy-btn:hover { - border-color: var(--accent); - color: var(--accent); - } + +
+
▸ select tokenizer
+
+ +
+
+ + + +
+
- .copy-btn.copied { - border-color: var(--success); - color: var(--success); - } - - - - -
+ +
-
-
-
Loading tokenizer...
-
This may take a moment for large vocabularies
+ +
+
+
+
+ Input Text +
+
+ + + + + +
+
+ +
0 characters
-
- ⚠️ - Error message -
+ +
-
-
- -

- Universal tokenizer visualization for any HuggingFace model. - See exactly how LLMs break down text into tokens, IDs, and bytes — all in your browser with zero GPU required. -

-
- - -
-
- 🤖 Select Model -
-
- -
-
- - -
+ +
+
+
Tokens
+
+
no model loaded
- - -
+ + + +
+ + +
+
+
Loading Tokenizer
+
Downloading tokenizer files from Hugging Face Hub…
This may take a moment on first load. Files are cached in your browser.
+
+
+
+
+
+ + +
+ + + \ No newline at end of file