Spaces:

quickgrid
/

Tokenizer-Visualizer

Running

App Files Files Community

quickgrid commited on 16 days ago

Commit

6e13729

verified ·

1 Parent(s): 087d59b

Update index.html

Browse files

Files changed (1) hide show

index.html +419 -113

index.html CHANGED Viewed

@@ -1,159 +1,465 @@
 <!DOCTYPE html>
-<html lang="en">
 <head>
     <meta charset="UTF-8">
     <meta name="viewport" content="width=device-width, initial-scale=1.0">
-    <title>Universal Tokenizer Visualizer</title>
     <script src="https://cdn.tailwindcss.com"></script>
-    <style>
-        .token-chip {
-            display: inline-block;
-            padding: 2px 4px;
-            margin: 2px;
-            border-radius: 4px;
-            font-family: monospace;
-            transition: all 0.2s;
-            cursor: default;
         }
-        .token-chip:hover {
-            filter: brightness(0.9);
-            transform: translateY(-1px);
         }
-        /* Color palette for tokens */
-        .color-0 { background-color: #fca5a5; color: #7f1d1d; }
-        .color-1 { background-color: #fcd34d; color: #78350f; }
-        .color-2 { background-color: #86efac; color: #064e3b; }
-        .color-3 { background-color: #93c5fd; color: #1e3a8a; }
-        .color-4 { background-color: #c4b5fd; color: #4c1d95; }
-        .color-5 { background-color: #f9a8d4; color: #701a75; }
     </style>
 </head>
-<body class="bg-slate-50 min-h-screen p-4 md:p-8">
-    <div class="max-w-5xl mx-auto bg-white rounded-xl shadow-lg overflow-hidden border border-slate-200">
-        <div class="bg-slate-900 p-6 text-white">
-            <h1 class="text-2xl font-bold">Universal Tokenizer Visualizer</h1>
-            <p class="text-slate-400 text-sm mt-1">Inspect how models see your text. Runs 100% in-browser.</p>
-        </div>
-        <div class="p-6 space-y-6">
-            <div class="grid grid-cols-1 md:grid-cols-2 gap-4">
-                <div>
-                    <label class="block text-sm font-medium text-slate-700 mb-1">Hugging Face Model ID</label>
                     <div class="flex gap-2">
-                        <input type="text" id="modelInput" value="deepseek-ai/DeepSeek-V2"
-                               class="flex-1 border rounded-lg px-3 py-2 text-sm focus:ring-2 focus:ring-blue-500 outline-none">
-                        <button id="loadBtn" class="bg-blue-600 text-white px-4 py-2 rounded-lg text-sm font-semibold hover:bg-blue-700 transition">
-                            Load
                         </button>
                     </div>
-                    <p class="text-xs text-slate-500 mt-1 italic">Note: Ensure the model has a tokenizer.json file.</p>
                 </div>
-                <div class="flex flex-col justify-end">
-                    <div id="status" class="text-sm font-medium text-slate-600 bg-slate-100 p-2 rounded-lg text-center border border-dashed border-slate-300">
-                        Ready to load
-                    </div>
                 </div>
             </div>
-            <div>
-                <label class="block text-sm font-medium text-slate-700 mb-1">Input Text</label>
-                <textarea id="inputText" rows="6"
-                          class="w-full border rounded-xl p-4 text-lg focus:ring-2 focus:ring-blue-500 outline-none"
-                          placeholder="Type something here to see tokens..."></textarea>
             </div>
-            <div class="flex gap-4 border-t border-b py-3 text-sm font-mono">
-                <div>Tokens: <span id="tokenCount" class="font-bold text-blue-600">0</span></div>
-                <div>Characters: <span id="charCount" class="font-bold text-slate-600">0</span></div>
             </div>
-            <div>
-                <label class="block text-sm font-medium text-slate-700 mb-2">Tokenized Output</label>
-                <div id="visualizer" class="min-h-[150px] p-4 bg-slate-50 rounded-xl border border-slate-200 leading-relaxed">
-                    </div>
             </div>
-            <div>
-                <label class="block text-sm font-medium text-slate-700 mb-2">Token IDs</label>
-                <div id="tokenIds" class="text-xs font-mono p-3 bg-slate-900 text-slate-300 rounded-lg overflow-x-auto whitespace-nowrap">
-                    []
                 </div>
             </div>
-        </div>
     </div>
     <script type="module">
-        import { AutoTokenizer } from 'https://cdn.jsdelivr.net/npm/@xenova/transformers@2.17.2';
         let tokenizer = null;
-        const modelInput = document.getElementById('modelInput');
-        const loadBtn = document.getElementById('loadBtn');
-        const status = document.getElementById('status');
-        const inputText = document.getElementById('inputText');
-        const visualizer = document.getElementById('visualizer');
-        const tokenCount = document.getElementById('tokenCount');
-        const charCount = document.getElementById('charCount');
-        const tokenIdsDiv = document.getElementById('tokenIds');
-        async function loadModel(modelId) {
             try {
-                status.innerText = "⏳ Loading tokenizer...";
-                status.className = "text-sm font-medium text-amber-600 bg-amber-50 p-2 rounded-lg text-center border border-amber-200";
-                // We specify legacy: false to ensure it looks for tokenizer.json
-                tokenizer = await AutoTokenizer.from_pretrained(modelId);
-                status.innerText = `✅ Loaded: ${modelId}`;
-                status.className = "text-sm font-medium text-emerald-600 bg-emerald-50 p-2 rounded-lg text-center border border-emerald-200";
-                updateTokenization();
-            } catch (e) {
-                status.innerText = "❌ Error loading model";
-                status.className = "text-sm font-medium text-red-600 bg-red-50 p-2 rounded-lg text-center border border-red-200";
-                console.error(e);
             }
         }
-        function updateTokenization() {
             if (!tokenizer) return;
-            const text = inputText.value;
-            if (!text) {
-                visualizer.innerHTML = "";
-                tokenCount.innerText = "0";
-                charCount.innerText = "0";
-                tokenIdsDiv.innerText = "[]";
                 return;
             }
-            // Encode text
-            const tokens = tokenizer.encode(text);
-            const decodedTokens = tokens.map(id => tokenizer.decode([id]));
-            // Update Stats
-            tokenCount.innerText = tokens.length;
-            charCount.innerText = text.length;
-            tokenIdsDiv.innerText = JSON.stringify(tokens);
-            // Clear visualizer
-            visualizer.innerHTML = "";
-            // Create Visual Chips
-            decodedTokens.forEach((token, index) => {
-                const span = document.createElement('span');
-                span.className = `token-chip color-${index % 6}`;
-                // Handle spaces and newlines for visibility
-                let displayToken = token.replace(/ /g, ' ').replace(/\n/g, '↵\n');
-                span.innerText = displayToken;
-                span.title = `ID: ${tokens[index]}`;
-                visualizer.appendChild(span);
             });
         }
-        loadBtn.addEventListener('click', () => loadModel(modelInput.value));
-        inputText.addEventListener('input', updateTokenization);
-        // Initial Load
-        loadModel(modelInput.value);
     </script>
 </body>
 </html>

 <!DOCTYPE html>
+<html lang="en" class="light">
 <head>
     <meta charset="UTF-8">
     <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <title>LLM Tokenizer Visualizer</title>
+    <!-- Tailwind CSS -->
     <script src="https://cdn.tailwindcss.com"></script>
+    <script>
+        tailwind.config = {
+            darkMode: 'class',
+            theme: {
+                extend: {
+                    fontFamily: {
+                        sans: ['Inter', 'sans-serif'],
+                        mono: ['Fira Code', 'monospace'],
+                    },
+                }
+            }
         }
+    </script>
+    <!-- Google Fonts -->
+    <link href="https://fonts.googleapis.com/css2?family=Inter:wght@400;500;600;700&family=Fira+Code:wght@400;500&display=swap" rel="stylesheet">
+    <!-- Transformers.js -->
+    <script type="module">
+        import { AutoTokenizer } from 'https://cdn.jsdelivr.net/npm/@xenova/transformers@2.17.1';
+        window.AutoTokenizer = AutoTokenizer;
+    </script>
+    <style type="text/tailwindcss">
+        @layer utilities {
+            .content-auto {
+                content-visibility: auto;
+            }
+            .token-box {
+                white-space: pre;
+                word-break: break-all;
+            }
         }
     </style>
 </head>
+<body class="bg-gray-50 dark:bg-gray-900 text-gray-900 dark:text-gray-100 min-h-screen transition-colors duration-200">
+    <div class="max-w-5xl mx-auto px-4 py-8">
+        <!-- Header -->
+        <header class="flex justify-between items-center mb-8">
+            <div>
+                <h1 class="text-3xl font-bold bg-gradient-to-r from-blue-600 to-purple-600 bg-clip-text text-transparent">LLM Tokenizer Visualizer</h1>
+                <p class="text-gray-600 dark:text-gray-400 mt-1">Client-side tokenization for any Hugging Face model, powered by Transformers.js</p>
+            </div>
+            <button id="themeToggle" class="p-2 rounded-full hover:bg-gray-200 dark:hover:bg-gray-800 transition-colors">
+                <!-- Sun Icon (for dark mode) -->
+                <svg id="sunIcon" class="w-6 h-6 hidden" fill="none" stroke="currentColor" viewBox="0 0 24 24">
+                    <path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M12 3v1m0 16v1m9-9h-1M4 12H3m15.364 6.364l-.707-.707M6.343 6.343l-.707-.707m12.728 0l-.707.707M6.343 17.657l-.707.707M16 12a4 4 0 11-8 0 4 4 0 018 0z"></path>
+                </svg>
+                <!-- Moon Icon (for light mode) -->
+                <svg id="moonIcon" class="w-6 h-6" fill="none" stroke="currentColor" viewBox="0 0 24 24">
+                    <path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M20.354 15.354A9 9 0 018.646 3.646 9.003 9.003 0 0012 21a9.003 9.003 0 008.354-5.646z"></path>
+                </svg>
+            </button>
+        </header>
+        <!-- Model Config Section -->
+        <section class="bg-white dark:bg-gray-800 rounded-xl shadow-sm p-6 mb-6">
+            <h2 class="text-lg font-semibold mb-4">Model Configuration</h2>
+            <div class="grid grid-cols-1 md:grid-cols-12 gap-4">
+                <div class="md:col-span-8">
+                    <label class="block text-sm font-medium mb-2" for="modelId">Hugging Face Model ID</label>
                     <div class="flex gap-2">
+                        <input
+                            type="text"
+                            id="modelId"
+                            value="deepseek-ai/DeepSeek-V4-Pro"
+                            class="flex-1 px-4 py-2 rounded-lg border border-gray-300 dark:border-gray-600 bg-white dark:bg-gray-700 focus:ring-2 focus:ring-blue-500 focus:border-transparent outline-none transition-all"
+                            placeholder="e.g. meta-llama/Meta-Llama-3-8B-Instruct"
+                        >
+                        <button
+                            id="loadModelBtn"
+                            class="px-6 py-2 bg-blue-600 hover:bg-blue-700 text-white rounded-lg font-medium transition-colors flex items-center gap-2 disabled:opacity-50 disabled:cursor-not-allowed"
+                        >
+                            <svg id="loadSpinner" class="animate-spin w-4 h-4 hidden" fill="none" viewBox="0 0 24 24">
+                                <circle class="opacity-25" cx="12" cy="12" r="10" stroke="currentColor" stroke-width="4"></circle>
+                                <path class="opacity-75" fill="currentColor" d="M4 12a8 8 0 018-8V0C5.373 0 0 5.373 0 12h4zm2 5.291A7.962 7.962 0 014 12H0c0 3.042 1.135 5.824 3 7.938l3-2.647z"></path>
+                            </svg>
+                            <span id="loadBtnText">Load Model</span>
                         </button>
                     </div>
                 </div>
+                <div class="md:col-span-4">
+                    <label class="block text-sm font-medium mb-2" for="hfToken">Optional HF Token (for gated models)</label>
+                    <input
+                        type="password"
+                        id="hfToken"
+                        class="w-full px-4 py-2 rounded-lg border border-gray-300 dark:border-gray-600 bg-white dark:bg-gray-700 focus:ring-2 focus:ring-blue-500 focus:border-transparent outline-none transition-all"
+                        placeholder="hf_..."
+                    >
                 </div>
             </div>
+            <div class="mt-4">
+                <label class="block text-sm font-medium mb-2">Popular Models</label>
+                <select
+                    id="popularModels"
+                    class="w-full md:w-1/3 px-4 py-2 rounded-lg border border-gray-300 dark:border-gray-600 bg-white dark:bg-gray-700 focus:ring-2 focus:ring-blue-500 focus:border-transparent outline-none transition-all"
+                >
+                    <option value="">Select a popular model...</option>
+                    <option value="deepseek-ai/DeepSeek-V4-Pro">DeepSeek-V4-Pro</option>
+                    <option value="meta-llama/Meta-Llama-3-8B-Instruct">Llama 3 8B Instruct</option>
+                    <option value="mistralai/Mistral-7B-Instruct-v0.3">Mistral 7B v0.3</option>
+                    <option value="google/gemma-2-2b-it">Gemma 2 2B IT</option>
+                    <option value="Qwen/Qwen2.5-7B-Instruct">Qwen 2.5 7B Instruct</option>
+                    <option value="microsoft/Phi-3-mini-4k-instruct">Phi-3 Mini 4K Instruct</option>
+                </select>
             </div>
+            <div id="modelError" class="hidden mt-3 p-3 bg-red-100 dark:bg-red-900/30 text-red-700 dark:text-red-300 rounded-lg text-sm"></div>
+            <div id="modelSuccess" class="hidden mt-3 p-3 bg-green-100 dark:bg-green-900/30 text-green-700 dark:text-green-300 rounded-lg text-sm"></div>
+        </section>
+        <!-- Text Input Section -->
+        <section class="bg-white dark:bg-gray-800 rounded-xl shadow-sm p-6 mb-6">
+            <div class="flex justify-between items-center mb-4">
+                <h2 class="text-lg font-semibold">Input Text</h2>
+                <button
+                    id="clearTextBtn"
+                    class="text-sm text-gray-600 dark:text-gray-400 hover:text-gray-900 dark:hover:text-gray-100 transition-colors"
+                >
+                    Clear
+                </button>
             </div>
+            <textarea
+                id="textInput"
+                rows="4"
+                class="w-full px-4 py-3 rounded-lg border border-gray-300 dark:border-gray-600 bg-white dark:bg-gray-700 focus:ring-2 focus:ring-blue-500 focus:border-transparent outline-none transition-all font-mono text-sm resize-none"
+                placeholder="Enter text to tokenize..."
+                disabled
+            >Hello, world! This is a test of the tokenizer. Let's see how many tokens this takes.
+你好，世界！这是一个分词器测试。
+Special tokens: <|endoftext|> <s> </s></textarea>
+        </section>
+        <!-- Stats Section -->
+        <section class="grid grid-cols-1 md:grid-cols-3 gap-4 mb-6">
+            <div class="bg-white dark:bg-gray-800 rounded-xl shadow-sm p-4">
+                <p class="text-sm text-gray-600 dark:text-gray-400">Total Tokens</p>
+                <p id="totalTokens" class="text-2xl font-bold mt-1">0</p>
+            </div>
+            <div class="bg-white dark:bg-gray-800 rounded-xl shadow-sm p-4">
+                <p class="text-sm text-gray-600 dark:text-gray-400">Total Characters</p>
+                <p id="totalChars" class="text-2xl font-bold mt-1">0</p>
             </div>
+            <div class="bg-white dark:bg-gray-800 rounded-xl shadow-sm p-4">
+                <p class="text-sm text-gray-600 dark:text-gray-400">Tokens per Character</p>
+                <p id="tokenRatio" class="text-2xl font-bold mt-1">0.00</p>
+            </div>
+        </section>
+        <!-- Token Visualization Section -->
+        <section class="bg-white dark:bg-gray-800 rounded-xl shadow-sm p-6 mb-6">
+            <div class="flex justify-between items-center mb-4">
+                <h2 class="text-lg font-semibold">Token Visualization</h2>
+                <div class="flex gap-2">
+                    <button
+                        id="copyTokensBtn"
+                        class="px-4 py-2 bg-gray-100 dark:bg-gray-700 hover:bg-gray-200 dark:hover:bg-gray-600 rounded-lg text-sm font-medium transition-colors flex items-center gap-2 disabled:opacity-50"
+                        disabled
+                    >
+                        <svg class="w-4 h-4" fill="none" stroke="currentColor" viewBox="0 0 24 24">
+                            <path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M8 16H6a2 2 0 01-2-2V6a2 2 0 012-2h8a2 2 0 012 2v2m-6 12h8a2 2 0 002-2v-8a2 2 0 00-2-2h-8a2 2 0 00-2 2v8a2 2 0 002 2z"></path>
+                        </svg>
+                        Copy Tokens
+                    </button>
+                    <button
+                        id="copyIdsBtn"
+                        class="px-4 py-2 bg-gray-100 dark:bg-gray-700 hover:bg-gray-200 dark:hover:bg-gray-600 rounded-lg text-sm font-medium transition-colors flex items-center gap-2 disabled:opacity-50"
+                        disabled
+                    >
+                        <svg class="w-4 h-4" fill="none" stroke="currentColor" viewBox="0 0 24 24">
+                            <path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M8 16H6a2 2 0 01-2-2V6a2 2 0 012-2h8a2 2 0 012 2v2m-6 12h8a2 2 0 002-2v-8a2 2 0 00-2-2h-8a2 2 0 00-2 2v8a2 2 0 002 2z"></path>
+                        </svg>
+                        Copy IDs
+                    </button>
                 </div>
             </div>
+            <div
+                id="tokenContainer"
+                class="min-h-[120px] p-4 bg-gray-50 dark:bg-gray-700/50 rounded-lg overflow-x-auto flex flex-wrap gap-2 items-start content-start"
+            >
+                <p id="emptyState" class="text-gray-500 dark:text-gray-400 w-full text-center py-8">Load a model and enter text to see tokens</p>
+            </div>
+            <p class="text-xs text-gray-500 dark:text-gray-400 mt-2">Hover over tokens to see their ID. Special tokens are marked with a dashed border.</p>
+        </section>
+        <!-- Footer -->
+        <footer class="text-center text-gray-600 dark:text-gray-400 text-sm">
+            <p>Powered by <a href="https://huggingface.co/docs/transformers.js" target="_blank" class="text-blue-600 dark:text-blue-400 hover:underline">Transformers.js</a> | Runs entirely in your browser, no server required</p>
+        </footer>
     </div>
     <script type="module">
+        // Wait for Transformers.js to load
+        await new Promise(resolve => {
+            const checkInterval = setInterval(() => {
+                if (window.AutoTokenizer) clearInterval(checkInterval);
+            }, 100);
+        });
+        const { AutoTokenizer } = window;
+        // DOM Elements
+        const themeToggle = document.getElementById('themeToggle');
+        const sunIcon = document.getElementById('sunIcon');
+        const moonIcon = document.getElementById('moonIcon');
+        const modelIdInput = document.getElementById('modelId');
+        const hfTokenInput = document.getElementById('hfToken');
+        const loadModelBtn = document.getElementById('loadModelBtn');
+        const loadSpinner = document.getElementById('loadSpinner');
+        const loadBtnText = document.getElementById('loadBtnText');
+        const popularModelsSelect = document.getElementById('popularModels');
+        const modelError = document.getElementById('modelError');
+        const modelSuccess = document.getElementById('modelSuccess');
+        const textInput = document.getElementById('textInput');
+        const clearTextBtn = document.getElementById('clearTextBtn');
+        const tokenContainer = document.getElementById('tokenContainer');
+        const emptyState = document.getElementById('emptyState');
+        const totalTokensEl = document.getElementById('totalTokens');
+        const totalCharsEl = document.getElementById('totalChars');
+        const tokenRatioEl = document.getElementById('tokenRatio');
+        const copyTokensBtn = document.getElementById('copyTokensBtn');
+        const copyIdsBtn = document.getElementById('copyIdsBtn');
+        // State
         let tokenizer = null;
+        let currentTokens = [];
+        let currentTokenIds = [];
+        let isDark = localStorage.getItem('theme') === 'dark' || (!localStorage.getItem('theme') && window.matchMedia('(prefers-color-scheme: dark)').matches);
+        // Initialize theme
+        function initTheme() {
+            if (isDark) {
+                document.documentElement.classList.add('dark');
+                sunIcon.classList.remove('hidden');
+                moonIcon.classList.add('hidden');
+            } else {
+                document.documentElement.classList.remove('dark');
+                sunIcon.classList.add('hidden');
+                moonIcon.classList.remove('hidden');
+            }
+        }
+        initTheme();
+        // Theme toggle
+        themeToggle.addEventListener('click', () => {
+            isDark = !isDark;
+            localStorage.setItem('theme', isDark ? 'dark' : 'light');
+            initTheme();
+            // Re-render tokens to update colors
+            if (currentTokens.length > 0) renderTokens();
+        });
+        // Popular models select
+        popularModelsSelect.addEventListener('change', (e) => {
+            if (e.target.value) {
+                modelIdInput.value = e.target.value;
+            }
+        });
+        // Load model function
+        async function loadModel() {
+            const modelId = modelIdInput.value.trim();
+            if (!modelId) {
+                showError('Please enter a Hugging Face model ID');
+                return;
+            }
+            // Show loading state
+            loadModelBtn.disabled = true;
+            loadSpinner.classList.remove('hidden');
+            loadBtnText.textContent = 'Loading...';
+            hideMessages();
+            textInput.disabled = true;
             try {
+                const token = hfTokenInput.value.trim() || undefined;
+                tokenizer = await AutoTokenizer.from_pretrained(modelId, { token });
+                // Success
+                showSuccess(`Successfully loaded tokenizer for ${modelId}`);
+                textInput.disabled = false;
+                textInput.focus();
+                // Auto-tokenize existing text
+                if (textInput.value.trim()) {
+                    tokenizeText();
+                }
+            } catch (err) {
+                console.error('Failed to load model:', err);
+                showError(`Failed to load model: ${err.message}. Check the model ID and ensure it's public (or provide a HF token for gated models).`);
+                tokenizer = null;
+                textInput.disabled = true;
+            } finally {
+                // Reset button state
+                loadModelBtn.disabled = false;
+                loadSpinner.classList.add('hidden');
+                loadBtnText.textContent = 'Load Model';
             }
         }
+        // Tokenize text function
+        function tokenizeText() {
             if (!tokenizer) return;
+            const text = textInput.value;
+            if (!text.trim()) {
+                currentTokens = [];
+                currentTokenIds = [];
+                renderTokens();
+                updateStats();
+                copyTokensBtn.disabled = true;
+                copyIdsBtn.disabled = true;
                 return;
             }
+            try {
+                const output = tokenizer(text);
+                currentTokens = output.tokens;
+                currentTokenIds = output.input_ids;
+                renderTokens();
+                updateStats();
+                copyTokensBtn.disabled = false;
+                copyIdsBtn.disabled = false;
+            } catch (err) {
+                console.error('Tokenization failed:', err);
+                showError(`Tokenization failed: ${err.message}`);
+            }
+        }
+        // Render tokens
+        function renderTokens() {
+            tokenContainer.innerHTML = '';
+            if (currentTokens.length === 0) {
+                tokenContainer.appendChild(emptyState);
+                emptyState.classList.remove('hidden');
+                return;
+            }
+            emptyState.classList.add('hidden');
+            currentTokens.forEach((token, index) => {
+                const tokenId = currentTokenIds[index];
+                // Generate consistent pastel color based on token ID
+                const hue = (tokenId * 137) % 360; // 137 is prime for even distribution
+                const lightness = isDark ? '80%' : '90%';
+                const bgColor = `hsl(${hue}, 70%, ${lightness})`;
+                const textColor = isDark ? '#1f2937' : '#1f2937';
+                const tokenEl = document.createElement('span');
+                tokenEl.className = 'token-box px-2 py-1 rounded-md text-sm font-mono cursor-pointer transition-all hover:scale-105 hover:shadow-md';
+                tokenEl.style.backgroundColor = bgColor;
+                tokenEl.style.color = textColor;
+                // Replace whitespace with visible symbols
+                const displayToken = token
+                    .replace(/\n/g, '↵')
+                    .replace(/\t/g, '→')
+                    .replace(/ /g, '·');
+                tokenEl.textContent = displayToken;
+                // Show token ID on hover
+                tokenEl.title = `Token ID: ${tokenId}\nRaw: ${token.replace(/\n/g, '\\n').replace(/\t/g, '\\t')}`;
+                // Highlight special tokens
+                if (tokenizer.special_tokens_map && Object.values(tokenizer.special_tokens_map).flat().includes(token)) {
+                    tokenEl.classList.add('border', 'border-dashed', 'border-gray-400', 'dark:border-gray-500');
+                }
+                tokenContainer.appendChild(tokenEl);
             });
         }
+        // Update stats
+        function updateStats() {
+            const totalTokens = currentTokens.length;
+            const totalChars = textInput.value.length;
+            const ratio = totalChars > 0 ? (totalTokens / totalChars).toFixed(2) : '0.00';
+            totalTokensEl.textContent = totalTokens.toLocaleString();
+            totalCharsEl.textContent = totalChars.toLocaleString();
+            tokenRatioEl.textContent = ratio;
+        }
+        // Copy functions
+        async function copyToClipboard(text, btn) {
+            try {
+                await navigator.clipboard.writeText(text);
+                const originalText = btn.innerHTML;
+                btn.innerHTML = `
+                    <svg class="w-4 h-4" fill="none" stroke="currentColor" viewBox="0 0 24 24">
+                        <path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M5 13l4 4L19 7"></path>
+                    </svg>
+                    Copied!
+                `;
+                setTimeout(() => {
+                    btn.innerHTML = originalText;
+                }, 2000);
+            } catch (err) {
+                showError('Failed to copy to clipboard');
+            }
+        }
+        copyTokensBtn.addEventListener('click', () => {
+            copyToClipboard(currentTokens.join('\n'), copyTokensBtn);
+        });
+        copyIdsBtn.addEventListener('click', () => {
+            copyToClipboard(JSON.stringify(currentTokenIds, null, 2), copyIdsBtn);
+        });
+        // Clear text
+        clearTextBtn.addEventListener('click', () => {
+            textInput.value = '';
+            tokenizeText();
+        });
+        // Helper functions
+        function showError(message) {
+            modelError.textContent = message;
+            modelError.classList.remove('hidden');
+            modelSuccess.classList.add('hidden');
+        }
+        function showSuccess(message) {
+            modelSuccess.textContent = message;
+            modelSuccess.classList.remove('hidden');
+            modelError.classList.add('hidden');
+        }
+        function hideMessages() {
+            modelError.classList.add('hidden');
+            modelSuccess.classList.add('hidden');
+        }
+        // Debounce for text input
+        function debounce(func, wait) {
+            let timeout;
+            return function(...args) {
+                clearTimeout(timeout);
+                timeout = setTimeout(() => func.apply(this, args), wait);
+            };
+        }
+        const debouncedTokenize = debounce(tokenizeText, 300);
+        // Event listeners
+        loadModelBtn.addEventListener('click', loadModel);
+        modelIdInput.addEventListener('keypress', (e) => {
+            if (e.key === 'Enter') loadModel();
+        });
+        textInput.addEventListener('input', debouncedTokenize);
+        // Auto-load default model on page load
+        window.addEventListener('load', () => {
+            // Wait a bit for Transformers.js to fully initialize
+            setTimeout(loadModel, 500);
+        });
     </script>
 </body>
 </html>