Spaces:

smcproject
/

malayalam-tokenizer-comparison

Running

App Files Files Community

santhosh commited on Feb 21

Commit

3b5c1dc

verified ·

1 Parent(s): 4ce299d

Deploy demo @ 04347af

Browse files

Files changed (6) hide show

README.md +15 -4
index.html +45 -18
js/app.js +426 -0
js/tokenizers.js +18 -0
js/worker.js +69 -0
style.css +402 -18

README.md CHANGED Viewed

@@ -1,10 +1,21 @@
 ---
 title: Malayalam Tokenizer Comparison
-emoji: 💻
-colorFrom: indigo
-colorTo: red
 sdk: static
 pinned: false
 ---
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

 ---
 title: Malayalam Tokenizer Comparison
+emoji: 🔤
+colorFrom: green
+colorTo: blue
 sdk: static
 pinned: false
+license: mit
+tags:
+  - malayalam
+  - tokenizer
+  - nlp
 ---
+# Malayalam Tokenizer Comparison
+Visualize how different tokenizers split Malayalam text into tokens.
+Compare Malayalam-specific tokenizers (BPE and Unigram) side-by-side with
+popular models like GPT-4, LLaMA, Mistral, and others.
+Built from [smc/malayalam-tokenizer](https://github.com/smc/malayalam-tokenizer).

index.html CHANGED Viewed

@@ -1,19 +1,46 @@
-<!doctype html>
-<html>
-	<head>
-		<meta charset="utf-8" />
-		<meta name="viewport" content="width=device-width" />
-		<title>My static Space</title>
-		<link rel="stylesheet" href="style.css" />
-	</head>
-	<body>
-		<div class="card">
-			<h1>Welcome to your static Space!</h1>
-			<p>You can modify this app directly by editing <i>index.html</i> in the Files and versions tab.</p>
-			<p>
-				Also don't forget to check the
-				<a href="https://huggingface.co/docs/hub/spaces" target="_blank">Spaces documentation</a>.
-			</p>
-		</div>
-	</body>
 </html>

+<!DOCTYPE html>
+<html lang="en">
+<head>
+  <meta charset="UTF-8">
+  <meta name="viewport" content="width=device-width, initial-scale=1.0">
+  <title>Malayalam Tokenizer Comparison</title>
+  <link rel="stylesheet" href="style.css">
+</head>
+<body>
+  <header>
+    <h1>Malayalam Tokenizer Comparison</h1>
+    <p>Visualize how different tokenizers split Malayalam text into tokens.</p>
+  </header>
+  <main>
+    <section class="input-section">
+      <label for="text-input">Input text</label>
+      <textarea
+        id="text-input"
+        rows="4"
+        placeholder="Type or paste Malayalam text here…"
+        spellcheck="false"
+        autocomplete="off"
+      >കേരളം ദക്ഷിണേന്ത്യയിലെ ഒരു സംസ്ഥാനമാണ്.</textarea>
+    </section>
+    <section class="add-tokenizer-section">
+      <select id="tokenizer-select">
+        <option value="">— choose a tokenizer —</option>
+      </select>
+      <button id="add-btn" type="button" disabled>Add tokenizer</button>
+    </section>
+    <div id="panels" role="list" aria-label="Tokenizer panels"></div>
+    <p id="empty-state" class="empty-state" aria-live="polite">
+      No tokenizers added yet. Choose one from the dropdown above.
+    </p>
+  </main>
+  <!-- Token-color highlight styles injected by app.js -->
+  <style id="highlight-styles"></style>
+  <script type="module" src="js/app.js"></script>
+</body>
 </html>

js/app.js ADDED Viewed

	@@ -0,0 +1,426 @@

+import { TOKENIZER_OPTIONS } from "./tokenizers.js";
+// ─── Token color palette ──────────────────────────────────────────────────────
+// Reference CSS custom properties so dark-mode overrides apply automatically.
+// The index wraps modulo 12; values are defined in style.css as --tok-N.
+const NUM_TOKEN_COLORS = 12;
+function tokenColorVar(index) {
+  return `var(--tok-${index % NUM_TOKEN_COLORS})`;
+}
+const DEBOUNCE_MS = 350;
+// ─── State ────────────────────────────────────────────────────────────────────
+/** @type {Map<number, PanelState>} */
+const panels = new Map();
+let panelCounter = 0;
+/**
+ * @typedef {{
+ *   modelId: string,
+ *   name: string,
+ *   worker: Worker,
+ *   panel: HTMLElement,
+ *   textMirror: HTMLElement,
+ *   statsEl: HTMLElement,
+ *   tbody: HTMLElement,
+ *   tokenCount: number,
+ * }} PanelState
+ */
+// ─── DOM refs ─────────────────────────────────────────────────────────────────
+const textInput = /** @type {HTMLTextAreaElement} */ (
+  document.getElementById("text-input")
+);
+const selectEl = /** @type {HTMLSelectElement}   */ (
+  document.getElementById("tokenizer-select")
+);
+const addBtn = /** @type {HTMLButtonElement}   */ (
+  document.getElementById("add-btn")
+);
+const panelsEl = /** @type {HTMLElement}         */ (
+  document.getElementById("panels")
+);
+const emptyState = /** @type {HTMLElement}         */ (
+  document.getElementById("empty-state")
+);
+const highlightStyleEl = /** @type {HTMLStyleElement}    */ (
+  document.getElementById("highlight-styles")
+);
+// ─── Populate dropdown ────────────────────────────────────────────────────────
+for (const [modelId, name] of Object.entries(TOKENIZER_OPTIONS)) {
+  if (!modelId) continue;
+  const opt = document.createElement("option");
+  opt.value = modelId;
+  opt.textContent = name;
+  selectEl.appendChild(opt);
+}
+selectEl.addEventListener("change", () => {
+  addBtn.disabled = !selectEl.value;
+});
+addBtn.addEventListener("click", () => {
+  const modelId = selectEl.value;
+  if (!modelId) return;
+  const name = TOKENIZER_OPTIONS[modelId] ?? modelId;
+  addPanel(modelId, name);
+  selectEl.value = "";
+  addBtn.disabled = true;
+});
+// ─── Text input ───────────────────────────────────────────────────────────────
+let debounceTimer = null;
+textInput.addEventListener("input", () => {
+  clearTimeout(debounceTimer);
+  debounceTimer = setTimeout(retokenizeAll, DEBOUNCE_MS);
+});
+function retokenizeAll() {
+  const text = textInput.value;
+  for (const id of panels.keys()) {
+    runTokenizer(id, text);
+  }
+}
+// ─── Panel lifecycle ──────────────────────────────────────────────────────────
+function addPanel(modelId, name) {
+  const id = panelCounter++;
+  const panel = document.createElement("div");
+  panel.className = "panel loading";
+  panel.setAttribute("role", "listitem");
+  panel.dataset.panelId = String(id);
+  panel.innerHTML = panelTemplate(name, modelId);
+  panel
+    .querySelector(".remove-btn")
+    .addEventListener("click", () => removePanel(id));
+  panelsEl.appendChild(panel);
+  updateEmptyState();
+  const worker = new Worker("js/worker.js", { type: "module" });
+  worker.addEventListener("message", (ev) => onWorkerMessage(id, ev.data));
+  worker.addEventListener("error", (ev) =>
+    onWorkerError(id, ev.message ?? "Worker error"),
+  );
+  panels.set(id, {
+    modelId,
+    name,
+    worker,
+    panel,
+    textMirror: panel.querySelector(".panel-text"),
+    statsEl: panel.querySelector(".panel-stats"),
+    tbody: panel.querySelector("tbody"),
+    tokenCount: 0,
+  });
+  runTokenizer(id, textInput.value);
+}
+function removePanel(id) {
+  const state = panels.get(id);
+  if (!state) return;
+  clearPanelHighlights(id, state.tokenCount);
+  state.worker.terminate();
+  state.panel.remove();
+  panels.delete(id);
+  updateEmptyState();
+}
+function updateEmptyState() {
+  emptyState.classList.toggle("hidden", panels.size > 0);
+}
+function panelTemplate(name, modelId) {
+  return `
+    <div class="panel-header">
+      <a class="panel-title" href="https://huggingface.co/${esc(modelId)}" target="_blank" rel="noopener noreferrer" title="${esc(modelId)}">${esc(name)}</a>
+      <button class="remove-btn" type="button" aria-label="Remove ${esc(name)}">Remove</button>
+    </div>
+    <div class="panel-loading">
+      <span class="spinner" aria-hidden="true"></span>
+      Loading tokenizer…
+    </div>
+    <div class="panel-text" aria-label="Tokenized text"></div>
+    <div class="panel-stats"></div>
+    <div class="panel-table-wrap">
+      <table aria-label="Token list">
+        <thead>
+          <tr>
+            <th></th>
+            <th>#</th>
+            <th>Token</th>
+            <th>ID</th>
+          </tr>
+        </thead>
+        <tbody></tbody>
+      </table>
+    </div>`;
+}
+// ─── Worker communication ─────────────────────────────────────────────────────
+function runTokenizer(id, text) {
+  const state = panels.get(id);
+  if (!state) return;
+  state.panel.classList.add("loading");
+  state.worker.postMessage({ model_id: state.modelId, text });
+}
+/**
+ * @param {number} id
+ * @param {{ token_ids: number[], decoded: string[], margins: number[] }} data
+ */
+function onWorkerMessage(id, data) {
+  const state = panels.get(id);
+  if (!state) return;
+  state.panel.classList.remove("loading");
+  const { token_ids, decoded, margins } = data;
+  const text = textInput.value;
+  // Clear stale highlights
+  clearPanelHighlights(id, state.tokenCount);
+  state.tokenCount = token_ids.length;
+  // Sync mirror text (single text node — required for Range offsets)
+  state.textMirror.textContent = text;
+  // Compute per-token character offsets in the original string
+  const offsets = computeOffsets(text, decoded, margins ?? []);
+  // Paint highlights via CSS Custom Highlight API
+  paintHighlights(id, state.textMirror, offsets);
+  // Stats bar
+  const ratio =
+    text.length > 0 ? (text.length / token_ids.length).toFixed(2) : "—";
+  state.statsEl.innerHTML =
+    `<span>Tokens: <strong>${token_ids.length}</strong></span>` +
+    `<span>Chars/token: <strong>${ratio}</strong></span>` +
+    `<span>Characters: <strong>${text.length}</strong></span>`;
+  // Table
+  renderTable(id, state, token_ids, decoded, offsets);
+}
+function onWorkerError(id, msg) {
+  const state = panels.get(id);
+  if (!state) return;
+  state.panel.classList.remove("loading");
+  state.textMirror.textContent = `! ${msg}`;
+}
+// ─── Offset computation ───────────────────────────────────────────────────────
+/**
+ * Walk `decoded` tokens greedily left-to-right against `text`, returning
+ * [{start, end}|null] for each token.
+ *
+ * The HuggingFace worker decodes each token individually.  Metaspace converts
+ * the leading ▁ (U+2581) back to a space, so `decoded` values should
+ * concatenate to the original text (modulo special tokens like <s>, </s>).
+ *
+ * @param {string} text
+ * @param {string[]} decoded
+ * @param {number[]} margins  — BERT: margin>0 means a word boundary space precedes
+ * @returns {Array<{start:number,end:number}|null>}
+ */
+function computeOffsets(text, decoded, margins) {
+  const offsets = [];
+  let cursor = 0;
+  for (let i = 0; i < decoded.length; i++) {
+    const tok = decoded[i] ?? "";
+    if (tok.length === 0) {
+      offsets.push(null);
+      continue;
+    }
+    // BERT word-boundary space: margin > 0 means we should skip a space
+    if (margins[i] > 0 && text[cursor] === " ") {
+      cursor++;
+    }
+    // Greedy forward search from cursor
+    const idx = text.indexOf(tok, cursor);
+    if (idx === -1) {
+      // Special token or mismatch — no visible range
+      offsets.push(null);
+      continue;
+    }
+    offsets.push({ start: idx, end: idx + tok.length });
+    cursor = idx + tok.length;
+  }
+  return offsets;
+}
+// ─── CSS Custom Highlight API ─────────────────────────────────────────────────
+// Track which ::highlight() rules have been injected to avoid duplication
+const injectedRules = new Set();
+/**
+ * Ensure ::highlight(name) rule exists in the shared <style> element.
+ * @param {string} name
+ * @param {string} bgColor
+ */
+function ensureRule(name, bgColor) {
+  if (injectedRules.has(name)) return;
+  injectedRules.add(name);
+  const sheet = highlightStyleEl.sheet;
+  sheet.insertRule(
+    `::highlight(${name}) { background-color: ${bgColor}; color: var(--tok-text); }`,
+    sheet.cssRules.length,
+  );
+}
+/**
+ * Ensure the hover ::highlight() rule exists for a panel.
+ * @param {number} panelId
+ */
+function ensureHoverRule(panelId) {
+  const name = `p${panelId}-hover`;
+  if (injectedRules.has(name)) return;
+  injectedRules.add(name);
+  const sheet = highlightStyleEl.sheet;
+  sheet.insertRule(
+    `::highlight(${name}) { outline: 2px solid var(--tok-hover); border-radius: 2px; }`,
+    sheet.cssRules.length,
+  );
+}
+/**
+ * Register one CSS Highlight per token offset.
+ * Name: `p{panelId}-t{tokenIndex}`
+ * Color: var(--tok-N) from CSS
+ */
+function paintHighlights(panelId, mirrorEl, offsets) {
+  if (!CSS.highlights) return;
+  const textNode = mirrorEl.firstChild;
+  if (!textNode) return;
+  for (let i = 0; i < offsets.length; i++) {
+    const off = offsets[i];
+    if (!off) continue;
+    const name = `p${panelId}-t${i}`;
+    ensureRule(name, tokenColorVar(i));
+    const range = new Range();
+    range.setStart(textNode, off.start);
+    range.setEnd(textNode, off.end);
+    CSS.highlights.set(name, new Highlight(range));
+  }
+}
+/**
+ * Remove all named highlights for a panel.
+ */
+function clearPanelHighlights(panelId, tokenCount) {
+  if (!CSS.highlights) return;
+  for (let i = 0; i < tokenCount; i++) {
+    CSS.highlights.delete(`p${panelId}-t${i}`);
+  }
+  CSS.highlights.delete(`p${panelId}-hover`);
+}
+// ─── Token table ──────────────────────────────────────────────────────────────
+/**
+ * @param {number} panelId
+ * @param {PanelState} state
+ * @param {number[]} token_ids
+ * @param {string[]} decoded
+ * @param {Array<{start:number,end:number}|null>} offsets
+ */
+function renderTable(panelId, state, token_ids, decoded, offsets) {
+  const { tbody, textMirror } = state;
+  tbody.innerHTML = "";
+  ensureHoverRule(panelId);
+  const fragment = document.createDocumentFragment();
+  for (let i = 0; i < token_ids.length; i++) {
+    const color = tokenColorVar(i);
+    const off = offsets[i];
+    const tr = document.createElement("tr");
+    tr.style.setProperty("--row-color", color);
+    tr.dataset.tokenIndex = String(i);
+    // Color swatch
+    const tdSwatch = document.createElement("td");
+    tdSwatch.className = "color-swatch";
+    const swatch = document.createElement("span");
+    swatch.style.background = color;
+    tdSwatch.appendChild(swatch);
+    // Index
+    const tdIdx = document.createElement("td");
+    tdIdx.textContent = String(i + 1);
+    // Token text
+    const tdTok = document.createElement("td");
+    tdTok.className = "tok-cell";
+    tdTok.textContent = decoded[i] ?? "";
+    // Token ID
+    const tdId = document.createElement("td");
+    tdId.className = "id-cell";
+    tdId.textContent = String(token_ids[i]);
+    tr.append(tdSwatch, tdIdx, tdTok, tdId);
+    // Hover: show distinct outline highlight on the mirrored text
+    if (off) {
+      tr.addEventListener("mouseenter", () => {
+        tr.classList.add("active");
+        if (!CSS.highlights) return;
+        const node = textMirror.firstChild;
+        if (!node) return;
+        const r = new Range();
+        r.setStart(node, off.start);
+        r.setEnd(node, off.end);
+        CSS.highlights.set(`p${panelId}-hover`, new Highlight(r));
+      });
+      tr.addEventListener("mouseleave", () => {
+        tr.classList.remove("active");
+        CSS.highlights?.delete(`p${panelId}-hover`);
+      });
+    }
+    fragment.appendChild(tr);
+  }
+  tbody.appendChild(fragment);
+}
+// ─── Utility ──────────────────────────────────────────────────────────────────
+function esc(str) {
+  return str
+    .replace(/&/g, "&amp;")
+    .replace(/</g, "&lt;")
+    .replace(/>/g, "&gt;")
+    .replace(/"/g, "&quot;");
+}

js/tokenizers.js ADDED Viewed

	@@ -0,0 +1,18 @@

+// Define list of tokenizers and their corresponding human-readable names
+export const TOKENIZER_OPTIONS = Object.freeze({
+  "smcproject/malayalam-bpe-tokenizer": "Malayalam BPE toklenizer",
+  "smcproject/malayalam-unigram-tokenizer": "Malayalam Unigram toklenizer",
+  "Xenova/gpt-4": "gpt-4 / gpt-3.5-turbo / text-embedding-ada-002",
+  "Xenova/text-davinci-003": "text-davinci-003 / text-davinci-002",
+  "Xenova/gpt-3": "gpt-3",
+  "Xenova/grok-1-tokenizer": "Grok-1",
+  "Xenova/claude-tokenizer": "Claude",
+  "Xenova/mistral-tokenizer-v3": "Mistral v3",
+  "Xenova/mistral-tokenizer-v1": "Mistral v1",
+  "Xenova/gemma-tokenizer": "Gemma",
+  "Xenova/llama-3-tokenizer": "Llama 3",
+  "Xenova/llama-tokenizer": "LLaMA / Llama 2",
+  "Xenova/c4ai-command-r-v01-tokenizer": "Cohere Command-R",
+  "Xenova/t5-small": "T5",
+  "Xenova/bert-base-cased": "bert-base-cased",
+});

js/worker.js ADDED Viewed

	@@ -0,0 +1,69 @@

+// Although not strictly necessary, we delegate the tokenization to a worker thread to avoid
+// any potential issues with the tokenizer blocking the main thread (especially for large inputs).
+import { AutoTokenizer } from "https://cdn.jsdelivr.net/npm/@huggingface/transformers@3.2.1";
+// This is a map of all the tokenizer instances that we have loaded.
+// model_id -> promise that resolves to tokenizer
+const TOKENIZER_MAPPINGS = new Map();
+// Listen for messages from the main thread
+self.addEventListener("message", async (event) => {
+  const { model_id, text } = event.data;
+  // Only load the tokenizer if it hasn't been loaded yet
+  let tokenizerPromise = TOKENIZER_MAPPINGS.get(model_id);
+  if (!tokenizerPromise) {
+    // For visualization purposes, we may need to modify the tokenizer slightly
+    tokenizerPromise = AutoTokenizer.from_pretrained(model_id).then(
+      (tokenizer) => {
+        // NOTE: We just remove the StripDecoder from the llama tokenizer
+        const tokenizer_class = (
+          tokenizer._tokenizer_config?.tokenizer_class ?? ""
+        ).replace(/Fast$/, "");
+        switch (tokenizer_class) {
+          case "LlamaTokenizer":
+          case "Grok1Tokenizer":
+            // tokenizer.decoder.decoders.at(-1).constructor.name === 'StripDecoder'
+            tokenizer.decoder.decoders.pop();
+            break;
+          case "T5Tokenizer":
+            tokenizer.decoder.addPrefixSpace = false;
+            break;
+        }
+        return tokenizer;
+      },
+    );
+    TOKENIZER_MAPPINGS.set(model_id, tokenizerPromise);
+  }
+  const tokenizer = await tokenizerPromise;
+  // Tokenize the input text
+  const token_ids = tokenizer.encode(text);
+  // Decode the token IDs back to text
+  let decoded = token_ids.map((x) => tokenizer.decode([x]));
+  // Minor post-processing for visualization purposes
+  let margins = [];
+  switch (tokenizer.constructor.name) {
+    case "BertTokenizer":
+      margins = decoded.map((x, i) => (i === 0 || x.startsWith("##") ? 0 : 8));
+      decoded = decoded.map((x) => x.replace("##", ""));
+      break;
+    case "T5Tokenizer":
+      if (decoded.length > 0 && decoded !== " ") {
+        decoded[0] = decoded[0].replace(/^ /, "");
+      }
+      break;
+  }
+  // Send the output back to the main thread
+  self.postMessage({
+    token_ids,
+    decoded,
+    margins,
+  });
+});

style.css CHANGED Viewed

@@ -1,28 +1,412 @@
 body {
-	padding: 2rem;
-	font-family: -apple-system, BlinkMacSystemFont, "Arial", sans-serif;
 }
-h1 {
-	font-size: 16px;
-	margin-top: 0;
 }
-p {
-	color: rgb(107, 114, 128);
-	font-size: 15px;
-	margin-bottom: 10px;
-	margin-top: 5px;
 }
-.card {
-	max-width: 620px;
-	margin: 0 auto;
-	padding: 16px;
-	border: 1px solid lightgray;
-	border-radius: 16px;
 }
-.card p:last-child {
-	margin-bottom: 0;
 }

+/* ─── Design tokens ─────────────────────────────────────────────────────── */
+:root {
+  color-scheme: light dark;
+  --radius: 8px;
+  --radius-sm: 4px;
+  --font-sans: system-ui, -apple-system, sans-serif;
+  --font-mono: ui-monospace, "Cascadia Code", "Fira Code", monospace;
+  --shadow:
+    0 1px 4px color-mix(in srgb, CanvasText 8%, transparent), 0 2px 12px
+    color-mix(in srgb, CanvasText 5%, transparent);
+  /* Foreground color rendered over token highlight backgrounds */
+  --tok-text: CanvasText;
+  /* Hover outline color for highlighted token ranges */
+  --tok-hover: AccentColor;
+  /* Token color palette — 12 distinct hues in oklch, light-mode values.
+     Dark-mode overrides follow in the @media block below. */
+  --tok-0: oklch(80% 0.13 30);
+  --tok-1: oklch(82% 0.13 80);
+  --tok-2: oklch(80% 0.13 140);
+  --tok-3: oklch(80% 0.13 195);
+  --tok-4: oklch(80% 0.13 250);
+  --tok-5: oklch(80% 0.13 300);
+  --tok-6: oklch(80% 0.13 340);
+  --tok-7: oklch(82% 0.13 55);
+  --tok-8: oklch(80% 0.13 165);
+  --tok-9: oklch(80% 0.13 220);
+  --tok-10: oklch(80% 0.13 270);
+  --tok-11: oklch(80% 0.13 320);
+}
+@media (prefers-color-scheme: dark) {
+  :root {
+    /* Slightly higher lightness so highlights read on dark Canvas */
+    --tok-0: oklch(68% 0.16 30);
+    --tok-1: oklch(70% 0.16 80);
+    --tok-2: oklch(68% 0.16 140);
+    --tok-3: oklch(68% 0.16 195);
+    --tok-4: oklch(68% 0.16 250);
+    --tok-5: oklch(68% 0.16 300);
+    --tok-6: oklch(68% 0.16 340);
+    --tok-7: oklch(70% 0.16 55);
+    --tok-8: oklch(68% 0.16 165);
+    --tok-9: oklch(68% 0.16 220);
+    --tok-10: oklch(68% 0.16 270);
+    --tok-11: oklch(68% 0.16 320);
+  }
+}
+/* ─── Reset ─────────────────────────────────────────────────────────────── */
+*,
+*::before,
+*::after {
+  box-sizing: border-box;
+  margin: 0;
+  padding: 0;
+}
+/* ─── Base ──────────────────────────────────────────────────────────────── */
 body {
+  font-family: var(--font-sans);
+  background: Canvas;
+  color: CanvasText;
+  line-height: 1.6;
+  min-height: 100dvh;
+  padding-block-end: 4rem;
+}
+/* ─── Header ────────────────────────────────────────────────────────────── */
+header {
+  background: Canvas;
+  padding: 1.25rem 2rem;
+  max-width: 1400px;
+  margin-inline: auto;
+  h1 {
+    font-size: 1.35rem;
+    font-weight: 700;
+    letter-spacing: -0.01em;
+  }
+  p {
+    font-size: 0.875rem;
+    color: GrayText;
+    margin-block-start: 0.2rem;
+  }
 }
+/* ─── Main layout ───────────────────────────────────────────────────────── */
+main {
+  max-width: 1400px;
+  margin-inline: auto;
+  padding: 1.5rem 2rem;
+  display: grid;
+  gap: 1.25rem;
 }
+/* ─── Input section ─────────────────────────────────────────────────────── */
+.input-section {
+  display: grid;
+  gap: 0.5rem;
+  label {
+    font-size: 0.8rem;
+    font-weight: 600;
+    text-transform: uppercase;
+    letter-spacing: 0.05em;
+    color: GrayText;
+  }
+  textarea {
+    font-family: var(--font-mono);
+    font-size: 1.1rem;
+    line-height: 1.7;
+    background: Field;
+    color: FieldText;
+    border: 1px solid ButtonBorder;
+    border-radius: var(--radius);
+    padding: 0.75rem 1rem;
+    resize: vertical;
+    width: 100%;
+    transition: border-color 0.15s;
+    &:focus {
+      outline: none;
+      border-color: AccentColor;
+      box-shadow: 0 0 0 3px color-mix(in srgb, AccentColor 20%, transparent);
+    }
+  }
 }
+/* ─── Add tokenizer row ─────────────────────────────────────────────────── */
+.add-tokenizer-section {
+  display: flex;
+  gap: 0.5rem;
+  align-items: center;
+  flex-wrap: wrap;
+  select {
+    font-family: var(--font-sans);
+    font-size: 0.9rem;
+    background: Field;
+    color: FieldText;
+    border: 1px solid ButtonBorder;
+    border-radius: var(--radius-sm);
+    padding: 0.45rem 0.75rem;
+    flex: 1 1 280px;
+    max-width: 480px;
+    cursor: pointer;
+    transition: border-color 0.15s;
+    &:focus {
+      outline: none;
+      border-color: AccentColor;
+    }
+  }
+  button {
+    font-family: var(--font-sans);
+    font-size: 0.9rem;
+    font-weight: 600;
+    background: AccentColor;
+    color: AccentColorText;
+    border: none;
+    border-radius: var(--radius-sm);
+    padding: 0.45rem 1.1rem;
+    cursor: pointer;
+    transition: opacity 0.15s;
+    white-space: nowrap;
+    &:hover:not(:disabled) {
+      opacity: 0.85;
+    }
+    &:disabled {
+      opacity: 0.4;
+      cursor: not-allowed;
+    }
+  }
+}
+/* ─── Empty state ───────────────────────────────────────────────────────── */
+.empty-state {
+  text-align: center;
+  color: GrayText;
+  font-size: 0.9rem;
+  padding: 2.5rem 1rem;
+  border: 1.5px dashed ButtonBorder;
+  border-radius: var(--radius);
+  background: Canvas;
+  &.hidden {
+    display: none;
+  }
+}
+/* ─── Panels grid ───────────────────────────────────────────────────────── */
+#panels {
+  display: grid;
+  grid-template-columns: repeat(auto-fit, minmax(340px, 1fr));
+  gap: 1rem;
+  align-items: start;
+}
+/* ─── Individual tokenizer panel ────────────────────────────────────────── */
+.panel {
+  background: Canvas;
+  border: 1px solid ButtonBorder;
+  border-radius: var(--radius);
+  box-shadow: var(--shadow);
+  display: grid;
+  grid-template-rows: auto auto auto 1fr;
+  overflow: hidden;
+  /* Panel header bar */
+  .panel-header {
+    display: flex;
+    align-items: center;
+    justify-content: space-between;
+    gap: 0.5rem;
+    padding: 0.65rem 1rem;
+    background: ButtonFace;
+    border-block-end: 1px solid ButtonBorder;
+    .panel-title {
+      font-size: 0.85rem;
+      font-weight: 600;
+      color: ButtonText;
+      text-decoration: none;
+      overflow: hidden;
+      text-overflow: ellipsis;
+      white-space: nowrap;
+    }
+    .remove-btn {
+      font-size: 0.75rem;
+      font-weight: 600;
+      color: GrayText;
+      background: none;
+      border: 1px solid ButtonBorder;
+      border-radius: var(--radius-sm);
+      padding: 0.2rem 0.5rem;
+      cursor: pointer;
+      flex-shrink: 0;
+      transition:
+        color 0.12s,
+        border-color 0.12s;
+      &:hover {
+        color: LinkText;
+        border-color: LinkText;
+      }
+    }
+  }
+  /* Text mirror with highlights */
+  .panel-text {
+    font-family: var(--font-mono);
+    font-size: 1.05rem;
+    line-height: 1.8;
+    padding: 0.85rem 1rem;
+    border-block-end: 1px solid ButtonBorder;
+    white-space: pre-wrap;
+    word-break: break-all;
+    min-height: 3.5rem;
+  }
+  /* Stats bar */
+  .panel-stats {
+    display: flex;
+    gap: 1.5rem;
+    padding: 0.45rem 1rem;
+    background: ButtonFace;
+    border-block-end: 1px solid ButtonBorder;
+    font-size: 0.78rem;
+    color: GrayText;
+    flex-wrap: wrap;
+    strong {
+      color: CanvasText;
+      font-weight: 600;
+    }
+  }
+  /* Token table */
+  .panel-table-wrap {
+    overflow-y: auto;
+    max-height: 360px;
+  }
+  table {
+    width: 100%;
+    border-collapse: collapse;
+    font-size: 0.82rem;
+    thead {
+      position: sticky;
+      top: 0;
+      background: ButtonFace;
+      z-index: 1;
+      th {
+        text-align: left;
+        padding: 0.4rem 0.75rem;
+        font-weight: 600;
+        font-size: 0.72rem;
+        text-transform: uppercase;
+        letter-spacing: 0.05em;
+        color: GrayText;
+        border-block-end: 1px solid ButtonBorder;
+      }
+    }
+    tbody tr {
+      cursor: default;
+      transition: background 0.08s;
+      &:hover {
+        background: color-mix(in srgb, ButtonFace 60%, Canvas);
+      }
+      &.active {
+        background: color-mix(
+          in srgb,
+          var(--row-color, AccentColor) 18%,
+          Canvas
+        );
+      }
+      td {
+        padding: 0.3rem 0.75rem;
+        border-block-end: 1px solid
+          color-mix(in srgb, ButtonBorder 60%, transparent);
+        vertical-align: middle;
+        &.tok-cell {
+          font-family: var(--font-mono);
+          font-size: 0.88rem;
+        }
+        &.id-cell {
+          font-family: var(--font-mono);
+          color: GrayText;
+          font-size: 0.8rem;
+          text-align: right;
+        }
+        &.color-swatch {
+          width: 18px;
+          padding-inline: 0.5rem;
+          span {
+            display: block;
+            width: 10px;
+            height: 10px;
+            border-radius: 50%;
+            background: var(--row-color, ButtonBorder);
+          }
+        }
+      }
+    }
+  }
+  /* Loading state */
+  &.loading {
+    .panel-text,
+    .panel-stats,
+    .panel-table-wrap {
+      opacity: 0.4;
+      pointer-events: none;
+    }
+  }
+  .panel-loading {
+    display: none;
+    align-items: center;
+    gap: 0.5rem;
+    padding: 0.6rem 1rem;
+    font-size: 0.8rem;
+    color: GrayText;
+    .spinner {
+      width: 14px;
+      height: 14px;
+      border: 2px solid ButtonBorder;
+      border-top-color: AccentColor;
+      border-radius: 50%;
+      animation: spin 0.7s linear infinite;
+      flex-shrink: 0;
+    }
+  }
+  &.loading .panel-loading {
+    display: flex;
+  }
 }
+@keyframes spin {
+  to {
+    transform: rotate(360deg);
+  }
 }