"""HTML rendering for the demo: coloured tokens by source, legend, stats, replay payload.""" from __future__ import annotations import html import json # token source -> (legend label, tooltip) SOURCES = { "copy": ("Copied from context", "copied from the prompt or earlier output by n-gram lookup, then verified"), "head": ("Exit head (layer 4)", "drafted by the shallow exit head, then verified"), "star": ("Head top-4 candidate", "an alternative from the head's top-4, then verified"), "draft": ("Draft model", "drafted by a separate model, then verified"), "model": ("Full model", "produced by the full model (correction or bonus token of a verification)"), } class Detokenizer: """Incremental decoding into (text, source) pieces. Tokens that end inside a multi-byte character are held back until the character is complete; a piece takes the source of its first token.""" def __init__(self, tokenizer): self.tok = tokenizer self.ids: list[int] = [] self.text = "" self.pending: list[str] = [] def _emit(self, final: bool) -> list[tuple[str, str]]: cur = self.tok.decode(self.ids) if cur.endswith("�") and not final: return [] piece = cur[len(self.text):] src = self.pending[0] if self.pending else "model" self.text, self.pending = cur, [] return [(piece, src)] if piece else [] def feed(self, ids: list[int], sources: list[str]) -> list[tuple[str, str]]: out = [] for t, s in zip(ids, sources): self.ids.append(int(t)) self.pending.append(s) out += self._emit(final=False) return out def flush(self) -> list[tuple[str, str]]: return self._emit(final=True) if self.pending else [] def tokens_html(pieces, colored: bool = True, cursor: bool = False) -> str: out = [] for text, src in pieces: t = html.escape(text) if colored and src != "model" and src in SOURCES: out.append(f'{t}') else: out.append(t) if cursor: out.append('') return f'