Confidence-gated ghost text for coding prompts: Kneser-Ney compile, 4.5 MB browser export + zero-dep runtime, held-out precision/keystrokes-saved benchmarks
4113449 verified | // Word-level tokenizer with GPT-2 byte-level pre-tokenization, matching tokenizer.json | |
| // (NFKC + lowercase, add_prefix_space, GPT-2 split regex). No Node or DOM APIs, so it | |
| // is shared by model.ts (Node/Bun) and web/autocomplete.ts (browser). | |
| function bytesToUnicode(): Map<number, string> { | |
| const bs: number[] = []; | |
| for (let b = 33; b <= 126; b++) bs.push(b); | |
| for (let b = 161; b <= 172; b++) bs.push(b); | |
| for (let b = 174; b <= 255; b++) bs.push(b); | |
| const cs = [...bs]; | |
| let n = 0; | |
| for (let b = 0; b < 256; b++) { | |
| if (!bs.includes(b)) { | |
| bs.push(b); | |
| cs.push(256 + n++); | |
| } | |
| } | |
| return new Map(bs.map((b, i) => [b, String.fromCodePoint(cs[i])])); | |
| } | |
| const BYTE_ENC = bytesToUnicode(); | |
| const BYTE_DEC = new Map([...BYTE_ENC].map(([b, c]) => [c, b])); | |
| const GPT2_SPLIT = /'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+/gu; | |
| const encoder = new TextEncoder(); | |
| const decoder = new TextDecoder(); | |
| /** Byte-level pre-tokens ("Ġword" style), exactly as the Python tokenizer splits text. */ | |
| export function preTokenize(text: string): string[] { | |
| let norm = text.normalize("NFKC").toLowerCase(); | |
| if (norm && !/^\s/.test(norm)) norm = " " + norm; // ByteLevel add_prefix_space | |
| return (norm.match(GPT2_SPLIT) ?? []).map((piece) => [...encoder.encode(piece)].map((b) => BYTE_ENC.get(b)!).join("")); | |
| } | |
| /** Byte-level piece(s) back to text. */ | |
| export function decodePieces(pieces: string): string { | |
| return decoder.decode(new Uint8Array([...pieces].map((c) => BYTE_DEC.get(c) ?? 63))); | |
| } | |
| export class WordTokenizer { | |
| vocab: Map<string, number>; | |
| inverse: string[]; | |
| special: Set<number>; | |
| unkId: number; | |
| bosId: number; | |
| constructor(pieces: string[], unkId: number, bosId: number, special: Iterable<number>) { | |
| this.unkId = unkId; | |
| this.bosId = bosId; | |
| this.inverse = pieces; | |
| this.vocab = new Map(pieces.map((p, i) => [p, i])); | |
| this.special = new Set(special); | |
| } | |
| /** tokenizer.json (WordLevel model, BOS template post-processor) -> tokenizer. */ | |
| static fromJson(json: any): WordTokenizer { | |
| const pieces: string[] = []; | |
| for (const [tok, id] of Object.entries<number>(json.model.vocab)) pieces[id] = tok; | |
| const bos = json.post_processor?.single?.find((p: any) => p.SpecialToken)?.SpecialToken?.id; | |
| const special = (json.added_tokens ?? []).filter((t: any) => t.special).map((t: any) => t.id); | |
| return new WordTokenizer(pieces, json.model.vocab[json.model.unk_token], json.model.vocab[bos], special); | |
| } | |
| idOf(piece: string): number { | |
| return this.vocab.get(piece) ?? this.unkId; | |
| } | |
| encode(text: string): number[] { | |
| return [this.bosId, ...preTokenize(text).map((p) => this.idOf(p))]; | |
| } | |
| decode(ids: number[]): string { | |
| return decodePieces(ids.filter((id) => !this.special.has(id)).map((id) => this.inverse[id]).join("")); | |
| } | |
| } | |