src/lib/model/tokenizer.ts

/**
 * Tokenizer helpers: char-offset reconstruction for detector highlighting.
 *
 * Byte-level BPE tokens don't decode well in isolation, so offsets are
 * derived from progressive decoding of prefixes (O(n^2) in token count, fine
 * for demo-sized texts).
 */

import type { TokenizedText } from '../detectors/kirchenbauer-detector';

export interface MinimalTokenizer {
  encode(text: string, options?: { add_special_tokens?: boolean }): number[];
  decode(ids: number[], options?: { skip_special_tokens?: boolean }): string;
}

export function tokenizeWithOffsets(tokenizer: MinimalTokenizer, text: string): TokenizedText {
  const ids = tokenizer.encode(text, { add_special_tokens: false });
  const tokenIds = Array.from(ids, Number);
  const offsets: Array<[number, number]> = [];
  const labels: string[] = [];
  let prevLen = 0;
  for (let i = 0; i < tokenIds.length; i++) {
    const decoded = tokenizer.decode(tokenIds.slice(0, i + 1), { skip_special_tokens: false });
    const end = decoded.length;
    offsets.push([prevLen, end]);
    labels.push(decoded.slice(prevLen, end));
    prevLen = end;
  }
  return { tokenIds, offsets, labels };
}