"""InDepth tokenizer — word-level vocab + hashed char-trigram subword features. Rules (mirrored bit-exact in the browser engine embedded in InDepth.html): - words: lowercase, split on [^a-z0-9]+, drop empties, max 16 tokens - unknown words -> id 1 (trigram features still apply) - char trigrams over "", capped at 16 per word - bucket id = 1 + (FNV-1a 32-bit of the trigram mod (buckets-1)); 0 = null/pad """ import json import re from pathlib import Path SUBWORD_SLOTS = 16 UNK = 1 def fnv1a(s): h = 2166136261 for ch in s: h ^= ord(ch) h = (h * 16777619) & 0xFFFFFFFF return h def word_trigram_buckets(word, buckets): tagged = "<" + word + ">" grams = [tagged[i:i + 3] for i in range(len(tagged) - 2)][:SUBWORD_SLOTS] ids = [0] * SUBWORD_SLOTS for i, g in enumerate(grams): ids[i] = 1 + (fnv1a(g) % (buckets - 1)) return ids def encode_prompt(prompt, stoi, buckets, max_len): """-> (tokens, subwords) flat lists of length max_len and max_len * SUBWORD_SLOTS.""" words = [t for t in re.split(r"[^a-z0-9]+", prompt.lower()) if t][:max_len] tokens = [stoi.get(w, UNK) for w in words] tri = [[0] * SUBWORD_SLOTS for _ in range(max_len)] for i, w in enumerate(words): tri[i] = word_trigram_buckets(w, buckets) return tokens, tri class IndepthTokenizer: def __init__(self, model_dir): vocab = json.loads((Path(model_dir) / "vocab.json").read_text()) self.stoi = {w: i for i, w in enumerate(vocab["vocab"])} self.max_len = vocab["max_len"] self.buckets = vocab.get("subword_buckets", 4096) def batch(self, prompts): rows, subs = [], [] for prompt in prompts: tokens, tri = encode_prompt(prompt, self.stoi, self.buckets, self.max_len) rows.append(tokens + [0] * (self.max_len - len(tokens))) subs.append(tri) # (max_len, SUBWORD_SLOTS) per prompt return rows, subs