File size: 1,973 Bytes
3711808 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 | """InDepth tokenizer — word-level vocab + hashed char-trigram subword features.
Rules (mirrored bit-exact in the browser engine embedded in InDepth.html):
- words: lowercase, split on [^a-z0-9]+, drop empties, max 16 tokens
- unknown words -> <unk> id 1 (trigram features still apply)
- char trigrams over "<word>", capped at 16 per word
- bucket id = 1 + (FNV-1a 32-bit of the trigram mod (buckets-1)); 0 = null/pad
"""
import json
import re
from pathlib import Path
SUBWORD_SLOTS = 16
UNK = 1
def fnv1a(s):
h = 2166136261
for ch in s:
h ^= ord(ch)
h = (h * 16777619) & 0xFFFFFFFF
return h
def word_trigram_buckets(word, buckets):
tagged = "<" + word + ">"
grams = [tagged[i:i + 3] for i in range(len(tagged) - 2)][:SUBWORD_SLOTS]
ids = [0] * SUBWORD_SLOTS
for i, g in enumerate(grams):
ids[i] = 1 + (fnv1a(g) % (buckets - 1))
return ids
def encode_prompt(prompt, stoi, buckets, max_len):
"""-> (tokens, subwords) flat lists of length max_len and max_len * SUBWORD_SLOTS."""
words = [t for t in re.split(r"[^a-z0-9]+", prompt.lower()) if t][:max_len]
tokens = [stoi.get(w, UNK) for w in words]
tri = [[0] * SUBWORD_SLOTS for _ in range(max_len)]
for i, w in enumerate(words):
tri[i] = word_trigram_buckets(w, buckets)
return tokens, tri
class IndepthTokenizer:
def __init__(self, model_dir):
vocab = json.loads((Path(model_dir) / "vocab.json").read_text())
self.stoi = {w: i for i, w in enumerate(vocab["vocab"])}
self.max_len = vocab["max_len"]
self.buckets = vocab.get("subword_buckets", 4096)
def batch(self, prompts):
rows, subs = [], []
for prompt in prompts:
tokens, tri = encode_prompt(prompt, self.stoi, self.buckets, self.max_len)
rows.append(tokens + [0] * (self.max_len - len(tokens)))
subs.append(tri) # (max_len, SUBWORD_SLOTS) per prompt
return rows, subs
|