indepth-10M / tokenizer.py
DexagonAI's picture
Upload folder using huggingface_hub
3711808 verified
Raw History Blame Contribute Delete
1.97 kB
"""InDepth tokenizer — word-level vocab + hashed char-trigram subword features.
Rules (mirrored bit-exact in the browser engine embedded in InDepth.html):
- words: lowercase, split on [^a-z0-9]+, drop empties, max 16 tokens
- unknown words -> <unk> id 1 (trigram features still apply)
- char trigrams over "<word>", capped at 16 per word
- bucket id = 1 + (FNV-1a 32-bit of the trigram mod (buckets-1)); 0 = null/pad
"""
import json
import re
from pathlib import Path
SUBWORD_SLOTS = 16
UNK = 1
def fnv1a(s):
h = 2166136261
for ch in s:
h ^= ord(ch)
h = (h * 16777619) & 0xFFFFFFFF
return h
def word_trigram_buckets(word, buckets):
tagged = "<" + word + ">"
grams = [tagged[i:i + 3] for i in range(len(tagged) - 2)][:SUBWORD_SLOTS]
ids = [0] * SUBWORD_SLOTS
for i, g in enumerate(grams):
ids[i] = 1 + (fnv1a(g) % (buckets - 1))
return ids
def encode_prompt(prompt, stoi, buckets, max_len):
"""-> (tokens, subwords) flat lists of length max_len and max_len * SUBWORD_SLOTS."""
words = [t for t in re.split(r"[^a-z0-9]+", prompt.lower()) if t][:max_len]
tokens = [stoi.get(w, UNK) for w in words]
tri = [[0] * SUBWORD_SLOTS for _ in range(max_len)]
for i, w in enumerate(words):
tri[i] = word_trigram_buckets(w, buckets)
return tokens, tri
class IndepthTokenizer:
def __init__(self, model_dir):
vocab = json.loads((Path(model_dir) / "vocab.json").read_text())
self.stoi = {w: i for i, w in enumerate(vocab["vocab"])}
self.max_len = vocab["max_len"]
self.buckets = vocab.get("subword_buckets", 4096)
def batch(self, prompts):
rows, subs = [], []
for prompt in prompts:
tokens, tri = encode_prompt(prompt, self.stoi, self.buckets, self.max_len)
rows.append(tokens + [0] * (self.max_len - len(tokens)))
subs.append(tri) # (max_len, SUBWORD_SLOTS) per prompt
return rows, subs