Download tokenizer.py from DexagonAI/indepth-10M: direct link, hf CLI and curl.
- Browser
- Download file 1.97 kB
-
https://huggingface.co/DexagonAI/indepth-10M/resolve/main/tokenizer.py
- Command line
-
hf download hf://DexagonAI/indepth-10M/tokenizer.py
-
curl -L -o tokenizer.py https://huggingface.co/DexagonAI/indepth-10M/resolve/main/tokenizer.py
1.97 kB
| """InDepth tokenizer — word-level vocab + hashed char-trigram subword features. | |
| Rules (mirrored bit-exact in the browser engine embedded in InDepth.html): | |
| - words: lowercase, split on [^a-z0-9]+, drop empties, max 16 tokens | |
| - unknown words -> <unk> id 1 (trigram features still apply) | |
| - char trigrams over "<word>", capped at 16 per word | |
| - bucket id = 1 + (FNV-1a 32-bit of the trigram mod (buckets-1)); 0 = null/pad | |
| """ | |
| import json | |
| import re | |
| from pathlib import Path | |
| SUBWORD_SLOTS = 16 | |
| UNK = 1 | |
| def fnv1a(s): | |
| h = 2166136261 | |
| for ch in s: | |
| h ^= ord(ch) | |
| h = (h * 16777619) & 0xFFFFFFFF | |
| return h | |
| def word_trigram_buckets(word, buckets): | |
| tagged = "<" + word + ">" | |
| grams = [tagged[i:i + 3] for i in range(len(tagged) - 2)][:SUBWORD_SLOTS] | |
| ids = [0] * SUBWORD_SLOTS | |
| for i, g in enumerate(grams): | |
| ids[i] = 1 + (fnv1a(g) % (buckets - 1)) | |
| return ids | |
| def encode_prompt(prompt, stoi, buckets, max_len): | |
| """-> (tokens, subwords) flat lists of length max_len and max_len * SUBWORD_SLOTS.""" | |
| words = [t for t in re.split(r"[^a-z0-9]+", prompt.lower()) if t][:max_len] | |
| tokens = [stoi.get(w, UNK) for w in words] | |
| tri = [[0] * SUBWORD_SLOTS for _ in range(max_len)] | |
| for i, w in enumerate(words): | |
| tri[i] = word_trigram_buckets(w, buckets) | |
| return tokens, tri | |
| class IndepthTokenizer: | |
| def __init__(self, model_dir): | |
| vocab = json.loads((Path(model_dir) / "vocab.json").read_text()) | |
| self.stoi = {w: i for i, w in enumerate(vocab["vocab"])} | |
| self.max_len = vocab["max_len"] | |
| self.buckets = vocab.get("subword_buckets", 4096) | |
| def batch(self, prompts): | |
| rows, subs = [], [] | |
| for prompt in prompts: | |
| tokens, tri = encode_prompt(prompt, self.stoi, self.buckets, self.max_len) | |
| rows.append(tokens + [0] * (self.max_len - len(tokens))) | |
| subs.append(tri) # (max_len, SUBWORD_SLOTS) per prompt | |
| return rows, subs | |