""" tokenizer_train.py — Word + Character hybrid tokenizer ====================================================== Each word token = one whole word. Everything else = single characters. No subwords. No merges. No BPE. No external dependencies (for base). Encode: greedy longest-match at word boundaries, character fallback. Decode: concatenate tokens directly (spaces are their own token). Corpus-aware mode (--corpus / --hf-dataset): Scans data for the most useful words and adds them to the vocab in both lowercase and Sentence case forms. Usage: # Build base tokenizer (no corpus) python tokenizer_train.py --output colm_tokenizer.json # Build with corpus from plain text file(s) python tokenizer_train.py --corpus data.txt --output colm_tokenizer.json python tokenizer_train.py --corpus "*.txt" --output colm_tokenizer.json # Build with corpus from HuggingFace dataset python tokenizer_train.py --hf-dataset wikitext --hf-config wikitext-103-raw-v1 \\ --hf-split train --hf-field text --output colm_tokenizer.json # Use in training script from tokenizer_train import COLMTokenizer tok = COLMTokenizer.load("colm_tokenizer.json") ids = tok.encode("The Machine God") text = tok.decode(ids) """ import json import re import argparse import glob from collections import Counter from pathlib import Path # ============================================================================= # TOKEN DEFINITIONS # ============================================================================= SPECIAL_TOKENS = ["", "", "", ""] # Whitespace (encoded as visible tokens so decode reconstructs perfectly) WHITESPACE_TOKENS = [" ", "\n", "\t"] # Top 100 English words — each is one token WORD_TOKENS = [ "the", "be", "to", "of", "and", "in", "that", "have", "it", "for", "not", "on", "with", "he", "as", "you", "do", "at", "this", "but", "his", "by", "from", "they", "we", "say", "her", "she", "or", "an", "will", "my", "one", "all", "would", "there", "their", "what", "so", "up", "out", "if", "about", "who", "get", "which", "go", "me", "when", "make", "can", "like", "time", "no", "just", "him", "know", "take", "people", "into", "year", "your", "good", "some", "could", "them", "see", "other", "than", "then", "now", "look", "only", "come", "it's", "over", "think", "also", "back", "after", "use", "two", "how", "our", "work", "first", "well", "way", "even", "new", "want", "because", "any", "these", "give", "day", "most", "us", ] # Individual letters UPPER = list("ABCDEFGHIJKLMNOPQRSTUVWXYZ") LOWER = list("abcdefghijklmnopqrstuvwxyz") # Digits DIGITS = list("0123456789") # Symbols (keyboard + corpus unicode) SYMBOLS = list(set([ "+", "_", "=", ")", "(", "*", "&", "^", "%", "$", "£", '"', "!", "`", "¬", "\\", "|", "<", ">", "/", "?", ":", "@", "~", "#", "'", ";", ",", ".", # Extended unicode (add corpus-specific symbols here) "\u2013", # – en-dash "\u2014", # — em-dash "\u2018", # ' left single curly quote "\u2019", # ' right single curly quote "-", ])) # ============================================================================= # CORPUS SCANNING # ============================================================================= # Matches whole words including contractions (don't, it's, they've, etc.) # Apostrophe variants: straight ' and curly ' WORD_RE = re.compile(r"[a-zA-Z]+(?:['\u2019][a-zA-Z]+)*") # Minimum frequency to consider a "long" word (filters OCR junk / typos) MIN_FREQ_FOR_LENGTH = 3 def iter_corpus_lines(corpus_path=None, hf_dataset=None, hf_config=None, hf_split="train", hf_field="text"): """Yield text lines from either local file(s) or a HuggingFace dataset. Args: corpus_path: Glob pattern or single path for local .txt files hf_dataset: HuggingFace dataset name (e.g. "wikitext") hf_config: HuggingFace config name (e.g. "wikitext-103-raw-v1") hf_split: Dataset split to use hf_field: Column name containing text """ if corpus_path is not None: # Local text file(s) — supports glob patterns paths = sorted(glob.glob(corpus_path)) if not paths: raise FileNotFoundError(f"No files matched: {corpus_path}") print(f"[corpus] Reading {len(paths)} file(s) from: {corpus_path}") for p in paths: with open(p, "r", encoding="utf-8", errors="replace") as f: for line in f: line = line.rstrip("\n\r") if line: yield line elif hf_dataset is not None: try: from datasets import load_dataset except ImportError: raise ImportError( "HuggingFace datasets not installed. Run: " "pip install datasets --break-system-packages" ) print(f"[corpus] Loading HuggingFace dataset: {hf_dataset}" f" (config={hf_config}, split={hf_split}, field={hf_field})") ds = load_dataset(hf_dataset, hf_config, split=hf_split, trust_remote_code=True) for row in ds: text = row.get(hf_field, "") if text: yield text else: return # No corpus — base tokenizer only def count_corpus_words(lines_iter): """Count all words in the corpus. Returns a Counter of lowercase words. Contractions are preserved as-is (don't, it's, etc.) — lowercased. """ counts = Counter() n_lines = 0 for line in lines_iter: words = WORD_RE.findall(line) # Normalise: lowercase, and normalise curly apostrophes to straight normalised = [w.lower().replace("\u2019", "'") for w in words] counts.update(normalised) n_lines += 1 if n_lines % 500_000 == 0: print(f" ...scanned {n_lines:,} lines, {sum(counts.values()):,} tokens") print(f"[corpus] Done: {n_lines:,} lines, {len(counts):,} unique words, " f"{sum(counts.values()):,} total tokens") return counts def select_corpus_words(counts, existing_words, n_freq=50, n_long=50): """Select corpus words for the tokenizer. Strategy: 1. Top 100 by frequency → take top 50 (excluding existing vocab) 2. Top 100 by length (min freq threshold) → take top 50 by length 3. Deduplicate across both lists 4. Return combined list Args: counts: Counter of lowercase words existing_words: Set of words already in the tokenizer (lowercase) n_freq: How many frequency-sorted words to keep n_long: How many length-sorted words to keep Returns: (freq_words, long_words, combined) — all lowercase, deduplicated """ # Single-char words are already covered by the character tokens eligible = {w: c for w, c in counts.items() if len(w) > 1 and w not in existing_words} # --- Pool 1: Top by frequency --- freq_sorted = sorted(eligible.items(), key=lambda x: x[1], reverse=True) freq_pool = [w for w, _ in freq_sorted[:100]] freq_selected = freq_pool[:n_freq] # --- Pool 2: Top by length (must meet minimum frequency) --- long_eligible = {w: c for w, c in eligible.items() if c >= MIN_FREQ_FOR_LENGTH} long_sorted = sorted(long_eligible.items(), key=lambda x: len(x[0]), reverse=True) long_pool = [w for w, _ in long_sorted[:100]] long_selected = long_pool[:n_long] # --- Combine and deduplicate (preserve selection order) --- seen = set() combined = [] # Frequency words first (they're higher-value for compression) for w in freq_selected: if w not in seen: combined.append(w) seen.add(w) # Then length words for w in long_selected: if w not in seen: combined.append(w) seen.add(w) print(f"[vocab] Selected {len(freq_selected)} by frequency, " f"{len(long_selected)} by length") print(f"[vocab] Combined (deduplicated): {len(combined)} corpus words") # Show what we picked if freq_selected: print(f"[vocab] Top 5 by freq: {freq_selected[:5]}") if long_selected: print(f"[vocab] Top 5 by length: {long_selected[:5]} " f"(lengths: {[len(w) for w in long_selected[:5]]})") return freq_selected, long_selected, combined def make_sentence_case(word): """Capitalise first letter only — as it would appear at start of sentence. Handles contractions: don't → Don't, it's → It's """ if not word: return word return word[0].upper() + word[1:] # ============================================================================= # TOKENIZER CLASS # ============================================================================= class COLMTokenizer: def __init__(self, token_list): """Build tokenizer from ordered token list.""" self.token_to_id = {token: i for i, token in enumerate(token_list)} self.id_to_token = {i: token for i, token in enumerate(token_list)} self.vocab_size = len(token_list) # Separate word tokens for boundary-aware matching self.word_tokens = set() for token in token_list: if len(token) > 1 and token not in SPECIAL_TOKENS: self.word_tokens.add(token) # Precompute max word length for bounded search self.max_word_len = max((len(w) for w in self.word_tokens), default=0) # Special token IDs self.pad_id = self.token_to_id[""] self.bos_id = self.token_to_id[""] self.eos_id = self.token_to_id[""] self.unk_id = self.token_to_id[""] def encode(self, text, add_bos=False, add_eos=False): """Encode text to token IDs. Greedy longest-match for word tokens at word boundaries. Single character fallback for everything else.""" ids = [] if add_bos: ids.append(self.bos_id) i = 0 n = len(text) while i < n: # Try longest word match first (only at word boundaries) matched = False before_ok = (i == 0) or not text[i - 1].isalpha() if before_ok: hi = min(self.max_word_len, n - i) for length in range(hi, 1, -1): candidate = text[i:i + length] if candidate in self.word_tokens: after_pos = i + length after_ok = (after_pos >= n) or not text[after_pos].isalpha() if after_ok: ids.append(self.token_to_id[candidate]) i += length matched = True break if not matched: ch = text[i] ids.append(self.token_to_id.get(ch, self.unk_id)) i += 1 if add_eos: ids.append(self.eos_id) return ids def decode(self, ids): """Decode token IDs back to text. Just concatenate.""" tokens = [] for token_id in ids: if token_id in (self.pad_id, self.bos_id, self.eos_id): continue tokens.append(self.id_to_token.get(token_id, "")) return "".join(tokens) def save(self, path): """Save tokenizer as JSON.""" data = { "version": "colm_v1", "vocab_size": self.vocab_size, "tokens": [self.id_to_token[i] for i in range(self.vocab_size)], } with open(path, "w", encoding="utf-8") as f: json.dump(data, f, ensure_ascii=False, indent=2) print(f"Saved tokenizer to {path}") print(f"Vocab size: {self.vocab_size}") @classmethod def load(cls, path): """Load tokenizer from JSON.""" with open(path, "r", encoding="utf-8") as f: data = json.load(f) return cls(data["tokens"]) def get_vocab(self): return dict(self.token_to_id) def get_vocab_size(self): return self.vocab_size # Backward compatibility alias EdenTokenizer = COLMTokenizer # ============================================================================= # BUILD # ============================================================================= def build_default_tokenizer(): """Build tokenizer from the default token definitions (no corpus).""" return build_tokenizer() def build_tokenizer(corpus_path=None, hf_dataset=None, hf_config=None, hf_split="train", hf_field="text"): """Build tokenizer, optionally augmented with corpus-derived words. Token order in the final vocab: 1. Special tokens (, , , ) 2. Whitespace (space, newline, tab) 3. Base word tokens (top 100 English — lowercase) 4. Sentence-case base (The, Be, To, ... — +100) 5. Corpus freq words (lowercase) 6. Corpus freq words (Sentence case) 7. Corpus long words (lowercase) 8. Corpus long words (Sentence case) 9. Letters (A-Z, a-z) 10. Digits (0-9) 11. Symbols """ token_list = [] seen = set() def add(tok): if tok not in seen: token_list.append(tok) seen.add(tok) # --- 1. Special tokens --- for t in SPECIAL_TOKENS: add(t) # --- 2. Whitespace --- for t in WHITESPACE_TOKENS: add(t) # --- 3. Base word tokens (lowercase) --- for t in WORD_TOKENS: add(t) # --- 4. Sentence-case base words --- for t in WORD_TOKENS: add(make_sentence_case(t)) # --- 5-8. Corpus-derived words (if corpus provided) --- corpus_words = [] has_corpus = (corpus_path is not None or hf_dataset is not None) if has_corpus: lines = iter_corpus_lines( corpus_path=corpus_path, hf_dataset=hf_dataset, hf_config=hf_config, hf_split=hf_split, hf_field=hf_field, ) counts = count_corpus_words(lines) # Build set of existing lowercase words to exclude existing = set(w.lower() for w in WORD_TOKENS) freq_words, long_words, combined = select_corpus_words( counts, existing, n_freq=50, n_long=50 ) corpus_words = combined # Add corpus frequency words: lowercase then sentence case for w in freq_words: add(w) for w in freq_words: add(make_sentence_case(w)) # Add corpus long words: lowercase then sentence case for w in long_words: add(w) for w in long_words: add(make_sentence_case(w)) # --- 9. Letters --- for t in UPPER: add(t) for t in LOWER: add(t) # --- 10. Digits --- for t in DIGITS: add(t) # --- 11. Symbols --- for t in sorted(SYMBOLS): add(t) tok = COLMTokenizer(token_list) # Print breakdown n_special = len(SPECIAL_TOKENS) n_ws = len(WHITESPACE_TOKENS) n_base_words = len(WORD_TOKENS) n_base_sc = sum(1 for t in WORD_TOKENS if make_sentence_case(t) not in set(WORD_TOKENS)) # non-overlapping sentence case n_corpus = len(corpus_words) * 2 if corpus_words else 0 # lc + sc n_letters = len(UPPER) + len(LOWER) n_digits = len(DIGITS) # Everything else is symbols n_symbols = (tok.vocab_size - n_special - n_ws - n_base_words - n_letters - n_digits) # Subtract corpus + sentence-case base from the "symbols" remainder n_symbols = n_symbols - n_base_sc - n_corpus # Recalculate properly accounted = (n_special + n_ws + n_base_words + n_letters + n_digits) n_other = tok.vocab_size - accounted print(f"\nBreakdown:") print(f" Special: {n_special}") print(f" Whitespace: {n_ws}") print(f" Base words (lc): {n_base_words}") if has_corpus: print(f" Base words (Sc): +sentence case") print(f" Corpus words (lc+Sc): {n_corpus}") print(f" Letters: {n_letters}") print(f" Digits: {n_digits}") print(f" Other (Sc/symbols): {n_other}") print(f" Total: {tok.vocab_size}") return tok # ============================================================================= # CLI # ============================================================================= if __name__ == "__main__": parser = argparse.ArgumentParser( description="Build word+char hybrid tokenizer", formatter_class=argparse.RawDescriptionHelpFormatter, epilog=""" Examples: # Base tokenizer only python tokenizer_train.py --output colm_tokenizer.json # Scan a local text file for extra vocab python tokenizer_train.py --corpus training_data.txt --output colm_tokenizer.json # Scan multiple files via glob python tokenizer_train.py --corpus "data/*.txt" --output colm_tokenizer.json # Use a HuggingFace dataset python tokenizer_train.py --hf-dataset wikitext --hf-config wikitext-103-raw-v1 \\ --hf-split train --hf-field text --output colm_tokenizer.json """) parser.add_argument("--output", default="colm_tokenizer.json", help="Output path (default: colm_tokenizer.json)") parser.add_argument("--corpus", default=None, help="Path or glob pattern for plain text file(s)") parser.add_argument("--hf-dataset", default=None, help="HuggingFace dataset name") parser.add_argument("--hf-config", default=None, help="HuggingFace dataset config") parser.add_argument("--hf-split", default="train", help="HuggingFace split (default: train)") parser.add_argument("--hf-field", default="text", help="HuggingFace text column (default: text)") parser.add_argument("--test", default=None, help="Test string to encode/decode") args = parser.parse_args() tok = build_tokenizer( corpus_path=args.corpus, hf_dataset=args.hf_dataset, hf_config=args.hf_config, hf_split=args.hf_split, hf_field=args.hf_field, ) tok.save(args.output) # Quick roundtrip test test = args.test or "The Machine God, in the beginning, with the Flesh God's design..." encoded = tok.encode(test) decoded = tok.decode(encoded) print(f"\nTest: \"{test}\"") print(f"IDs: {encoded}") print(f"Back: \"{decoded}\"") print(f"Match: {test == decoded}")