Download tokenizer_train.py from edeneldith/COLM: direct link, hf CLI and curl.
- Browser
- Download file 18.9 kB
-
https://huggingface.co/edeneldith/COLM/resolve/main/tokenizer_train.py
- Command line
-
hf download hf://edeneldith/COLM/tokenizer_train.py
-
curl -L -o tokenizer_train.py https://huggingface.co/edeneldith/COLM/resolve/main/tokenizer_train.py
18.9 kB
| """ | |
| tokenizer_train.py β Word + Character hybrid tokenizer | |
| ====================================================== | |
| Each word token = one whole word. Everything else = single characters. | |
| No subwords. No merges. No BPE. No external dependencies (for base). | |
| Encode: greedy longest-match at word boundaries, character fallback. | |
| Decode: concatenate tokens directly (spaces are their own token). | |
| Corpus-aware mode (--corpus / --hf-dataset): | |
| Scans data for the most useful words and adds them to the vocab | |
| in both lowercase and Sentence case forms. | |
| Usage: | |
| # Build base tokenizer (no corpus) | |
| python tokenizer_train.py --output colm_tokenizer.json | |
| # Build with corpus from plain text file(s) | |
| python tokenizer_train.py --corpus data.txt --output colm_tokenizer.json | |
| python tokenizer_train.py --corpus "*.txt" --output colm_tokenizer.json | |
| # Build with corpus from HuggingFace dataset | |
| python tokenizer_train.py --hf-dataset wikitext --hf-config wikitext-103-raw-v1 \\ | |
| --hf-split train --hf-field text --output colm_tokenizer.json | |
| # Use in training script | |
| from tokenizer_train import COLMTokenizer | |
| tok = COLMTokenizer.load("colm_tokenizer.json") | |
| ids = tok.encode("The Machine God") | |
| text = tok.decode(ids) | |
| """ | |
| import json | |
| import re | |
| import argparse | |
| import glob | |
| from collections import Counter | |
| from pathlib import Path | |
| # ============================================================================= | |
| # TOKEN DEFINITIONS | |
| # ============================================================================= | |
| SPECIAL_TOKENS = ["<pad>", "<bos>", "<eos>", "<unk>"] | |
| # Whitespace (encoded as visible tokens so decode reconstructs perfectly) | |
| WHITESPACE_TOKENS = [" ", "\n", "\t"] | |
| # Top 100 English words β each is one token | |
| WORD_TOKENS = [ | |
| "the", "be", "to", "of", "and", "in", "that", "have", "it", "for", | |
| "not", "on", "with", "he", "as", "you", "do", "at", "this", "but", | |
| "his", "by", "from", "they", "we", "say", "her", "she", "or", "an", | |
| "will", "my", "one", "all", "would", "there", "their", "what", "so", "up", | |
| "out", "if", "about", "who", "get", "which", "go", "me", "when", "make", | |
| "can", "like", "time", "no", "just", "him", "know", "take", "people", "into", | |
| "year", "your", "good", "some", "could", "them", "see", "other", "than", "then", | |
| "now", "look", "only", "come", "it's", "over", "think", "also", "back", "after", | |
| "use", "two", "how", "our", "work", "first", "well", "way", "even", "new", | |
| "want", "because", "any", "these", "give", "day", "most", "us", | |
| ] | |
| # Individual letters | |
| UPPER = list("ABCDEFGHIJKLMNOPQRSTUVWXYZ") | |
| LOWER = list("abcdefghijklmnopqrstuvwxyz") | |
| # Digits | |
| DIGITS = list("0123456789") | |
| # Symbols (keyboard + corpus unicode) | |
| SYMBOLS = list(set([ | |
| "+", "_", "=", ")", "(", "*", "&", "^", "%", "$", | |
| "Β£", '"', "!", "`", "Β¬", "\\", "|", "<", ">", "/", | |
| "?", ":", "@", "~", "#", "'", ";", ",", ".", | |
| # Extended unicode (add corpus-specific symbols here) | |
| "\u2013", # β en-dash | |
| "\u2014", # β em-dash | |
| "\u2018", # ' left single curly quote | |
| "\u2019", # ' right single curly quote | |
| "-", | |
| ])) | |
| # ============================================================================= | |
| # CORPUS SCANNING | |
| # ============================================================================= | |
| # Matches whole words including contractions (don't, it's, they've, etc.) | |
| # Apostrophe variants: straight ' and curly ' | |
| WORD_RE = re.compile(r"[a-zA-Z]+(?:['\u2019][a-zA-Z]+)*") | |
| # Minimum frequency to consider a "long" word (filters OCR junk / typos) | |
| MIN_FREQ_FOR_LENGTH = 3 | |
| def iter_corpus_lines(corpus_path=None, hf_dataset=None, hf_config=None, | |
| hf_split="train", hf_field="text"): | |
| """Yield text lines from either local file(s) or a HuggingFace dataset. | |
| Args: | |
| corpus_path: Glob pattern or single path for local .txt files | |
| hf_dataset: HuggingFace dataset name (e.g. "wikitext") | |
| hf_config: HuggingFace config name (e.g. "wikitext-103-raw-v1") | |
| hf_split: Dataset split to use | |
| hf_field: Column name containing text | |
| """ | |
| if corpus_path is not None: | |
| # Local text file(s) β supports glob patterns | |
| paths = sorted(glob.glob(corpus_path)) | |
| if not paths: | |
| raise FileNotFoundError(f"No files matched: {corpus_path}") | |
| print(f"[corpus] Reading {len(paths)} file(s) from: {corpus_path}") | |
| for p in paths: | |
| with open(p, "r", encoding="utf-8", errors="replace") as f: | |
| for line in f: | |
| line = line.rstrip("\n\r") | |
| if line: | |
| yield line | |
| elif hf_dataset is not None: | |
| try: | |
| from datasets import load_dataset | |
| except ImportError: | |
| raise ImportError( | |
| "HuggingFace datasets not installed. Run: " | |
| "pip install datasets --break-system-packages" | |
| ) | |
| print(f"[corpus] Loading HuggingFace dataset: {hf_dataset}" | |
| f" (config={hf_config}, split={hf_split}, field={hf_field})") | |
| ds = load_dataset(hf_dataset, hf_config, split=hf_split, | |
| trust_remote_code=True) | |
| for row in ds: | |
| text = row.get(hf_field, "") | |
| if text: | |
| yield text | |
| else: | |
| return # No corpus β base tokenizer only | |
| def count_corpus_words(lines_iter): | |
| """Count all words in the corpus. Returns a Counter of lowercase words. | |
| Contractions are preserved as-is (don't, it's, etc.) β lowercased. | |
| """ | |
| counts = Counter() | |
| n_lines = 0 | |
| for line in lines_iter: | |
| words = WORD_RE.findall(line) | |
| # Normalise: lowercase, and normalise curly apostrophes to straight | |
| normalised = [w.lower().replace("\u2019", "'") for w in words] | |
| counts.update(normalised) | |
| n_lines += 1 | |
| if n_lines % 500_000 == 0: | |
| print(f" ...scanned {n_lines:,} lines, {sum(counts.values()):,} tokens") | |
| print(f"[corpus] Done: {n_lines:,} lines, {len(counts):,} unique words, " | |
| f"{sum(counts.values()):,} total tokens") | |
| return counts | |
| def select_corpus_words(counts, existing_words, n_freq=50, n_long=50): | |
| """Select corpus words for the tokenizer. | |
| Strategy: | |
| 1. Top 100 by frequency β take top 50 (excluding existing vocab) | |
| 2. Top 100 by length (min freq threshold) β take top 50 by length | |
| 3. Deduplicate across both lists | |
| 4. Return combined list | |
| Args: | |
| counts: Counter of lowercase words | |
| existing_words: Set of words already in the tokenizer (lowercase) | |
| n_freq: How many frequency-sorted words to keep | |
| n_long: How many length-sorted words to keep | |
| Returns: | |
| (freq_words, long_words, combined) β all lowercase, deduplicated | |
| """ | |
| # Single-char words are already covered by the character tokens | |
| eligible = {w: c for w, c in counts.items() | |
| if len(w) > 1 and w not in existing_words} | |
| # --- Pool 1: Top by frequency --- | |
| freq_sorted = sorted(eligible.items(), key=lambda x: x[1], reverse=True) | |
| freq_pool = [w for w, _ in freq_sorted[:100]] | |
| freq_selected = freq_pool[:n_freq] | |
| # --- Pool 2: Top by length (must meet minimum frequency) --- | |
| long_eligible = {w: c for w, c in eligible.items() | |
| if c >= MIN_FREQ_FOR_LENGTH} | |
| long_sorted = sorted(long_eligible.items(), key=lambda x: len(x[0]), reverse=True) | |
| long_pool = [w for w, _ in long_sorted[:100]] | |
| long_selected = long_pool[:n_long] | |
| # --- Combine and deduplicate (preserve selection order) --- | |
| seen = set() | |
| combined = [] | |
| # Frequency words first (they're higher-value for compression) | |
| for w in freq_selected: | |
| if w not in seen: | |
| combined.append(w) | |
| seen.add(w) | |
| # Then length words | |
| for w in long_selected: | |
| if w not in seen: | |
| combined.append(w) | |
| seen.add(w) | |
| print(f"[vocab] Selected {len(freq_selected)} by frequency, " | |
| f"{len(long_selected)} by length") | |
| print(f"[vocab] Combined (deduplicated): {len(combined)} corpus words") | |
| # Show what we picked | |
| if freq_selected: | |
| print(f"[vocab] Top 5 by freq: {freq_selected[:5]}") | |
| if long_selected: | |
| print(f"[vocab] Top 5 by length: {long_selected[:5]} " | |
| f"(lengths: {[len(w) for w in long_selected[:5]]})") | |
| return freq_selected, long_selected, combined | |
| def make_sentence_case(word): | |
| """Capitalise first letter only β as it would appear at start of sentence. | |
| Handles contractions: don't β Don't, it's β It's | |
| """ | |
| if not word: | |
| return word | |
| return word[0].upper() + word[1:] | |
| # ============================================================================= | |
| # TOKENIZER CLASS | |
| # ============================================================================= | |
| class COLMTokenizer: | |
| def __init__(self, token_list): | |
| """Build tokenizer from ordered token list.""" | |
| self.token_to_id = {token: i for i, token in enumerate(token_list)} | |
| self.id_to_token = {i: token for i, token in enumerate(token_list)} | |
| self.vocab_size = len(token_list) | |
| # Separate word tokens for boundary-aware matching | |
| self.word_tokens = set() | |
| for token in token_list: | |
| if len(token) > 1 and token not in SPECIAL_TOKENS: | |
| self.word_tokens.add(token) | |
| # Precompute max word length for bounded search | |
| self.max_word_len = max((len(w) for w in self.word_tokens), default=0) | |
| # Special token IDs | |
| self.pad_id = self.token_to_id["<pad>"] | |
| self.bos_id = self.token_to_id["<bos>"] | |
| self.eos_id = self.token_to_id["<eos>"] | |
| self.unk_id = self.token_to_id["<unk>"] | |
| def encode(self, text, add_bos=False, add_eos=False): | |
| """Encode text to token IDs. | |
| Greedy longest-match for word tokens at word boundaries. | |
| Single character fallback for everything else.""" | |
| ids = [] | |
| if add_bos: | |
| ids.append(self.bos_id) | |
| i = 0 | |
| n = len(text) | |
| while i < n: | |
| # Try longest word match first (only at word boundaries) | |
| matched = False | |
| before_ok = (i == 0) or not text[i - 1].isalpha() | |
| if before_ok: | |
| hi = min(self.max_word_len, n - i) | |
| for length in range(hi, 1, -1): | |
| candidate = text[i:i + length] | |
| if candidate in self.word_tokens: | |
| after_pos = i + length | |
| after_ok = (after_pos >= n) or not text[after_pos].isalpha() | |
| if after_ok: | |
| ids.append(self.token_to_id[candidate]) | |
| i += length | |
| matched = True | |
| break | |
| if not matched: | |
| ch = text[i] | |
| ids.append(self.token_to_id.get(ch, self.unk_id)) | |
| i += 1 | |
| if add_eos: | |
| ids.append(self.eos_id) | |
| return ids | |
| def decode(self, ids): | |
| """Decode token IDs back to text. Just concatenate.""" | |
| tokens = [] | |
| for token_id in ids: | |
| if token_id in (self.pad_id, self.bos_id, self.eos_id): | |
| continue | |
| tokens.append(self.id_to_token.get(token_id, "<unk>")) | |
| return "".join(tokens) | |
| def save(self, path): | |
| """Save tokenizer as JSON.""" | |
| data = { | |
| "version": "colm_v1", | |
| "vocab_size": self.vocab_size, | |
| "tokens": [self.id_to_token[i] for i in range(self.vocab_size)], | |
| } | |
| with open(path, "w", encoding="utf-8") as f: | |
| json.dump(data, f, ensure_ascii=False, indent=2) | |
| print(f"Saved tokenizer to {path}") | |
| print(f"Vocab size: {self.vocab_size}") | |
| def load(cls, path): | |
| """Load tokenizer from JSON.""" | |
| with open(path, "r", encoding="utf-8") as f: | |
| data = json.load(f) | |
| return cls(data["tokens"]) | |
| def get_vocab(self): | |
| return dict(self.token_to_id) | |
| def get_vocab_size(self): | |
| return self.vocab_size | |
| # Backward compatibility alias | |
| EdenTokenizer = COLMTokenizer | |
| # ============================================================================= | |
| # BUILD | |
| # ============================================================================= | |
| def build_default_tokenizer(): | |
| """Build tokenizer from the default token definitions (no corpus).""" | |
| return build_tokenizer() | |
| def build_tokenizer(corpus_path=None, hf_dataset=None, hf_config=None, | |
| hf_split="train", hf_field="text"): | |
| """Build tokenizer, optionally augmented with corpus-derived words. | |
| Token order in the final vocab: | |
| 1. Special tokens (<pad>, <bos>, <eos>, <unk>) | |
| 2. Whitespace (space, newline, tab) | |
| 3. Base word tokens (top 100 English β lowercase) | |
| 4. Sentence-case base (The, Be, To, ... β +100) | |
| 5. Corpus freq words (lowercase) | |
| 6. Corpus freq words (Sentence case) | |
| 7. Corpus long words (lowercase) | |
| 8. Corpus long words (Sentence case) | |
| 9. Letters (A-Z, a-z) | |
| 10. Digits (0-9) | |
| 11. Symbols | |
| """ | |
| token_list = [] | |
| seen = set() | |
| def add(tok): | |
| if tok not in seen: | |
| token_list.append(tok) | |
| seen.add(tok) | |
| # --- 1. Special tokens --- | |
| for t in SPECIAL_TOKENS: | |
| add(t) | |
| # --- 2. Whitespace --- | |
| for t in WHITESPACE_TOKENS: | |
| add(t) | |
| # --- 3. Base word tokens (lowercase) --- | |
| for t in WORD_TOKENS: | |
| add(t) | |
| # --- 4. Sentence-case base words --- | |
| for t in WORD_TOKENS: | |
| add(make_sentence_case(t)) | |
| # --- 5-8. Corpus-derived words (if corpus provided) --- | |
| corpus_words = [] | |
| has_corpus = (corpus_path is not None or hf_dataset is not None) | |
| if has_corpus: | |
| lines = iter_corpus_lines( | |
| corpus_path=corpus_path, | |
| hf_dataset=hf_dataset, | |
| hf_config=hf_config, | |
| hf_split=hf_split, | |
| hf_field=hf_field, | |
| ) | |
| counts = count_corpus_words(lines) | |
| # Build set of existing lowercase words to exclude | |
| existing = set(w.lower() for w in WORD_TOKENS) | |
| freq_words, long_words, combined = select_corpus_words( | |
| counts, existing, n_freq=50, n_long=50 | |
| ) | |
| corpus_words = combined | |
| # Add corpus frequency words: lowercase then sentence case | |
| for w in freq_words: | |
| add(w) | |
| for w in freq_words: | |
| add(make_sentence_case(w)) | |
| # Add corpus long words: lowercase then sentence case | |
| for w in long_words: | |
| add(w) | |
| for w in long_words: | |
| add(make_sentence_case(w)) | |
| # --- 9. Letters --- | |
| for t in UPPER: | |
| add(t) | |
| for t in LOWER: | |
| add(t) | |
| # --- 10. Digits --- | |
| for t in DIGITS: | |
| add(t) | |
| # --- 11. Symbols --- | |
| for t in sorted(SYMBOLS): | |
| add(t) | |
| tok = COLMTokenizer(token_list) | |
| # Print breakdown | |
| n_special = len(SPECIAL_TOKENS) | |
| n_ws = len(WHITESPACE_TOKENS) | |
| n_base_words = len(WORD_TOKENS) | |
| n_base_sc = sum(1 for t in WORD_TOKENS if make_sentence_case(t) not in | |
| set(WORD_TOKENS)) # non-overlapping sentence case | |
| n_corpus = len(corpus_words) * 2 if corpus_words else 0 # lc + sc | |
| n_letters = len(UPPER) + len(LOWER) | |
| n_digits = len(DIGITS) | |
| # Everything else is symbols | |
| n_symbols = (tok.vocab_size - n_special - n_ws - n_base_words | |
| - n_letters - n_digits) | |
| # Subtract corpus + sentence-case base from the "symbols" remainder | |
| n_symbols = n_symbols - n_base_sc - n_corpus | |
| # Recalculate properly | |
| accounted = (n_special + n_ws + n_base_words + n_letters + n_digits) | |
| n_other = tok.vocab_size - accounted | |
| print(f"\nBreakdown:") | |
| print(f" Special: {n_special}") | |
| print(f" Whitespace: {n_ws}") | |
| print(f" Base words (lc): {n_base_words}") | |
| if has_corpus: | |
| print(f" Base words (Sc): +sentence case") | |
| print(f" Corpus words (lc+Sc): {n_corpus}") | |
| print(f" Letters: {n_letters}") | |
| print(f" Digits: {n_digits}") | |
| print(f" Other (Sc/symbols): {n_other}") | |
| print(f" Total: {tok.vocab_size}") | |
| return tok | |
| # ============================================================================= | |
| # CLI | |
| # ============================================================================= | |
| if __name__ == "__main__": | |
| parser = argparse.ArgumentParser( | |
| description="Build word+char hybrid tokenizer", | |
| formatter_class=argparse.RawDescriptionHelpFormatter, | |
| epilog=""" | |
| Examples: | |
| # Base tokenizer only | |
| python tokenizer_train.py --output colm_tokenizer.json | |
| # Scan a local text file for extra vocab | |
| python tokenizer_train.py --corpus training_data.txt --output colm_tokenizer.json | |
| # Scan multiple files via glob | |
| python tokenizer_train.py --corpus "data/*.txt" --output colm_tokenizer.json | |
| # Use a HuggingFace dataset | |
| python tokenizer_train.py --hf-dataset wikitext --hf-config wikitext-103-raw-v1 \\ | |
| --hf-split train --hf-field text --output colm_tokenizer.json | |
| """) | |
| parser.add_argument("--output", default="colm_tokenizer.json", | |
| help="Output path (default: colm_tokenizer.json)") | |
| parser.add_argument("--corpus", default=None, | |
| help="Path or glob pattern for plain text file(s)") | |
| parser.add_argument("--hf-dataset", default=None, | |
| help="HuggingFace dataset name") | |
| parser.add_argument("--hf-config", default=None, | |
| help="HuggingFace dataset config") | |
| parser.add_argument("--hf-split", default="train", | |
| help="HuggingFace split (default: train)") | |
| parser.add_argument("--hf-field", default="text", | |
| help="HuggingFace text column (default: text)") | |
| parser.add_argument("--test", default=None, | |
| help="Test string to encode/decode") | |
| args = parser.parse_args() | |
| tok = build_tokenizer( | |
| corpus_path=args.corpus, | |
| hf_dataset=args.hf_dataset, | |
| hf_config=args.hf_config, | |
| hf_split=args.hf_split, | |
| hf_field=args.hf_field, | |
| ) | |
| tok.save(args.output) | |
| # Quick roundtrip test | |
| test = args.test or "The Machine God, in the beginning, with the Flesh God's design..." | |
| encoded = tok.encode(test) | |
| decoded = tok.decode(encoded) | |
| print(f"\nTest: \"{test}\"") | |
| print(f"IDs: {encoded}") | |
| print(f"Back: \"{decoded}\"") | |
| print(f"Match: {test == decoded}") |