File size: 18,913 Bytes
3a1536e
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
"""
tokenizer_train.py β€” Word + Character hybrid tokenizer
======================================================
Each word token = one whole word. Everything else = single characters.
No subwords. No merges. No BPE. No external dependencies (for base).

Encode: greedy longest-match at word boundaries, character fallback.
Decode: concatenate tokens directly (spaces are their own token).

Corpus-aware mode (--corpus / --hf-dataset):
    Scans data for the most useful words and adds them to the vocab
    in both lowercase and Sentence case forms.

Usage:
    # Build base tokenizer (no corpus)
    python tokenizer_train.py --output colm_tokenizer.json

    # Build with corpus from plain text file(s)
    python tokenizer_train.py --corpus data.txt --output colm_tokenizer.json
    python tokenizer_train.py --corpus "*.txt" --output colm_tokenizer.json

    # Build with corpus from HuggingFace dataset
    python tokenizer_train.py --hf-dataset wikitext --hf-config wikitext-103-raw-v1 \\
        --hf-split train --hf-field text --output colm_tokenizer.json

    # Use in training script
    from tokenizer_train import COLMTokenizer
    tok = COLMTokenizer.load("colm_tokenizer.json")
    ids = tok.encode("The Machine God")
    text = tok.decode(ids)
"""

import json
import re
import argparse
import glob
from collections import Counter
from pathlib import Path


# =============================================================================
# TOKEN DEFINITIONS
# =============================================================================

SPECIAL_TOKENS = ["<pad>", "<bos>", "<eos>", "<unk>"]

# Whitespace (encoded as visible tokens so decode reconstructs perfectly)
WHITESPACE_TOKENS = [" ", "\n", "\t"]

# Top 100 English words β€” each is one token
WORD_TOKENS = [
    "the", "be", "to", "of", "and", "in", "that", "have", "it", "for",
    "not", "on", "with", "he", "as", "you", "do", "at", "this", "but",
    "his", "by", "from", "they", "we", "say", "her", "she", "or", "an",
    "will", "my", "one", "all", "would", "there", "their", "what", "so", "up",
    "out", "if", "about", "who", "get", "which", "go", "me", "when", "make",
    "can", "like", "time", "no", "just", "him", "know", "take", "people", "into",
    "year", "your", "good", "some", "could", "them", "see", "other", "than", "then",
    "now", "look", "only", "come", "it's", "over", "think", "also", "back", "after",
    "use", "two", "how", "our", "work", "first", "well", "way", "even", "new",
    "want", "because", "any", "these", "give", "day", "most", "us",
]

# Individual letters
UPPER = list("ABCDEFGHIJKLMNOPQRSTUVWXYZ")
LOWER = list("abcdefghijklmnopqrstuvwxyz")

# Digits
DIGITS = list("0123456789")

# Symbols (keyboard + corpus unicode)
SYMBOLS = list(set([
    "+", "_", "=", ")", "(", "*", "&", "^", "%", "$",
    "Β£", '"', "!", "`", "Β¬", "\\", "|", "<", ">", "/",
    "?", ":", "@", "~", "#", "'", ";", ",", ".",
    # Extended unicode (add corpus-specific symbols here)
    "\u2013",  # – en-dash
    "\u2014",  # β€” em-dash
    "\u2018",  # ' left single curly quote
    "\u2019",  # ' right single curly quote
    "-",
]))


# =============================================================================
# CORPUS SCANNING
# =============================================================================

# Matches whole words including contractions (don't, it's, they've, etc.)
# Apostrophe variants: straight ' and curly '
WORD_RE = re.compile(r"[a-zA-Z]+(?:['\u2019][a-zA-Z]+)*")

# Minimum frequency to consider a "long" word (filters OCR junk / typos)
MIN_FREQ_FOR_LENGTH = 3


def iter_corpus_lines(corpus_path=None, hf_dataset=None, hf_config=None,
                      hf_split="train", hf_field="text"):
    """Yield text lines from either local file(s) or a HuggingFace dataset.

    Args:
        corpus_path: Glob pattern or single path for local .txt files
        hf_dataset:  HuggingFace dataset name (e.g. "wikitext")
        hf_config:   HuggingFace config name (e.g. "wikitext-103-raw-v1")
        hf_split:    Dataset split to use
        hf_field:    Column name containing text
    """
    if corpus_path is not None:
        # Local text file(s) β€” supports glob patterns
        paths = sorted(glob.glob(corpus_path))
        if not paths:
            raise FileNotFoundError(f"No files matched: {corpus_path}")
        print(f"[corpus] Reading {len(paths)} file(s) from: {corpus_path}")
        for p in paths:
            with open(p, "r", encoding="utf-8", errors="replace") as f:
                for line in f:
                    line = line.rstrip("\n\r")
                    if line:
                        yield line

    elif hf_dataset is not None:
        try:
            from datasets import load_dataset
        except ImportError:
            raise ImportError(
                "HuggingFace datasets not installed. Run: "
                "pip install datasets --break-system-packages"
            )
        print(f"[corpus] Loading HuggingFace dataset: {hf_dataset}"
              f" (config={hf_config}, split={hf_split}, field={hf_field})")
        ds = load_dataset(hf_dataset, hf_config, split=hf_split,
                          trust_remote_code=True)
        for row in ds:
            text = row.get(hf_field, "")
            if text:
                yield text

    else:
        return  # No corpus β€” base tokenizer only


def count_corpus_words(lines_iter):
    """Count all words in the corpus. Returns a Counter of lowercase words.

    Contractions are preserved as-is (don't, it's, etc.) β€” lowercased.
    """
    counts = Counter()
    n_lines = 0
    for line in lines_iter:
        words = WORD_RE.findall(line)
        # Normalise: lowercase, and normalise curly apostrophes to straight
        normalised = [w.lower().replace("\u2019", "'") for w in words]
        counts.update(normalised)
        n_lines += 1
        if n_lines % 500_000 == 0:
            print(f"  ...scanned {n_lines:,} lines, {sum(counts.values()):,} tokens")
    print(f"[corpus] Done: {n_lines:,} lines, {len(counts):,} unique words, "
          f"{sum(counts.values()):,} total tokens")
    return counts


def select_corpus_words(counts, existing_words, n_freq=50, n_long=50):
    """Select corpus words for the tokenizer.

    Strategy:
        1. Top 100 by frequency β†’ take top 50 (excluding existing vocab)
        2. Top 100 by length (min freq threshold) β†’ take top 50 by length
        3. Deduplicate across both lists
        4. Return combined list

    Args:
        counts:         Counter of lowercase words
        existing_words: Set of words already in the tokenizer (lowercase)
        n_freq:         How many frequency-sorted words to keep
        n_long:         How many length-sorted words to keep

    Returns:
        (freq_words, long_words, combined) β€” all lowercase, deduplicated
    """
    # Single-char words are already covered by the character tokens
    eligible = {w: c for w, c in counts.items()
                if len(w) > 1 and w not in existing_words}

    # --- Pool 1: Top by frequency ---
    freq_sorted = sorted(eligible.items(), key=lambda x: x[1], reverse=True)
    freq_pool = [w for w, _ in freq_sorted[:100]]
    freq_selected = freq_pool[:n_freq]

    # --- Pool 2: Top by length (must meet minimum frequency) ---
    long_eligible = {w: c for w, c in eligible.items()
                     if c >= MIN_FREQ_FOR_LENGTH}
    long_sorted = sorted(long_eligible.items(), key=lambda x: len(x[0]), reverse=True)
    long_pool = [w for w, _ in long_sorted[:100]]
    long_selected = long_pool[:n_long]

    # --- Combine and deduplicate (preserve selection order) ---
    seen = set()
    combined = []
    # Frequency words first (they're higher-value for compression)
    for w in freq_selected:
        if w not in seen:
            combined.append(w)
            seen.add(w)
    # Then length words
    for w in long_selected:
        if w not in seen:
            combined.append(w)
            seen.add(w)

    print(f"[vocab] Selected {len(freq_selected)} by frequency, "
          f"{len(long_selected)} by length")
    print(f"[vocab] Combined (deduplicated): {len(combined)} corpus words")

    # Show what we picked
    if freq_selected:
        print(f"[vocab] Top 5 by freq: {freq_selected[:5]}")
    if long_selected:
        print(f"[vocab] Top 5 by length: {long_selected[:5]} "
              f"(lengths: {[len(w) for w in long_selected[:5]]})")

    return freq_selected, long_selected, combined


def make_sentence_case(word):
    """Capitalise first letter only β€” as it would appear at start of sentence.

    Handles contractions: don't β†’ Don't, it's β†’ It's
    """
    if not word:
        return word
    return word[0].upper() + word[1:]


# =============================================================================
# TOKENIZER CLASS
# =============================================================================

class COLMTokenizer:
    def __init__(self, token_list):
        """Build tokenizer from ordered token list."""
        self.token_to_id = {token: i for i, token in enumerate(token_list)}
        self.id_to_token = {i: token for i, token in enumerate(token_list)}
        self.vocab_size = len(token_list)

        # Separate word tokens for boundary-aware matching
        self.word_tokens = set()
        for token in token_list:
            if len(token) > 1 and token not in SPECIAL_TOKENS:
                self.word_tokens.add(token)

        # Precompute max word length for bounded search
        self.max_word_len = max((len(w) for w in self.word_tokens), default=0)

        # Special token IDs
        self.pad_id = self.token_to_id["<pad>"]
        self.bos_id = self.token_to_id["<bos>"]
        self.eos_id = self.token_to_id["<eos>"]
        self.unk_id = self.token_to_id["<unk>"]

    def encode(self, text, add_bos=False, add_eos=False):
        """Encode text to token IDs.
        Greedy longest-match for word tokens at word boundaries.
        Single character fallback for everything else."""
        ids = []
        if add_bos:
            ids.append(self.bos_id)

        i = 0
        n = len(text)
        while i < n:
            # Try longest word match first (only at word boundaries)
            matched = False
            before_ok = (i == 0) or not text[i - 1].isalpha()

            if before_ok:
                hi = min(self.max_word_len, n - i)
                for length in range(hi, 1, -1):
                    candidate = text[i:i + length]
                    if candidate in self.word_tokens:
                        after_pos = i + length
                        after_ok = (after_pos >= n) or not text[after_pos].isalpha()
                        if after_ok:
                            ids.append(self.token_to_id[candidate])
                            i += length
                            matched = True
                            break

            if not matched:
                ch = text[i]
                ids.append(self.token_to_id.get(ch, self.unk_id))
                i += 1

        if add_eos:
            ids.append(self.eos_id)
        return ids

    def decode(self, ids):
        """Decode token IDs back to text. Just concatenate."""
        tokens = []
        for token_id in ids:
            if token_id in (self.pad_id, self.bos_id, self.eos_id):
                continue
            tokens.append(self.id_to_token.get(token_id, "<unk>"))
        return "".join(tokens)

    def save(self, path):
        """Save tokenizer as JSON."""
        data = {
            "version": "colm_v1",
            "vocab_size": self.vocab_size,
            "tokens": [self.id_to_token[i] for i in range(self.vocab_size)],
        }
        with open(path, "w", encoding="utf-8") as f:
            json.dump(data, f, ensure_ascii=False, indent=2)
        print(f"Saved tokenizer to {path}")
        print(f"Vocab size: {self.vocab_size}")

    @classmethod
    def load(cls, path):
        """Load tokenizer from JSON."""
        with open(path, "r", encoding="utf-8") as f:
            data = json.load(f)
        return cls(data["tokens"])

    def get_vocab(self):
        return dict(self.token_to_id)

    def get_vocab_size(self):
        return self.vocab_size


# Backward compatibility alias
EdenTokenizer = COLMTokenizer


# =============================================================================
# BUILD
# =============================================================================

def build_default_tokenizer():
    """Build tokenizer from the default token definitions (no corpus)."""
    return build_tokenizer()


def build_tokenizer(corpus_path=None, hf_dataset=None, hf_config=None,
                    hf_split="train", hf_field="text"):
    """Build tokenizer, optionally augmented with corpus-derived words.

    Token order in the final vocab:
        1. Special tokens         (<pad>, <bos>, <eos>, <unk>)
        2. Whitespace             (space, newline, tab)
        3. Base word tokens       (top 100 English β€” lowercase)
        4. Sentence-case base     (The, Be, To, ... β€” +100)
        5. Corpus freq words      (lowercase)
        6. Corpus freq words      (Sentence case)
        7. Corpus long words      (lowercase)
        8. Corpus long words      (Sentence case)
        9. Letters                (A-Z, a-z)
       10. Digits                 (0-9)
       11. Symbols
    """
    token_list = []
    seen = set()

    def add(tok):
        if tok not in seen:
            token_list.append(tok)
            seen.add(tok)

    # --- 1. Special tokens ---
    for t in SPECIAL_TOKENS:
        add(t)

    # --- 2. Whitespace ---
    for t in WHITESPACE_TOKENS:
        add(t)

    # --- 3. Base word tokens (lowercase) ---
    for t in WORD_TOKENS:
        add(t)

    # --- 4. Sentence-case base words ---
    for t in WORD_TOKENS:
        add(make_sentence_case(t))

    # --- 5-8. Corpus-derived words (if corpus provided) ---
    corpus_words = []
    has_corpus = (corpus_path is not None or hf_dataset is not None)

    if has_corpus:
        lines = iter_corpus_lines(
            corpus_path=corpus_path,
            hf_dataset=hf_dataset,
            hf_config=hf_config,
            hf_split=hf_split,
            hf_field=hf_field,
        )
        counts = count_corpus_words(lines)

        # Build set of existing lowercase words to exclude
        existing = set(w.lower() for w in WORD_TOKENS)

        freq_words, long_words, combined = select_corpus_words(
            counts, existing, n_freq=50, n_long=50
        )
        corpus_words = combined

        # Add corpus frequency words: lowercase then sentence case
        for w in freq_words:
            add(w)
        for w in freq_words:
            add(make_sentence_case(w))

        # Add corpus long words: lowercase then sentence case
        for w in long_words:
            add(w)
        for w in long_words:
            add(make_sentence_case(w))

    # --- 9. Letters ---
    for t in UPPER:
        add(t)
    for t in LOWER:
        add(t)

    # --- 10. Digits ---
    for t in DIGITS:
        add(t)

    # --- 11. Symbols ---
    for t in sorted(SYMBOLS):
        add(t)

    tok = COLMTokenizer(token_list)

    # Print breakdown
    n_special = len(SPECIAL_TOKENS)
    n_ws = len(WHITESPACE_TOKENS)
    n_base_words = len(WORD_TOKENS)
    n_base_sc = sum(1 for t in WORD_TOKENS if make_sentence_case(t) not in
                    set(WORD_TOKENS))  # non-overlapping sentence case
    n_corpus = len(corpus_words) * 2 if corpus_words else 0  # lc + sc
    n_letters = len(UPPER) + len(LOWER)
    n_digits = len(DIGITS)
    # Everything else is symbols
    n_symbols = (tok.vocab_size - n_special - n_ws - n_base_words
                 - n_letters - n_digits)
    # Subtract corpus + sentence-case base from the "symbols" remainder
    n_symbols = n_symbols - n_base_sc - n_corpus
    # Recalculate properly
    accounted = (n_special + n_ws + n_base_words + n_letters + n_digits)
    n_other = tok.vocab_size - accounted

    print(f"\nBreakdown:")
    print(f"  Special:              {n_special}")
    print(f"  Whitespace:           {n_ws}")
    print(f"  Base words (lc):      {n_base_words}")
    if has_corpus:
        print(f"  Base words (Sc):      +sentence case")
        print(f"  Corpus words (lc+Sc): {n_corpus}")
    print(f"  Letters:              {n_letters}")
    print(f"  Digits:               {n_digits}")
    print(f"  Other (Sc/symbols):   {n_other}")
    print(f"  Total:                {tok.vocab_size}")

    return tok


# =============================================================================
# CLI
# =============================================================================

if __name__ == "__main__":
    parser = argparse.ArgumentParser(
        description="Build word+char hybrid tokenizer",
        formatter_class=argparse.RawDescriptionHelpFormatter,
        epilog="""
Examples:
  # Base tokenizer only
  python tokenizer_train.py --output colm_tokenizer.json

  # Scan a local text file for extra vocab
  python tokenizer_train.py --corpus training_data.txt --output colm_tokenizer.json

  # Scan multiple files via glob
  python tokenizer_train.py --corpus "data/*.txt" --output colm_tokenizer.json

  # Use a HuggingFace dataset
  python tokenizer_train.py --hf-dataset wikitext --hf-config wikitext-103-raw-v1 \\
      --hf-split train --hf-field text --output colm_tokenizer.json
""")
    parser.add_argument("--output", default="colm_tokenizer.json",
                        help="Output path (default: colm_tokenizer.json)")
    parser.add_argument("--corpus", default=None,
                        help="Path or glob pattern for plain text file(s)")
    parser.add_argument("--hf-dataset", default=None,
                        help="HuggingFace dataset name")
    parser.add_argument("--hf-config", default=None,
                        help="HuggingFace dataset config")
    parser.add_argument("--hf-split", default="train",
                        help="HuggingFace split (default: train)")
    parser.add_argument("--hf-field", default="text",
                        help="HuggingFace text column (default: text)")
    parser.add_argument("--test", default=None,
                        help="Test string to encode/decode")
    args = parser.parse_args()

    tok = build_tokenizer(
        corpus_path=args.corpus,
        hf_dataset=args.hf_dataset,
        hf_config=args.hf_config,
        hf_split=args.hf_split,
        hf_field=args.hf_field,
    )
    tok.save(args.output)

    # Quick roundtrip test
    test = args.test or "The Machine God, in the beginning, with the Flesh God's design..."
    encoded = tok.encode(test)
    decoded = tok.decode(encoded)
    print(f"\nTest: \"{test}\"")
    print(f"IDs:  {encoded}")
    print(f"Back: \"{decoded}\"")
    print(f"Match: {test == decoded}")