File size: 2,696 Bytes
d993048 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 | """Shared runtime modules imported by the numbered pipeline steps.
Contents:
core.model — GPT architecture (RMSNorm, RoPE, SwiGLU)
core.config — training presets and argparse-backed get_config()
core.char_tokenizer — character-level tokenizer
core.bpe_tokenizer — SentencePiece BPE tokenizer
And the path/resolver helpers re-exported here at package level:
tokenizer_path(data_dir, type) → data_dir/tokenizer_{type}.json
bin_paths(data_dir, type) → (train_{type}.bin, val_{type}.bin)
detect_tokenizer_type(data_dir) → "char" or "bpe"
load_tokenizer(data_dir, type) → instantiated tokenizer
Both the char and BPE pipelines write their artifacts side-by-side using a
tokenizer-type suffix:
data/train_char.bin data/train_bpe.bin
data/val_char.bin data/val_bpe.bin
data/tokenizer_char.json data/tokenizer_bpe.json
The helpers centralize path resolution so callers never hand-build filenames.
"""
import os
def tokenizer_path(data_dir, tokenizer_type):
return os.path.join(data_dir, f"tokenizer_{tokenizer_type}.json")
def bin_paths(data_dir, tokenizer_type):
return (
os.path.join(data_dir, f"train_{tokenizer_type}.bin"),
os.path.join(data_dir, f"val_{tokenizer_type}.bin"),
)
def detect_tokenizer_type(data_dir):
"""Return the tokenizer type (char or bpe) present in data_dir.
Raises FileNotFoundError if neither is present, or ValueError if both are
present (caller must disambiguate via --tokenizer).
"""
has_char = os.path.exists(tokenizer_path(data_dir, "char"))
has_bpe = os.path.exists(tokenizer_path(data_dir, "bpe"))
if has_char and has_bpe:
raise ValueError(
f"Both tokenizer_char.json and tokenizer_bpe.json exist in {data_dir}. "
"Pass --tokenizer char|bpe to choose."
)
if has_char:
return "char"
if has_bpe:
return "bpe"
raise FileNotFoundError(
f"No tokenizer_char.json or tokenizer_bpe.json in {data_dir}. "
"Run 3_tokenize.py first."
)
def load_tokenizer(data_dir, tokenizer_type):
"""Instantiate and load the right tokenizer class for the given type."""
path = tokenizer_path(data_dir, tokenizer_type)
if tokenizer_type == "char":
from .char_tokenizer import CharTokenizer
return CharTokenizer.load(path)
if tokenizer_type == "bpe":
from .bpe_tokenizer import BPETokenizer
return BPETokenizer.load(path)
raise ValueError(f"Unknown tokenizer type: {tokenizer_type!r}")
|