Remove unused tokenizer training code (train_bpe, load_sentencepiece, load_trained_hf)

Since the project now uses the pretrained LLaMA 2 tokenizer exclusively,
remove all custom tokenizer training infrastructure that is no longer called.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

llm_lab/config/data_config.py

@@ -33,9 +33,5 @@ class DataConfig:
     num_workers: int = 2                       # number of DataLoader workers
     prefetch_factor: int = 4                   # number of batches to prefetch
 
-    # ── Tokenizer training settings (when training from scratch) ──
-    tokenizer_train_samples: int = 50_000      # number of documents to use for training
-    tokenizer_save_dir: str = "./tokenizer"
-
     # ── Validation data ──
     val_ratio: float = 0.001                   # use 0.1% of total data for validation

llm_lab/data/__init__.py

@@ -1,11 +1,11 @@
 """Data pipeline module — tokenizer, streaming, and sequence packing."""
 from .tokenizer import Tokenizer
 from .dataset import PackedStreamingDataset, ValidationDataset
-from .pipeline import create_train_dataloader, train_tokenizer_from_dataset, setup_data_pipeline
+from .pipeline import create_train_dataloader, setup_data_pipeline
 from .diagnostics import DataPipelineDiagnostics
 
 __all__ = [
     "Tokenizer", "PackedStreamingDataset", "ValidationDataset",
-    "create_train_dataloader", "train_tokenizer_from_dataset",
-    "setup_data_pipeline", "DataPipelineDiagnostics",
+    "create_train_dataloader", "setup_data_pipeline",
+    "DataPipelineDiagnostics",
 ]

llm_lab/data/pipeline.py

@@ -1,4 +1,4 @@
-"""Data pipeline integration — DataLoader creation, tokenizer training, and Quick Start."""
+"""Data pipeline integration — DataLoader creation and Quick Start."""
 
 from typing import Optional
 
@@ -47,45 +47,6 @@ def create_train_dataloader(
     return dataloader
 
 
-def train_tokenizer_from_dataset(config: DataConfig) -> Tokenizer:
-    """Trains a BPE tokenizer from the dataset.
-
-    There is no need to use the entire dataset; 50K documents is sufficient,
-    since the tokenizer vocab only needs to reflect the statistics of the full data.
-    """
-    from datasets import load_dataset
-
-    print(f"[Train Tokenizer] Training tokenizer from {config.dataset_name}")
-    print(f"[Train Tokenizer] Number of training documents: {config.tokenizer_train_samples:,}")
-
-    # Create text iterator
-    ds = load_dataset(
-        config.dataset_name,
-        name=config.dataset_subset,
-        split=config.dataset_split,
-        streaming=True,
-        trust_remote_code=True,
-    )
-
-    def text_iterator():
-        count = 0
-        for example in ds:
-            if count >= config.tokenizer_train_samples:
-                break
-            text = example[config.text_column]
-            if text and text.strip():
-                yield text
-                count += 1
-                if count % 10_000 == 0:
-                    print(f"  ... {count:,} documents processed")
-
-    # Train tokenizer
-    tokenizer = Tokenizer(config)
-    tokenizer.train_bpe(text_iterator(), save_dir=config.tokenizer_save_dir)
-
-    return tokenizer
-
-
 def setup_data_pipeline(
     config: Optional[DataConfig] = None,
 ) -> tuple:

llm_lab/data/tokenizer.py

@@ -1,24 +1,15 @@
-"""Tokenizer wrapper — SentencePiece / HuggingFace BPE integration."""
+"""Tokenizer wrapper — loads a pretrained HuggingFace tokenizer."""
 
-import os
-import json
-from typing import Optional, Iterator, List
+from typing import Optional, List
 
 from llm_lab.config import DataConfig
 
 
 class Tokenizer:
-    """Unified tokenizer wrapper.
+    """Pretrained tokenizer wrapper.
 
-    Supports three methods:
-      1) Load an existing SentencePiece model
-      2) Train a new tokenizer using the HuggingFace tokenizers library
-      3) Load a pretrained HF tokenizer (e.g., LLaMA tokenizer)
-
-    Why not implement from scratch?
-      - Training a BPE tokenizer involves large-scale text statistics processing,
-        which has little direct relevance to understanding model architecture.
-      - However, understanding how a tokenizer works (BPE merge rules) is still important.
+    Loads a pretrained HF tokenizer (e.g., LLaMA 2 tokenizer) and provides
+    a unified encode/decode interface for the training pipeline.
 
     BPE (Byte Pair Encoding) core principle:
       1) Split text into byte/character units
@@ -37,106 +28,6 @@ class Tokenizer:
         self.eos_id: int = 2   # End of Sequence
         self.pad_id: int = 0   # Padding
 
-    # ────────────────────────────────────────────────
-    # Method 1: Load a SentencePiece model
-    # ────────────────────────────────────────────────
-
-    def load_sentencepiece(self, model_path: str):
-        """Loads an existing SentencePiece model."""
-        import sentencepiece as spm
-
-        self._tokenizer = spm.SentencePieceProcessor()
-        self._tokenizer.Load(model_path)
-
-        self.vocab_size = self._tokenizer.GetPieceSize()
-        self.bos_id = self._tokenizer.bos_id()
-        self.eos_id = self._tokenizer.eos_id()
-        self.pad_id = self._tokenizer.pad_id()
-        self._encode_fn = self._tokenizer.Encode
-        self._decode_fn = self._tokenizer.Decode
-
-        print(f"[Tokenizer] SentencePiece loaded: vocab_size={self.vocab_size}")
-
-    # ────────────────────────────────────────────────
-    # Method 2: Train a BPE tokenizer with HuggingFace tokenizers
-    # ────────────────────────────────────────────────
-
-    def train_bpe(self, text_iterator: Iterator[str], save_dir: Optional[str] = None):
-        """Trains a BPE tokenizer from scratch.
-
-        Args:
-            text_iterator: Iterator that yields training text strings
-            save_dir: Directory path to save the trained tokenizer
-
-        Key insights:
-          - Larger vocab_size: common expressions become 1 token → shorter sequences
-          - Smaller vocab_size: saves embedding parameters, but sequences get longer
-          - 32K is a good balance point for English
-        """
-        from tokenizers import Tokenizer as HFTokenizer
-        from tokenizers.models import BPE
-        from tokenizers.trainers import BpeTrainer
-        from tokenizers.pre_tokenizers import ByteLevel
-        from tokenizers.decoders import ByteLevel as ByteLevelDecoder
-        from tokenizers.processors import TemplateProcessing
-
-        print("[Tokenizer] Starting BPE tokenizer training...")
-
-        # Create BPE model
-        tokenizer = HFTokenizer(BPE(unk_token="<unk>"))
-        tokenizer.pre_tokenizer = ByteLevel(add_prefix_space=False)
-        tokenizer.decoder = ByteLevelDecoder()
-
-        # Define special tokens
-        special_tokens = ["<pad>", "<s>", "</s>", "<unk>"]
-
-        # Configure trainer
-        trainer = BpeTrainer(
-            vocab_size=self.config.vocab_size,
-            special_tokens=special_tokens,
-            min_frequency=2,           # Only merge pairs that appear at least twice
-            show_progress=True,
-        )
-
-        # Run training
-        tokenizer.train_from_iterator(text_iterator, trainer=trainer)
-
-        # Post-processing: automatically add BOS/EOS
-        tokenizer.post_processor = TemplateProcessing(
-            single="<s> $A </s>",
-            special_tokens=[("<s>", 1), ("</s>", 2)],
-        )
-
-        self._tokenizer = tokenizer
-        self.vocab_size = tokenizer.get_vocab_size()
-        self.pad_id = 0
-        self.bos_id = 1
-        self.eos_id = 2
-
-        self._encode_fn = lambda text: tokenizer.encode(text).ids
-        self._decode_fn = lambda ids: tokenizer.decode(ids)
-
-        # Save
-        save_dir = save_dir or self.config.tokenizer_save_dir
-        os.makedirs(save_dir, exist_ok=True)
-        tokenizer.save(os.path.join(save_dir, "tokenizer.json"))
-        # Save metadata
-        meta = {
-            "vocab_size": self.vocab_size,
-            "bos_id": self.bos_id,
-            "eos_id": self.eos_id,
-            "pad_id": self.pad_id,
-        }
-        with open(os.path.join(save_dir, "tokenizer_meta.json"), "w") as f:
-            json.dump(meta, f, indent=2)
-
-        print(f"[Tokenizer] Training complete: vocab_size={self.vocab_size}")
-        print(f"[Tokenizer] Saved to: {save_dir}")
-
-    # ────────────────────────────────────────────────
-    # Method 3: Load a pretrained HF tokenizer
-    # ────────────────────────────────────────────────
-
     def load_pretrained_hf(self, name_or_path: Optional[str] = None):
         """Loads a pretrained tokenizer from HuggingFace.
 
@@ -164,29 +55,6 @@ class Tokenizer:
 
         print(f"[Tokenizer] Loaded: vocab_size={self.vocab_size}")
 
-    def load_trained_hf(self, path: str):
-        """Reloads a tokenizer previously trained with train_bpe()."""
-        from tokenizers import Tokenizer as HFTokenizer
-        from tokenizers.decoders import ByteLevel as ByteLevelDecoder
-
-        tokenizer = HFTokenizer.from_file(os.path.join(path, "tokenizer.json"))
-        # Ensure ByteLevel decoder is set (may be missing in older tokenizer files)
-        if tokenizer.decoder is None:
-            tokenizer.decoder = ByteLevelDecoder()
-        with open(os.path.join(path, "tokenizer_meta.json"), "r") as f:
-            meta = json.load(f)
-
-        self._tokenizer = tokenizer
-        self.vocab_size = meta["vocab_size"]
-        self.bos_id = meta["bos_id"]
-        self.eos_id = meta["eos_id"]
-        self.pad_id = meta["pad_id"]
-
-        self._encode_fn = lambda text: tokenizer.encode(text).ids
-        self._decode_fn = lambda ids: tokenizer.decode(ids)
-
-        print(f"[Tokenizer] Loaded: vocab_size={self.vocab_size}")
-
     # ────────────────────────────────────────────────
     # Common interface
     # ────────────────────────────────────────────────