llm_lab/data/__init__.py

"""Data pipeline module — tokenizer, streaming, and sequence packing."""
from .tokenizer import Tokenizer
from .dataset import PackedStreamingDataset, MixedStreamingDataset, ValidationDataset
from .pipeline import create_train_dataloader, setup_data_pipeline, setup_cpt_data_pipeline
from .diagnostics import DataPipelineDiagnostics

__all__ = [
    "Tokenizer", "PackedStreamingDataset", "MixedStreamingDataset", "ValidationDataset",
    "create_train_dataloader", "setup_data_pipeline", "setup_cpt_data_pipeline",
    "DataPipelineDiagnostics",
]