src/data/__init__.py

# Data loading and tokenizer components

from .tokenizer import SLMTokenizer
from .dataset import (
    ConversationalDataset,
    StreamingTextDataset,
    PackedDataset,
    create_train_val_split,
    load_jsonl,
    save_jsonl,
)
from .dataloader import (
    DataModule,
    StreamingDataModule,
    create_dataloader,
    estimate_dataset_tokens,
)

__all__ = [
    "SLMTokenizer",
    "ConversationalDataset",
    "StreamingTextDataset",
    "PackedDataset",
    "create_train_val_split",
    "load_jsonl",
    "save_jsonl",
    "DataModule",
    "StreamingDataModule",
    "create_dataloader",
    "estimate_dataset_tokens",
]