data_loader.py

"""Load raw documents from the data/raw directory."""

import os
from utils import list_files


def load_text_file(path: str) -> dict:
    """Read a single .txt file and return its content with metadata."""
    with open(path, "r", encoding="utf-8") as f:
        text = f.read()
    filename = os.path.basename(path)
    return {
        "source": filename,
        "text": text,
    }


def load_all_documents(raw_dir: str) -> list[dict]:
    """Load all .txt files from raw_dir. Raises if directory is empty."""
    paths = list_files(raw_dir, extension=".txt")
    if not paths:
        raise FileNotFoundError(
            f"No .txt documents found in '{raw_dir}'.\n"
            "Add scientific documents as plain .txt files and rebuild the index."
        )
    documents = [load_text_file(p) for p in paths]
    print(f"[data_loader] Loaded {len(documents)} documents.")
    return documents