src/text_utils.py

"""Text chunking helpers for the RAG pipeline."""

from langchain_text_splitters import RecursiveCharacterTextSplitter

from src.config import CHUNK_OVERLAP, CHUNK_SIZE


def split_pages_into_chunks(pages: list[dict]) -> list[dict]:
    """Split extracted page text into smaller chunks with source metadata."""
    splitter = RecursiveCharacterTextSplitter(
        chunk_size=CHUNK_SIZE,
        chunk_overlap=CHUNK_OVERLAP,
        separators=["\n\n", "\n", ". ", " ", ""],
    )

    chunks = []
    chunk_number = 1

    for page in pages:
        page_chunks = splitter.split_text(page["text"])

        for chunk_text in page_chunks:
            chunks.append(
                {
                    "text": chunk_text,
                    "chunk_id": chunk_number,
                    "page_number": page["page_number"],
                    "source_file": page["source_file"],
                    "extraction_method": page.get("extraction_method", "embedded_text"),
                    "ocr_mode": page.get("ocr_mode", "not_used"),
                    "ocr_engine": page.get("ocr_engine", "not_used"),
                }
            )
            chunk_number += 1

    if not chunks:
        raise ValueError("The uploaded notes could not be split into text chunks.")

    return chunks