app/tests/test_chunker.py

"""Unit tests for the LangChain-based text splitter.

LangChain experiment branch: the splitter now operates on
``List[langchain_core.documents.Document]`` and returns the same type.
The old ``split_chunks()`` / ``ParsedBlock`` / ``Chunk`` API is replaced
by ``split_documents()`` using ``RecursiveCharacterTextSplitter``.
"""

from langchain_core.documents import Document

from app.chunking.splitter import count_tokens, split_documents


def _make_docs(texts: list[str]) -> list[Document]:
    return [Document(page_content=t, metadata={"source": "test"}) for t in texts]


def test_count_tokens_basic() -> None:
    """count_tokens should return a positive integer for non-empty strings."""
    assert count_tokens("hello world") > 0
    assert count_tokens("") == 0


def test_split_documents_returns_documents() -> None:
    """split_documents should return a list of LangChain Document objects."""
    docs = _make_docs(["This is a short test paragraph."])
    chunks = split_documents(docs)
    assert isinstance(chunks, list)
    assert len(chunks) >= 1
    assert all(isinstance(c, Document) for c in chunks)


def test_split_documents_inherits_metadata() -> None:
    """Each chunk Document should carry the parent's metadata."""
    docs = _make_docs(["Short paragraph."])
    docs[0].metadata["custom_key"] = "custom_value"
    chunks = split_documents(docs)
    for chunk in chunks:
        assert chunk.metadata.get("custom_key") == "custom_value"
        assert chunk.metadata.get("source") == "test"


def test_split_documents_respects_chunk_size() -> None:
    """No chunk should substantially exceed the configured chunk_size in tokens."""
    long_text = " ".join(["word"] * 1000)
    docs = _make_docs([long_text])
    chunk_size = 100
    chunks = split_documents(docs, chunk_size=chunk_size, chunk_overlap=10)
    for chunk in chunks:
        actual_tokens = count_tokens(chunk.page_content)
        # Allow a small overshoot due to overlap prepending
        assert actual_tokens <= chunk_size + 25, (
            f"Chunk too large: {actual_tokens} tokens (limit {chunk_size + 25})"
        )


def test_split_documents_empty_input() -> None:
    """split_documents with an empty list should return an empty list."""
    assert split_documents([]) == []


def test_split_documents_large_document() -> None:
    """split_documents should handle a large document without error."""
    docs = _make_docs([" ".join(["word"] * 200) for _ in range(50)])
    chunks = split_documents(docs, chunk_size=300)
    assert len(chunks) > 0


def test_split_documents_multiple_pages() -> None:
    """Each page document should be independently split."""
    docs = [
        Document(page_content="Page one content. " * 10, metadata={"page": 1}),
        Document(page_content="Page two content. " * 10, metadata={"page": 2}),
    ]
    chunks = split_documents(docs, chunk_size=50, chunk_overlap=10)
    assert len(chunks) >= 2
    # Each chunk should have non-empty page_content
    assert all(c.page_content.strip() for c in chunks)