Spaces:
Sleeping
Sleeping
File size: 3,045 Bytes
49f0cfb dc1b199 49f0cfb dc1b199 49f0cfb dc1b199 49f0cfb dc1b199 49f0cfb dc1b199 49f0cfb dc1b199 49f0cfb dc1b199 49f0cfb dc1b199 49f0cfb dc1b199 49f0cfb dc1b199 49f0cfb dc1b199 49f0cfb dc1b199 49f0cfb dc1b199 49f0cfb dc1b199 49f0cfb dc1b199 d611f45 49f0cfb | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 | """Unit tests for the LangChain-based text splitter.
LangChain experiment branch: the splitter now operates on
``List[langchain_core.documents.Document]`` and returns the same type.
The old ``split_chunks()`` / ``ParsedBlock`` / ``Chunk`` API is replaced
by ``split_documents()`` using ``RecursiveCharacterTextSplitter``.
"""
from langchain_core.documents import Document
from app.chunking.splitter import count_tokens, split_documents
def _make_docs(texts: list[str]) -> list[Document]:
return [Document(page_content=t, metadata={"source": "test"}) for t in texts]
def test_count_tokens_basic() -> None:
"""count_tokens should return a positive integer for non-empty strings."""
assert count_tokens("hello world") > 0
assert count_tokens("") == 0
def test_split_documents_returns_documents() -> None:
"""split_documents should return a list of LangChain Document objects."""
docs = _make_docs(["This is a short test paragraph."])
chunks = split_documents(docs)
assert isinstance(chunks, list)
assert len(chunks) >= 1
assert all(isinstance(c, Document) for c in chunks)
def test_split_documents_inherits_metadata() -> None:
"""Each chunk Document should carry the parent's metadata."""
docs = _make_docs(["Short paragraph."])
docs[0].metadata["custom_key"] = "custom_value"
chunks = split_documents(docs)
for chunk in chunks:
assert chunk.metadata.get("custom_key") == "custom_value"
assert chunk.metadata.get("source") == "test"
def test_split_documents_respects_chunk_size() -> None:
"""No chunk should substantially exceed the configured chunk_size in tokens."""
long_text = " ".join(["word"] * 1000)
docs = _make_docs([long_text])
chunk_size = 100
chunks = split_documents(docs, chunk_size=chunk_size, chunk_overlap=10)
for chunk in chunks:
actual_tokens = count_tokens(chunk.page_content)
# Allow a small overshoot due to overlap prepending
assert actual_tokens <= chunk_size + 25, (
f"Chunk too large: {actual_tokens} tokens (limit {chunk_size + 25})"
)
def test_split_documents_empty_input() -> None:
"""split_documents with an empty list should return an empty list."""
assert split_documents([]) == []
def test_split_documents_large_document() -> None:
"""split_documents should handle a large document without error."""
docs = _make_docs([" ".join(["word"] * 200) for _ in range(50)])
chunks = split_documents(docs, chunk_size=300)
assert len(chunks) > 0
def test_split_documents_multiple_pages() -> None:
"""Each page document should be independently split."""
docs = [
Document(page_content="Page one content. " * 10, metadata={"page": 1}),
Document(page_content="Page two content. " * 10, metadata={"page": 2}),
]
chunks = split_documents(docs, chunk_size=50, chunk_overlap=10)
assert len(chunks) >= 2
# Each chunk should have non-empty page_content
assert all(c.page_content.strip() for c in chunks)
|