File size: 1,546 Bytes
b915243 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 | from typing import List
from langchain_text_splitters import RecursiveCharacterTextSplitter
from langchain_core.documents import Document
def get_text_splitter(chunk_size: int = 1000, chunk_overlap: int = 200) -> RecursiveCharacterTextSplitter:
"""
Returns a RecursiveCharacterTextSplitter configured for medical documents.
Why chunking is required:
1. Context Windows: LLMs have a fixed context window limit (e.g., 4k tokens for Phi-3 Mini).
We cannot pass an entire 50-page PDF at once.
2. Precision vs Recall:
- Smaller chunks (precision) allow the retriever to find exact answers and reduce distraction/hallucination.
- Overlap (recall) ensures context is not broken across chunks (e.g., a diagnosis spanning two paragraphs).
The RecursiveCharacterTextSplitter splits by paragraphs, then sentences, then words,
preserving semantic meaning better than a simple character splitter.
"""
return RecursiveCharacterTextSplitter(
chunk_size=chunk_size,
chunk_overlap=chunk_overlap,
length_function=len,
add_start_index=True,
)
def split_documents(documents: List[Document], chunk_size: int = 1000, chunk_overlap: int = 200) -> List[Document]:
"""
Splits a list of LangChain Document objects into smaller chunks.
Metadata (like filename and page number) from the parent document is preserved in all child chunks.
"""
splitter = get_text_splitter(chunk_size, chunk_overlap)
return splitter.split_documents(documents)
|