Multi-Rag / src /ingestion /chunking /splitter.py
VashuTheGreat2's picture
Upload folder using huggingface_hub
a76db1d verified
Raw History Blame Contribute Delete
993 Bytes
from langchain_text_splitters import RecursiveCharacterTextSplitter
from langchain_core.documents import Document
from typing import List
from src.core.exception import MyException
from src.core.logger import logger
import sys
def split(documents: List[Document]) -> List[Document]:
with logger.span("Splitting documents into chunks", doc_count=len(documents)):
try:
logger.info("Beginning chunking process", doc_count=len(documents))
splitter = RecursiveCharacterTextSplitter(
separators=["\n\n", "\n",". "," ", ""],
chunk_size=1000,
chunk_overlap=200,
)
chunks = splitter.split_documents(documents=documents)
logger.info("Document splitting complete", input_docs=len(documents), output_chunks=len(chunks))
return chunks
except Exception as e:
logger.error("Document splitting failed", error=str(e))
raise MyException(e, sys)