Spaces:
Runtime error
Runtime error
File size: 993 Bytes
644f326 a76db1d 644f326 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 | from langchain_text_splitters import RecursiveCharacterTextSplitter
from langchain_core.documents import Document
from typing import List
from src.core.exception import MyException
from src.core.logger import logger
import sys
def split(documents: List[Document]) -> List[Document]:
with logger.span("Splitting documents into chunks", doc_count=len(documents)):
try:
logger.info("Beginning chunking process", doc_count=len(documents))
splitter = RecursiveCharacterTextSplitter(
separators=["\n\n", "\n",". "," ", ""],
chunk_size=1000,
chunk_overlap=200,
)
chunks = splitter.split_documents(documents=documents)
logger.info("Document splitting complete", input_docs=len(documents), output_chunks=len(chunks))
return chunks
except Exception as e:
logger.error("Document splitting failed", error=str(e))
raise MyException(e, sys)
|