Spaces:
Runtime error
Runtime error
Download src/ingestion/chunking/splitter.py from VashuTheGreat2/Multi-Rag: direct link, hf CLI and curl.
- Browser
- Download file 993 Bytes
-
https://huggingface.co/spaces/VashuTheGreat2/Multi-Rag/resolve/main/src/ingestion/chunking/splitter.py
- Command line
-
hf download hf://spaces/VashuTheGreat2/Multi-Rag/src/ingestion/chunking/splitter.py
-
curl -L -o splitter.py https://huggingface.co/spaces/VashuTheGreat2/Multi-Rag/resolve/main/src/ingestion/chunking/splitter.py
993 Bytes
| from langchain_text_splitters import RecursiveCharacterTextSplitter | |
| from langchain_core.documents import Document | |
| from typing import List | |
| from src.core.exception import MyException | |
| from src.core.logger import logger | |
| import sys | |
| def split(documents: List[Document]) -> List[Document]: | |
| with logger.span("Splitting documents into chunks", doc_count=len(documents)): | |
| try: | |
| logger.info("Beginning chunking process", doc_count=len(documents)) | |
| splitter = RecursiveCharacterTextSplitter( | |
| separators=["\n\n", "\n",". "," ", ""], | |
| chunk_size=1000, | |
| chunk_overlap=200, | |
| ) | |
| chunks = splitter.split_documents(documents=documents) | |
| logger.info("Document splitting complete", input_docs=len(documents), output_chunks=len(chunks)) | |
| return chunks | |
| except Exception as e: | |
| logger.error("Document splitting failed", error=str(e)) | |
| raise MyException(e, sys) | |