Spaces:
Configuration error
Configuration error
File size: 1,889 Bytes
05269f9 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 | from typing import List, Dict, Any, Optional, Union
from langchain_text_splitters import RecursiveCharacterTextSplitter
import asyncio
import logging
from langchain_core.documents import Document
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)
class DocumentProcessor:
"""Document processor for chunking and preprocessing"""
def __init__(self,
chunk_size: int = 1000,
chunk_overlap: int = 200,
separators: Optional[List[str]] = None):
self.chunk_size = chunk_size
self.chunk_overlap = chunk_overlap
if separators is None:
separators = ["\n\n", "\n", " ", ""]
self.text_splitter = RecursiveCharacterTextSplitter(
chunk_size=chunk_size,
chunk_overlap=chunk_overlap,
separators=separators,
length_function=len
)
async def process_documents(self, documents: List[Document]) -> List[Document]:
"""Process documents by splitting into chunks"""
try:
logger.info(f"Processing {len(documents)} documents")
chunks = await asyncio.get_event_loop().run_in_executor(
None, self.text_splitter.split_documents, documents
)
for i, chunk in enumerate(chunks):
chunk.metadata.update({
"chunk_id": i,
"chunk_size": len(chunk.page_content),
"processed_at": str(asyncio.get_event_loop().time())
})
logger.info(f"Created {len(chunks)} chunks from {len(documents)} documents")
return chunks
except Exception as e:
logger.error(f"Error processing documents: {str(e)}")
raise |