Spaces:
Runtime error
Runtime error
Download src/ingestion/processor.py from VashuTheGreat2/Multi-Rag: direct link, hf CLI and curl.
- Browser
- Download file 2.67 kB
-
https://huggingface.co/spaces/VashuTheGreat2/Multi-Rag/resolve/main/src/ingestion/processor.py
- Command line
-
hf download hf://spaces/VashuTheGreat2/Multi-Rag/src/ingestion/processor.py
-
curl -L -o processor.py https://huggingface.co/spaces/VashuTheGreat2/Multi-Rag/resolve/main/src/ingestion/processor.py
2.67 kB
| from src.ingestion.loaders import html, office, pdf, text | |
| from src.ingestion.chunking.splitter import split | |
| from src.service import vector_store as vector_store_module | |
| from src.service.vector_store import get_vector_store, get_client | |
| from src.core.logger import logger | |
| from src.core.exception import MyException | |
| import os | |
| import sys | |
| client = get_client() | |
| def map_file_type(file_path: str): | |
| ext = file_path.split(".")[-1].lower() if "." in file_path else "" | |
| logger.debug("Mapping file extension", file_path=file_path, extension=ext) | |
| match ext: | |
| case "pdf": | |
| return pdf.pdf_loader | |
| case "txt": | |
| return text.text_loader | |
| case "html" | "htm": | |
| return html.html_loader | |
| case "docx" | "doc" | "ppt" | "pptx" | "word": | |
| return office.office_loader | |
| case _: | |
| return text.text_loader | |
| async def process_file(file_path: str = "data/text.txt"): | |
| with logger.span("Ingestion pipeline: {file_path}", file_path=file_path): | |
| try: | |
| logger.info("Starting file processing", file_path=file_path) | |
| loader = map_file_type(file_path=file_path) | |
| documents = await loader(file_path=file_path) | |
| logger.info("Loaded raw documents", file_path=file_path, count=len(documents)) | |
| documents = split(documents) | |
| logger.info("Documents split into chunks", file_path=file_path, chunk_count=len(documents)) | |
| vs_client = vector_store_module.get_client() | |
| result = await vs_client.upsert(documents) | |
| logger.info("File processing and upsert complete", file_path=file_path) | |
| return result if result is not None else True | |
| except Exception as e: | |
| logger.error("Failed to process file", file_path=file_path, error=str(e)) | |
| raise MyException(e, sys) | |
| async def process_folder(folder_path: str = "data"): | |
| with logger.span("Ingesting folder: {folder_path}", folder_path=folder_path): | |
| try: | |
| logger.info("Traversing folder", folder_path=folder_path) | |
| results = [] | |
| for root, _, files in os.walk(folder_path): | |
| for file in sorted(files): | |
| file_path = os.path.join(root, file) | |
| res = await process_file(file_path=file_path) | |
| results.append(res) | |
| logger.info("Folder processing complete", folder_path=folder_path, processed_files=len(results)) | |
| return results | |
| except Exception as e: | |
| logger.error("Failed to process folder", folder_path=folder_path, error=str(e)) | |
| raise MyException(e, sys) | |