from src.ingestion.loaders import html, office, pdf, text
from src.ingestion.chunking.splitter import split
from src.service import vector_store as vector_store_module
from src.service.vector_store import get_vector_store, get_client
from src.core.logger import logger
from src.core.exception import MyException
import os
import sys
client = get_client()
def map_file_type(file_path: str):
ext = file_path.split(".")[-1].lower() if "." in file_path else ""
logger.debug("Mapping file extension", file_path=file_path, extension=ext)
match ext:
case "pdf":
return pdf.pdf_loader
case "txt":
return text.text_loader
case "html" | "htm":
return html.html_loader
case "docx" | "doc" | "ppt" | "pptx" | "word":
return office.office_loader
case _:
return text.text_loader
async def process_file(file_path: str = "data/text.txt"):
with logger.span("Ingestion pipeline: {file_path}", file_path=file_path):
try:
logger.info("Starting file processing", file_path=file_path)
loader = map_file_type(file_path=file_path)
documents = await loader(file_path=file_path)
logger.info("Loaded raw documents", file_path=file_path, count=len(documents))
documents = split(documents)
logger.info("Documents split into chunks", file_path=file_path, chunk_count=len(documents))
vs_client = vector_store_module.get_client()
result = await vs_client.upsert(documents)
logger.info("File processing and upsert complete", file_path=file_path)
return result if result is not None else True
except Exception as e:
logger.error("Failed to process file", file_path=file_path, error=str(e))
raise MyException(e, sys)
async def process_folder(folder_path: str = "data"):
with logger.span("Ingesting folder: {folder_path}", folder_path=folder_path):
try:
logger.info("Traversing folder", folder_path=folder_path)
results = []
for root, _, files in os.walk(folder_path):
for file in sorted(files):
file_path = os.path.join(root, file)
res = await process_file(file_path=file_path)
results.append(res)
logger.info("Folder processing complete", folder_path=folder_path, processed_files=len(results))
return results
except Exception as e:
logger.error("Failed to process folder", folder_path=folder_path, error=str(e))
raise MyException(e, sys)