scripts/ingest.py

"""
One-time (or re-runnable) ingestion script.

Loads all PDFs from the documents/ directory, chunks them, embeds the
chunks, and upserts them into Pinecone. Run this once before starting the
API, and re-run whenever documents/ changes.

Usage:
    python -m scripts.ingest
"""

import logging
import sys
from pathlib import Path

from app.core.config import settings
from app.core.logging_config import configure_logging
from app.services import chunker, embedder, pdf_processor, vector_store

configure_logging()
logger = logging.getLogger(__name__)


def main() -> None:
    docs_dir = Path(settings.DOCUMENTS_DIR)

    logger.info(f"Starting ingestion from '{docs_dir}'...")

    try:
        all_pages = pdf_processor.load_all_documents(docs_dir)
    except pdf_processor.PDFProcessingError as e:
        logger.error(f"Ingestion aborted — PDF processing failed: {e}")
        sys.exit(1)

    try:
        all_chunks = chunker.chunk_all_documents(
            all_pages,
            chunk_size_words=settings.CHUNK_SIZE_WORDS,
            overlap_words=settings.CHUNK_OVERLAP_WORDS,
        )
    except ValueError as e:
        logger.error(f"Ingestion aborted — chunking failed: {e}")
        sys.exit(1)

    try:
        texts = [c.text for c in all_chunks]
        embeddings = embedder.embed_texts(texts)
    except embedder.EmbeddingError as e:
        logger.error(f"Ingestion aborted — embedding failed: {e}")
        sys.exit(1)

    try:
        vector_store.ensure_index_exists()
        upserted_count = vector_store.upsert_chunks(all_chunks, embeddings)
    except vector_store.VectorStoreError as e:
        logger.error(f"Ingestion aborted — vector store operation failed: {e}")
        sys.exit(1)

    logger.info(f"✅ Ingestion complete. {upserted_count} chunks indexed into Pinecone.")


if __name__ == "__main__":
    main()