tools/document_loader_tool.py

# COST: OpenAI text-embedding-3-small called during indexing (one-time per document).
# No chat LLM is invoked. Embedding cost is ~$0.00002/1K tokens — negligible for
# typical documents. Upsert semantics mean re-indexing the same file is idempotent
# but does re-embed (and re-bill) all chunks.

import sys
from pathlib import Path

sys.path.insert(0, str(Path(__file__).parent.parent))

from dotenv import load_dotenv

load_dotenv()

from ingestion import chunk_text, embed_and_store, parse_document


def load_and_index_document(file_path: str) -> str:
    """Parse, chunk, embed, and store a single document into the vector store.

    Accepts a .pdf or .docx file, extracts text (preserving per-page structure
    for PDFs), splits it into overlapping 500-word chunks with a 50-word overlap,
    and upserts every chunk with source metadata into the persistent ChromaDB
    collection.  Calling this function on an already-indexed file is safe — all
    chunks are upserted so duplicates are automatically overwritten.

    Args:
        file_path: Absolute or relative path to the document to ingest.
                   Must be a .pdf or .docx file.

    Returns:
        A human-readable confirmation string such as:
            "Successfully indexed 'report.pdf': 42 chunks stored across 10 page(s)."
        If the file cannot be parsed an error message is returned instead of
        raising, so the agent can surface it gracefully.
    """
    path = Path(file_path)

    if not path.exists():
        return f"Error: file not found at '{file_path}'."

    try:
        pages = parse_document(path)
    except ValueError as exc:
        return f"Error: {exc}"

    total_chunks = 0
    for page in pages:
        chunks = chunk_text(page["text"])
        embed_and_store(
            chunks,
            metadata={
                "source_file": path.name,
                "page_number": page["page_number"],
            },
        )
        total_chunks += len(chunks)

    page_count = len(pages)
    page_label = f"{page_count} page(s)" if pages[0]["page_number"] is not None else "1 section"
    return (
        f"Successfully indexed '{path.name}': "
        f"{total_chunks} chunks stored across {page_label}."
    )