server/modules/load_vectorstore.py

import os
import time
import uuid
from pathlib import Path

from dotenv import load_dotenv
from pinecone import Pinecone, ServerlessSpec
from langchain_community.document_loaders import PyPDFLoader
from langchain_text_splitters import RecursiveCharacterTextSplitter
from langchain_huggingface import HuggingFaceEmbeddings
from core.settings import get_settings

load_dotenv()

UPLOAD_DIR = Path("./uploaded_docs")
UPLOAD_DIR.mkdir(exist_ok=True)

_splitter = RecursiveCharacterTextSplitter(chunk_size=500, chunk_overlap=50)

# Lazy singleton — Pinecone is NOT connected at import time
_index = None


def get_index():
    """Connect to Pinecone on first call, then reuse the connection."""
    global _index
    if _index is not None:
        return _index

    settings = get_settings()

    pc = Pinecone(api_key=settings.pinecone_api_key)
    spec = ServerlessSpec(cloud="aws", region=settings.pinecone_region)

    if settings.pinecone_index_name not in {i["name"] for i in pc.list_indexes()}:
        pc.create_index(
            name=settings.pinecone_index_name,
            dimension=768,
            metric="dotproduct",
            spec=spec,
        )
        while not pc.describe_index(settings.pinecone_index_name).status["ready"]:
            time.sleep(1)

    _index = pc.Index(settings.pinecone_index_name)
    return _index


def load_vectorstore(uploaded_files) -> int:
    settings = get_settings()
    embed_model = HuggingFaceEmbeddings(model_name="sentence-transformers/all-mpnet-base-v2")
    index = get_index()
    total_chunks = 0

    for file in uploaded_files:
        save_path = UPLOAD_DIR / file.filename
        save_path.write_bytes(file.file.read())

        chunks = _splitter.split_documents(PyPDFLoader(str(save_path)).load())
        if not chunks:
            continue

        texts = [c.page_content for c in chunks]
        metadatas = [c.metadata for c in chunks]
        embeddings = embed_model.embed_documents(texts)

        vectors = [
            {
                "id": f"{save_path.stem}-{i}-{uuid.uuid4().hex[:8]}",
                "values": embeddings[i],
                "metadata": {**metadatas[i], "text": texts[i]},
            }
            for i in range(len(chunks))
        ]

        index.upsert(vectors=vectors)
        total_chunks += len(vectors)

    return total_chunks