Multi-Rag / src /service /vector_store /pinecone.py
VashuTheGreat2's picture
Upload folder using huggingface_hub
644f326 verified
Raw History Blame Contribute Delete
6.3 kB
from functools import lru_cache
from typing import List, Optional, Union, Any
import uuid
import sys
import os
from langchain_core.documents import Document
from src.app_config import get_app_config
from src.core.logger import logger
from src.core.exception import MyException
from src.service.embedding.base import BaseEmbeddingService
from src.service.vector_store.base import BaseVectorStore
class PineconeVectorStore(BaseVectorStore):
def __init__(
self,
index_name: Optional[str] = None,
api_key: Optional[str] = None,
embedding_service: Optional[BaseEmbeddingService] = None,
**kwargs,
):
super().__init__(embedding_service=embedding_service, **kwargs)
self.app_config = get_app_config()
self.index_name = index_name or getattr(self.app_config, "collection_name", None) or "default-index"
self.api_key = api_key or os.getenv("PINECONE_API_KEY")
self._pc = None
self._index = None
def _get_client(self):
if self._pc is None:
try:
from pinecone import Pinecone
if not self.api_key:
raise ValueError("PINECONE_API_KEY is not set or provided.")
logger.info("pinecone.client_initializing")
self._pc = Pinecone(api_key=self.api_key)
logger.info("pinecone.client_initialized", index=self.index_name)
except ImportError:
raise ImportError("Pinecone client is not installed. Install with `pip install pinecone`.")
except Exception:
logger.exception("pinecone.client_init_failed")
raise MyException(sys.exc_info()[1], sys)
return self._pc
def _get_index(self):
if self._index is None:
pc = self._get_client()
self._index = pc.Index(self.index_name)
return self._index
async def ensure_collection(self) -> None:
with logger.span("pinecone.ensure_collection", index=self.index_name):
try:
pc = self._get_client()
existing_indexes = [idx.name for idx in pc.list_indexes()]
if self.index_name not in existing_indexes:
dim = int(self.app_config.embedding_dim) if self.app_config.embedding_dim else 384
from pinecone import ServerlessSpec
pc.create_index(
name=self.index_name,
dimension=dim,
metric="cosine",
spec=ServerlessSpec(cloud="aws", region="us-east-1"),
)
logger.info("pinecone.index_created", index=self.index_name, dim=dim)
else:
logger.debug("pinecone.index_exists", index=self.index_name)
except Exception:
logger.exception("pinecone.collection_ensure_failed", index=self.index_name)
raise MyException(sys.exc_info()[1], sys)
async def upsert(self, documents: Union[Document, List[Document]], **kwargs) -> Any:
if isinstance(documents, Document):
documents = [documents]
if not documents:
logger.debug("pinecone.empty_upsert", index=self.index_name)
return True
with logger.span("pinecone.upsert", index=self.index_name, count=len(documents)):
try:
logger.info("pinecone.upsert_started", index=self.index_name, count=len(documents))
await self.ensure_collection()
texts = [
doc.page_content.decode("utf-8") if isinstance(doc.page_content, bytes) else str(doc.page_content)
for doc in documents
]
embeddings = self.embedder.embed_documents(texts)
vectors = []
for doc, emb in zip(documents, embeddings):
metadata = doc.metadata.copy() if getattr(doc, "metadata", None) else {}
content = (
doc.page_content.decode("utf-8")
if isinstance(doc.page_content, bytes)
else str(doc.page_content)
)
metadata["page_content"] = content
vectors.append({
"id": str(uuid.uuid4()),
"values": emb,
"metadata": metadata,
})
index = self._get_index()
result = index.upsert(vectors=vectors)
logger.info("pinecone.upsert_completed", index=self.index_name, count=len(vectors))
return result or True
except Exception:
logger.exception("pinecone.upsert_failed", index=self.index_name, count=len(documents))
raise MyException(sys.exc_info()[1], sys)
async def similarity_search(self, query: str, k: int = 4, **kwargs) -> List[Document]:
with logger.span("pinecone.similarity_search", index=self.index_name, k=k):
try:
logger.info("pinecone.search_started", index=self.index_name, query_preview=query[:50], k=k)
query_vector = self.embedder.embed_query(query)
index = self._get_index()
response = index.query(vector=query_vector, top_k=k, include_metadata=True)
documents = []
for match in response.get("matches", []):
meta = match.get("metadata", {})
content = meta.pop("page_content", "")
documents.append(Document(page_content=content, metadata=meta))
logger.info("pinecone.search_completed", index=self.index_name, results_count=len(documents))
return documents
except Exception:
logger.exception("pinecone.search_failed", index=self.index_name, k=k)
raise MyException(sys.exc_info()[1], sys)
@lru_cache
def get_pinecone_service(
index_name: Optional[str] = None,
embedding_service: Optional[BaseEmbeddingService] = None,
) -> PineconeVectorStore:
return PineconeVectorStore(index_name=index_name, embedding_service=embedding_service)