Spaces:
Runtime error
Runtime error
Download src/service/vector_store/pinecone.py from VashuTheGreat2/Multi-Rag: direct link, hf CLI and curl.
- Browser
- Download file 6.3 kB
-
https://huggingface.co/spaces/VashuTheGreat2/Multi-Rag/resolve/main/src/service/vector_store/pinecone.py
- Command line
-
hf download hf://spaces/VashuTheGreat2/Multi-Rag/src/service/vector_store/pinecone.py
-
curl -L -o pinecone.py https://huggingface.co/spaces/VashuTheGreat2/Multi-Rag/resolve/main/src/service/vector_store/pinecone.py
6.3 kB
| from functools import lru_cache | |
| from typing import List, Optional, Union, Any | |
| import uuid | |
| import sys | |
| import os | |
| from langchain_core.documents import Document | |
| from src.app_config import get_app_config | |
| from src.core.logger import logger | |
| from src.core.exception import MyException | |
| from src.service.embedding.base import BaseEmbeddingService | |
| from src.service.vector_store.base import BaseVectorStore | |
| class PineconeVectorStore(BaseVectorStore): | |
| def __init__( | |
| self, | |
| index_name: Optional[str] = None, | |
| api_key: Optional[str] = None, | |
| embedding_service: Optional[BaseEmbeddingService] = None, | |
| **kwargs, | |
| ): | |
| super().__init__(embedding_service=embedding_service, **kwargs) | |
| self.app_config = get_app_config() | |
| self.index_name = index_name or getattr(self.app_config, "collection_name", None) or "default-index" | |
| self.api_key = api_key or os.getenv("PINECONE_API_KEY") | |
| self._pc = None | |
| self._index = None | |
| def _get_client(self): | |
| if self._pc is None: | |
| try: | |
| from pinecone import Pinecone | |
| if not self.api_key: | |
| raise ValueError("PINECONE_API_KEY is not set or provided.") | |
| logger.info("pinecone.client_initializing") | |
| self._pc = Pinecone(api_key=self.api_key) | |
| logger.info("pinecone.client_initialized", index=self.index_name) | |
| except ImportError: | |
| raise ImportError("Pinecone client is not installed. Install with `pip install pinecone`.") | |
| except Exception: | |
| logger.exception("pinecone.client_init_failed") | |
| raise MyException(sys.exc_info()[1], sys) | |
| return self._pc | |
| def _get_index(self): | |
| if self._index is None: | |
| pc = self._get_client() | |
| self._index = pc.Index(self.index_name) | |
| return self._index | |
| async def ensure_collection(self) -> None: | |
| with logger.span("pinecone.ensure_collection", index=self.index_name): | |
| try: | |
| pc = self._get_client() | |
| existing_indexes = [idx.name for idx in pc.list_indexes()] | |
| if self.index_name not in existing_indexes: | |
| dim = int(self.app_config.embedding_dim) if self.app_config.embedding_dim else 384 | |
| from pinecone import ServerlessSpec | |
| pc.create_index( | |
| name=self.index_name, | |
| dimension=dim, | |
| metric="cosine", | |
| spec=ServerlessSpec(cloud="aws", region="us-east-1"), | |
| ) | |
| logger.info("pinecone.index_created", index=self.index_name, dim=dim) | |
| else: | |
| logger.debug("pinecone.index_exists", index=self.index_name) | |
| except Exception: | |
| logger.exception("pinecone.collection_ensure_failed", index=self.index_name) | |
| raise MyException(sys.exc_info()[1], sys) | |
| async def upsert(self, documents: Union[Document, List[Document]], **kwargs) -> Any: | |
| if isinstance(documents, Document): | |
| documents = [documents] | |
| if not documents: | |
| logger.debug("pinecone.empty_upsert", index=self.index_name) | |
| return True | |
| with logger.span("pinecone.upsert", index=self.index_name, count=len(documents)): | |
| try: | |
| logger.info("pinecone.upsert_started", index=self.index_name, count=len(documents)) | |
| await self.ensure_collection() | |
| texts = [ | |
| doc.page_content.decode("utf-8") if isinstance(doc.page_content, bytes) else str(doc.page_content) | |
| for doc in documents | |
| ] | |
| embeddings = self.embedder.embed_documents(texts) | |
| vectors = [] | |
| for doc, emb in zip(documents, embeddings): | |
| metadata = doc.metadata.copy() if getattr(doc, "metadata", None) else {} | |
| content = ( | |
| doc.page_content.decode("utf-8") | |
| if isinstance(doc.page_content, bytes) | |
| else str(doc.page_content) | |
| ) | |
| metadata["page_content"] = content | |
| vectors.append({ | |
| "id": str(uuid.uuid4()), | |
| "values": emb, | |
| "metadata": metadata, | |
| }) | |
| index = self._get_index() | |
| result = index.upsert(vectors=vectors) | |
| logger.info("pinecone.upsert_completed", index=self.index_name, count=len(vectors)) | |
| return result or True | |
| except Exception: | |
| logger.exception("pinecone.upsert_failed", index=self.index_name, count=len(documents)) | |
| raise MyException(sys.exc_info()[1], sys) | |
| async def similarity_search(self, query: str, k: int = 4, **kwargs) -> List[Document]: | |
| with logger.span("pinecone.similarity_search", index=self.index_name, k=k): | |
| try: | |
| logger.info("pinecone.search_started", index=self.index_name, query_preview=query[:50], k=k) | |
| query_vector = self.embedder.embed_query(query) | |
| index = self._get_index() | |
| response = index.query(vector=query_vector, top_k=k, include_metadata=True) | |
| documents = [] | |
| for match in response.get("matches", []): | |
| meta = match.get("metadata", {}) | |
| content = meta.pop("page_content", "") | |
| documents.append(Document(page_content=content, metadata=meta)) | |
| logger.info("pinecone.search_completed", index=self.index_name, results_count=len(documents)) | |
| return documents | |
| except Exception: | |
| logger.exception("pinecone.search_failed", index=self.index_name, k=k) | |
| raise MyException(sys.exc_info()[1], sys) | |
| def get_pinecone_service( | |
| index_name: Optional[str] = None, | |
| embedding_service: Optional[BaseEmbeddingService] = None, | |
| ) -> PineconeVectorStore: | |
| return PineconeVectorStore(index_name=index_name, embedding_service=embedding_service) | |