Download src/vector_store.py from AdryanAzul/DocuMindAI_RAG_Application: direct link, hf CLI and curl.
- Browser
- Download file 4.76 kB
-
https://huggingface.co/spaces/AdryanAzul/DocuMindAI_RAG_Application/resolve/main/src/vector_store.py
- Command line
-
hf download hf://spaces/AdryanAzul/DocuMindAI_RAG_Application/src/vector_store.py
-
curl -L -o vector_store.py https://huggingface.co/spaces/AdryanAzul/DocuMindAI_RAG_Application/resolve/main/src/vector_store.py
4.76 kB
| """Gerenciamento do banco vetorial ChromaDB com persistência local.""" | |
| import gc | |
| import shutil | |
| from dataclasses import dataclass | |
| from pathlib import Path | |
| from typing import Any | |
| import chromadb | |
| from src.text_splitter import TextChunk | |
| class StoredDocument: | |
| """Representa um trecho recuperado do ChromaDB com metadados e distância.""" | |
| text: str | |
| source: str | |
| page: int | |
| distance: float | None = None | |
| class VectorStore: | |
| """Abstração sobre uma coleção persistente do ChromaDB.""" | |
| def __init__( | |
| self, | |
| persist_directory: str | Path = "data/chroma_db", | |
| collection_name: str = "documind_documents", | |
| ) -> None: | |
| self.persist_directory = Path(persist_directory) | |
| self.persist_directory.mkdir(parents=True, exist_ok=True) | |
| self._client = chromadb.PersistentClient(path=str(self.persist_directory)) | |
| self.collection_name = collection_name | |
| self._collection = self._client.get_or_create_collection( | |
| name=self.collection_name, | |
| metadata={"hnsw:space": "cosine"}, | |
| ) | |
| def add_chunks(self, chunks: list[TextChunk], embeddings: list[list[float]]) -> int: | |
| """Armazena chunks e seus embeddings no ChromaDB, retornando a quantidade inserida.""" | |
| if not chunks: | |
| return 0 | |
| if len(chunks) != len(embeddings): | |
| raise ValueError("A quantidade de chunks deve ser igual à de embeddings.") | |
| ids = [ | |
| f"{chunk.source}::p{chunk.page}::c{chunk.chunk_index}" | |
| for chunk in chunks | |
| ] | |
| documents = [chunk.text for chunk in chunks] | |
| metadatas: list[dict[str, Any]] = [ | |
| {"source": chunk.source, "page": chunk.page, "chunk_index": chunk.chunk_index} | |
| for chunk in chunks | |
| ] | |
| # Upsert evita falha caso o mesmo documento seja reenviado | |
| self._collection.upsert( | |
| ids=ids, | |
| documents=documents, | |
| embeddings=embeddings, | |
| metadatas=metadatas, | |
| ) | |
| return len(chunks) | |
| def query(self, query_embedding: list[float], top_k: int = 4) -> list[StoredDocument]: | |
| """Busca os top_k trechos mais similares ao embedding da pergunta.""" | |
| total = self._collection.count() | |
| if total == 0: | |
| return [] | |
| n_results = min(max(top_k, 1), total) | |
| results = self._collection.query( | |
| query_embeddings=[query_embedding], | |
| n_results=n_results, | |
| include=["documents", "metadatas", "distances"], | |
| ) | |
| docs = results.get("documents", [[]])[0] | |
| metas = results.get("metadatas", [[]])[0] | |
| distances = results.get("distances", [[]])[0] | |
| retrieved: list[StoredDocument] = [] | |
| for text, meta, dist in zip(docs, metas, distances): | |
| retrieved.append( | |
| StoredDocument( | |
| text=text, | |
| source=str(meta.get("source", "Desconhecido")), | |
| page=int(meta.get("page", 0)), | |
| distance=float(dist) if dist is not None else None, | |
| ) | |
| ) | |
| return retrieved | |
| def count(self) -> int: | |
| """Retorna o total de chunks armazenados na coleção.""" | |
| return self._collection.count() | |
| def list_sources(self) -> list[str]: | |
| """Lista os nomes únicos de documentos atualmente indexados.""" | |
| if self._collection.count() == 0: | |
| return [] | |
| data = self._collection.get(include=["metadatas"]) | |
| metadatas = data.get("metadatas") or [] | |
| sources = {str(meta["source"]) for meta in metadatas if meta and "source" in meta} | |
| return sorted(sources) | |
| def delete_by_source(self, source: str) -> None: | |
| """Remove todos os chunks associados a um arquivo específico.""" | |
| if self._collection.count() == 0: | |
| return | |
| self._collection.delete(where={"source": source}) | |
| def clear(self) -> None: | |
| """Remove todos os documentos da coleção recriando-a vazia.""" | |
| self._client.delete_collection(name=self.collection_name) | |
| self._collection = self._client.get_or_create_collection( | |
| name=self.collection_name, | |
| metadata={"hnsw:space": "cosine"}, | |
| ) | |
| def destroy_storage(self) -> None: | |
| """Limpa a coleção e remove o diretório local quando isolado por sessão.""" | |
| try: | |
| self._client.delete_collection(name=self.collection_name) | |
| except Exception: | |
| pass | |
| self._collection = None # type: ignore[assignment] | |
| self._client = None # type: ignore[assignment] | |
| gc.collect() | |
| shutil.rmtree(self.persist_directory, ignore_errors=True) | |