DocuMindAI_RAG_Application / src /vector_store.py
AdryanAzul's picture
feat: deploy DocuMind AI (RAG + BYOK + local embeddings)
68e8c03 verified
Raw History Blame Contribute Delete
4.76 kB
"""Gerenciamento do banco vetorial ChromaDB com persistência local."""
import gc
import shutil
from dataclasses import dataclass
from pathlib import Path
from typing import Any
import chromadb
from src.text_splitter import TextChunk
@dataclass(frozen=True)
class StoredDocument:
"""Representa um trecho recuperado do ChromaDB com metadados e distância."""
text: str
source: str
page: int
distance: float | None = None
class VectorStore:
"""Abstração sobre uma coleção persistente do ChromaDB."""
def __init__(
self,
persist_directory: str | Path = "data/chroma_db",
collection_name: str = "documind_documents",
) -> None:
self.persist_directory = Path(persist_directory)
self.persist_directory.mkdir(parents=True, exist_ok=True)
self._client = chromadb.PersistentClient(path=str(self.persist_directory))
self.collection_name = collection_name
self._collection = self._client.get_or_create_collection(
name=self.collection_name,
metadata={"hnsw:space": "cosine"},
)
def add_chunks(self, chunks: list[TextChunk], embeddings: list[list[float]]) -> int:
"""Armazena chunks e seus embeddings no ChromaDB, retornando a quantidade inserida."""
if not chunks:
return 0
if len(chunks) != len(embeddings):
raise ValueError("A quantidade de chunks deve ser igual à de embeddings.")
ids = [
f"{chunk.source}::p{chunk.page}::c{chunk.chunk_index}"
for chunk in chunks
]
documents = [chunk.text for chunk in chunks]
metadatas: list[dict[str, Any]] = [
{"source": chunk.source, "page": chunk.page, "chunk_index": chunk.chunk_index}
for chunk in chunks
]
# Upsert evita falha caso o mesmo documento seja reenviado
self._collection.upsert(
ids=ids,
documents=documents,
embeddings=embeddings,
metadatas=metadatas,
)
return len(chunks)
def query(self, query_embedding: list[float], top_k: int = 4) -> list[StoredDocument]:
"""Busca os top_k trechos mais similares ao embedding da pergunta."""
total = self._collection.count()
if total == 0:
return []
n_results = min(max(top_k, 1), total)
results = self._collection.query(
query_embeddings=[query_embedding],
n_results=n_results,
include=["documents", "metadatas", "distances"],
)
docs = results.get("documents", [[]])[0]
metas = results.get("metadatas", [[]])[0]
distances = results.get("distances", [[]])[0]
retrieved: list[StoredDocument] = []
for text, meta, dist in zip(docs, metas, distances):
retrieved.append(
StoredDocument(
text=text,
source=str(meta.get("source", "Desconhecido")),
page=int(meta.get("page", 0)),
distance=float(dist) if dist is not None else None,
)
)
return retrieved
def count(self) -> int:
"""Retorna o total de chunks armazenados na coleção."""
return self._collection.count()
def list_sources(self) -> list[str]:
"""Lista os nomes únicos de documentos atualmente indexados."""
if self._collection.count() == 0:
return []
data = self._collection.get(include=["metadatas"])
metadatas = data.get("metadatas") or []
sources = {str(meta["source"]) for meta in metadatas if meta and "source" in meta}
return sorted(sources)
def delete_by_source(self, source: str) -> None:
"""Remove todos os chunks associados a um arquivo específico."""
if self._collection.count() == 0:
return
self._collection.delete(where={"source": source})
def clear(self) -> None:
"""Remove todos os documentos da coleção recriando-a vazia."""
self._client.delete_collection(name=self.collection_name)
self._collection = self._client.get_or_create_collection(
name=self.collection_name,
metadata={"hnsw:space": "cosine"},
)
def destroy_storage(self) -> None:
"""Limpa a coleção e remove o diretório local quando isolado por sessão."""
try:
self._client.delete_collection(name=self.collection_name)
except Exception:
pass
self._collection = None # type: ignore[assignment]
self._client = None # type: ignore[assignment]
gc.collect()
shutil.rmtree(self.persist_directory, ignore_errors=True)