File size: 4,761 Bytes
68e8c03
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
"""Gerenciamento do banco vetorial ChromaDB com persistência local."""

import gc
import shutil
from dataclasses import dataclass
from pathlib import Path
from typing import Any

import chromadb

from src.text_splitter import TextChunk


@dataclass(frozen=True)
class StoredDocument:
    """Representa um trecho recuperado do ChromaDB com metadados e distância."""

    text: str
    source: str
    page: int
    distance: float | None = None


class VectorStore:
    """Abstração sobre uma coleção persistente do ChromaDB."""

    def __init__(
        self,
        persist_directory: str | Path = "data/chroma_db",
        collection_name: str = "documind_documents",
    ) -> None:
        self.persist_directory = Path(persist_directory)
        self.persist_directory.mkdir(parents=True, exist_ok=True)
        self._client = chromadb.PersistentClient(path=str(self.persist_directory))
        self.collection_name = collection_name
        self._collection = self._client.get_or_create_collection(
            name=self.collection_name,
            metadata={"hnsw:space": "cosine"},
        )

    def add_chunks(self, chunks: list[TextChunk], embeddings: list[list[float]]) -> int:
        """Armazena chunks e seus embeddings no ChromaDB, retornando a quantidade inserida."""
        if not chunks:
            return 0
        if len(chunks) != len(embeddings):
            raise ValueError("A quantidade de chunks deve ser igual à de embeddings.")

        ids = [
            f"{chunk.source}::p{chunk.page}::c{chunk.chunk_index}"
            for chunk in chunks
        ]
        documents = [chunk.text for chunk in chunks]
        metadatas: list[dict[str, Any]] = [
            {"source": chunk.source, "page": chunk.page, "chunk_index": chunk.chunk_index}
            for chunk in chunks
        ]

        # Upsert evita falha caso o mesmo documento seja reenviado
        self._collection.upsert(
            ids=ids,
            documents=documents,
            embeddings=embeddings,
            metadatas=metadatas,
        )
        return len(chunks)

    def query(self, query_embedding: list[float], top_k: int = 4) -> list[StoredDocument]:
        """Busca os top_k trechos mais similares ao embedding da pergunta."""
        total = self._collection.count()
        if total == 0:
            return []

        n_results = min(max(top_k, 1), total)
        results = self._collection.query(
            query_embeddings=[query_embedding],
            n_results=n_results,
            include=["documents", "metadatas", "distances"],
        )

        docs = results.get("documents", [[]])[0]
        metas = results.get("metadatas", [[]])[0]
        distances = results.get("distances", [[]])[0]

        retrieved: list[StoredDocument] = []
        for text, meta, dist in zip(docs, metas, distances):
            retrieved.append(
                StoredDocument(
                    text=text,
                    source=str(meta.get("source", "Desconhecido")),
                    page=int(meta.get("page", 0)),
                    distance=float(dist) if dist is not None else None,
                )
            )
        return retrieved

    def count(self) -> int:
        """Retorna o total de chunks armazenados na coleção."""
        return self._collection.count()

    def list_sources(self) -> list[str]:
        """Lista os nomes únicos de documentos atualmente indexados."""
        if self._collection.count() == 0:
            return []
        data = self._collection.get(include=["metadatas"])
        metadatas = data.get("metadatas") or []
        sources = {str(meta["source"]) for meta in metadatas if meta and "source" in meta}
        return sorted(sources)

    def delete_by_source(self, source: str) -> None:
        """Remove todos os chunks associados a um arquivo específico."""
        if self._collection.count() == 0:
            return
        self._collection.delete(where={"source": source})

    def clear(self) -> None:
        """Remove todos os documentos da coleção recriando-a vazia."""
        self._client.delete_collection(name=self.collection_name)
        self._collection = self._client.get_or_create_collection(
            name=self.collection_name,
            metadata={"hnsw:space": "cosine"},
        )

    def destroy_storage(self) -> None:
        """Limpa a coleção e remove o diretório local quando isolado por sessão."""
        try:
            self._client.delete_collection(name=self.collection_name)
        except Exception:
            pass
        self._collection = None  # type: ignore[assignment]
        self._client = None  # type: ignore[assignment]
        gc.collect()
        shutil.rmtree(self.persist_directory, ignore_errors=True)