explore_store.py

"""
Explore ChromaDB store
======================
Lecture seule (aucun appel Azure). Donne un état complet de la base :
- total de chunks, nombre de mémoires distincts
- répartition par catégorie
- chunks par mémoire (min / max / moyenne)
- DOUBLONS : (source, chunk_index) répétés = fichier ingéré plusieurs fois
- DOUBLONS de texte exact (chunks identiques)
- mémoires manquants vs train_data/

Usage : ./.venv/Scripts/python.exe explore_store.py
"""

import os
import hashlib
from pathlib import Path
from collections import Counter, defaultdict

os.environ.setdefault("ANONYMIZED_TELEMETRY", "False")
import chromadb
from chromadb.config import Settings

PROJECT_ROOT = Path(__file__).parent
DATA_DIR = Path("/data") if Path("/data").is_dir() else PROJECT_ROOT / "data"
TRAIN_DATA_DIR = Path("/train_data") if Path("/train_data").is_dir() else PROJECT_ROOT / "train_data"
COLLECTION_NAME = "rag_documents"

client = chromadb.PersistentClient(path=str(DATA_DIR / "chroma_db"), settings=Settings(anonymized_telemetry=False))
col = client.get_or_create_collection(name=COLLECTION_NAME, metadata={"hnsw:space": "cosine"})

total = col.count()
print("=" * 70)
print(f"TOTAL CHUNKS EN BASE : {total}")
print("=" * 70)

per_source = Counter()                 # chunks par mémoire
src_idx = Counter()                    # (source, chunk_index) -> nb d'occurrences
text_hashes = Counter()                # hash texte -> nb d'occurrences
id_counter = Counter()                 # ids -> détecte doublons d'id
empty_texts = 0
short_texts = 0
hash_to_example = {}

off, step = 0, 2000
while off < total:
    got = col.get(include=["metadatas", "documents"], limit=step, offset=off)
    ids = got["ids"]
    metas = got["metadatas"]
    docs = got["documents"]
    for _id, m, d in zip(ids, metas, docs):
        id_counter[_id] += 1
        src = m.get("source", "??")
        idx = m.get("chunk_index", -1)
        per_source[src] += 1
        src_idx[(src, idx)] += 1
        d = d or ""
        if not d.strip():
            empty_texts += 1
        elif len(d.strip()) < 30:
            short_texts += 1
        h = hashlib.md5(d.encode("utf-8", errors="ignore")).hexdigest()
        text_hashes[h] += 1
        if h not in hash_to_example:
            hash_to_example[h] = (src, d[:80].replace("\n", " "))
    off += step
    print(f"  ...lu {min(off, total)}/{total}")

print()
print(f"Mémoires distincts (sources)      : {len(per_source)}")
print(f"IDs en double                     : {sum(1 for v in id_counter.values() if v > 1)}")
print()

# --- DOUBLONS (source, chunk_index) -> fichier ingéré plusieurs fois ---
dup_pairs = {k: v for k, v in src_idx.items() if v > 1}
dup_sources = sorted({src for (src, _idx) in dup_pairs})
print("-" * 70)
print(f"DOUBLONS (source, chunk_index) répétés : {len(dup_pairs)} paires")
print(f"  -> mémoires concernés (ingérés >1 fois) : {len(dup_sources)}")
for s in dup_sources[:30]:
    occ = max(v for (src, _i), v in src_idx.items() if src == s)
    print(f"     x{occ}  {s}  ({per_source[s]} chunks au total)")

print()
# --- DOUBLONS de texte exact ---
dup_text = {h: v for h, v in text_hashes.items() if v > 1}
nb_extra = sum(v - 1 for v in dup_text.values())
print("-" * 70)
print(f"Textes EXACTEMENT identiques : {len(dup_text)} contenus dupliqués "
      f"= {nb_extra} chunks redondants")
for h, v in sorted(dup_text.items(), key=lambda x: -x[1])[:10]:
    src, ex = hash_to_example[h]
    print(f"     x{v}  [{src}]  «{ex}…»")

print()
print(f"Chunks vides   : {empty_texts}")
print(f"Chunks < 30 car: {short_texts}")

print()
# --- Répartition par catégorie ---
cat = Counter()
cat_docs = defaultdict(set)
for src, n in per_source.items():
    c = src.split("\\")[0].split("/")[0]
    cat[c] += n
    cat_docs[c].add(src)
print("-" * 70)
print("RÉPARTITION PAR CATÉGORIE :")
for c, n in cat.most_common():
    print(f"   {c:<22} {len(cat_docs[c]):>3} mémoires | {n:>7} chunks")

print()
# --- chunks par mémoire ---
counts = sorted(per_source.values())
if counts:
    print("-" * 70)
    print(f"Chunks par mémoire : min={counts[0]} | max={counts[-1]} | "
          f"moyenne={sum(counts)//len(counts)}")

# --- mémoires manquants vs disque ---
if TRAIN_DATA_DIR.exists():
    allfiles = set(str(p.relative_to(TRAIN_DATA_DIR)) for p in TRAIN_DATA_DIR.rglob("*.pdf"))
    missing = sorted(allfiles - set(per_source.keys()))
    print()
    print("-" * 70)
    print(f"PDF sur disque : {len(allfiles)} | en base : {len(per_source)} | MANQUANTS : {len(missing)}")
    for m in missing[:50]:
        print(f"     - {m}")