code/compare_csvs.py

#!/usr/bin/env python3
"""Compare tous les candidats CSV (depuis la découverte multilingue) au meilleur confirmé.
Sortie : stats par fichier, matrice de similarité (lignes identiques), distance au best 0.6739,
drapeaux d'hallucination, détection de quasi-doublons. Aide à choisir quoi soumettre."""
import csv, os, glob, statistics
from collections import Counter
from difflib import SequenceMatcher

REF = "/root/sub_autodetect_beam.csv"   # meilleur confirmé = 0.6739
LABEL = {
    "sub_s51_autodetect.csv": "51 autodetect greedy (0.6691 connu)",
    "sub_autodetect_beam.csv": "51 autodetect beam5 (0.6739 CONFIRME=REF)",
    "sub_autodetect_beam_whole.csv": "51 autodetect beam5 + WHOLE-CLIP (nouveau)",
    "sub_v3nyn_beam.csv": "51 + Runyankole->v3 beam5",
    "sub_v3nyn_beam_norm.csv": "v3nyn + normalisation NFC",
    "sub_mmslid.csv": "LID mms-lid-4017 + 51",
    "sub_v3salt_beam.csv": "v3-salt SPECIALISTE beam5",
    "sub_v3salt_beam_whole.csv": "v3-salt SPECIALISTE + WHOLE-CLIP (final)",
}


def load(p):
    return {r["ID"]: (r.get("Target") or "").strip() for r in csv.DictReader(open(p, encoding="utf-8"))}


def halluc(t):
    w = t.split()
    if len(w) >= 6 and Counter(w).most_common(1)[0][1] / len(w) > 0.5:
        return True
    return False


files = [f for f in LABEL if os.path.exists("/root/" + f)]
ref = load(REF)
ids = list(ref.keys())
data = {f: load("/root/" + f) for f in files}

print("=" * 90)
print(f"{'FICHIER':<34}{'lignes':>7}{'vides':>6}{'mots/clip':>10}{'halluc':>7}{'=REF%':>7}{'sim.REF':>8}")
print("-" * 90)
for f in files:
    d = data[f]
    wl = [len(v.split()) for v in d.values()]
    nl = len(d); ne = sum(1 for v in d.values() if not v); nh = sum(1 for v in d.values() if halluc(v))
    same = sum(1 for i in ids if d.get(i, "") == ref.get(i, ""))
    diff_ids = [i for i in ids if d.get(i, "") != ref.get(i, "")]
    sim = statistics.mean(SequenceMatcher(None, ref.get(i, ""), d.get(i, "")).ratio() for i in diff_ids) if diff_ids else 1.0
    tag = " <<REF" if f == os.path.basename(REF) else ""
    print(f"{f:<34}{nl:>7}{ne:>6}{statistics.mean(wl):>10.1f}{nh:>7}{100*same//nl:>7}{sim:>8.2f}{tag}")
    print(f"    -> {LABEL[f]}")

print("\n" + "=" * 90)
print("MATRICE % lignes identiques (proximité entre candidats — >99% = quasi-doublon)")
print("-" * 90)
short = {f: f.replace("sub_", "").replace(".csv", "")[:14] for f in files}
hdr = "".join(f"{short[f]:>16}" for f in files)
print(f"{'':<16}{hdr}")
for a in files:
    row = ""
    for b in files:
        s = sum(1 for i in ids if data[a].get(i, "") == data[b].get(i, ""))
        row += f"{100*s//len(ids):>16}"
    print(f"{short[a]:<16}{row}")