#!/usr/bin/env python3
"""URGENT — `KasuleTrevor/lg_100hrs` est DANS NOTRE CHAMPION `joint_cont` et
declare dans COMPLIANCE.md comme derive de Mozilla Common Voice.

Or le MEME auteur publie `Lingala_100hrs`, dont on vient de mesurer qu'il
contient 5,4 % de transcriptions du SPLIT TEST de WAXAL. Si lg_100hrs est
construit de la meme facon, notre propre systeme serait entraine sur du test.

Test : comparer les transcriptions de lg_100hrs aux splits test / train /
validation LUGANDA de google/WaxalNLP.
"""
import glob, os, re

os.environ.setdefault("HF_HUB_DISABLE_XET", "1")
os.environ.setdefault("HF_HOME", "/scratch/hf_home")
import pyarrow.parquet as pq
from huggingface_hub import HfApi, hf_hub_download, snapshot_download

tok = open(os.path.expanduser("~/.cache/huggingface/token")).read().strip()
api = HfApi(token=tok)


def norm(s):
    return " ".join(re.sub(r"[^\w ]", " ", str(s).lower()).split())


def texts(paths, cands=("text", "sentence", "transcription", "transcript")):
    out = []
    for p in paths:
        names = pq.ParquetFile(p).schema_arrow.names
        c = next((x for x in cands if x in names), None)
        if not c:
            continue
        out += [norm(v) for v in pq.read_table(p, columns=[c]).column(c).to_pylist() if v]
    return out


print("=== splits LUGANDA de WAXAL ===", flush=True)
snapshot_download("google/WaxalNLP", repo_type="dataset",
                  allow_patterns=["data/ASR/lug/lug-test-*.parquet",
                                  "data/ASR/lug/lug-validation-*.parquet",
                                  "data/ASR/lug/lug-train-00000*.parquet"],
                  local_dir="/scratch/waxlug", token=tok, max_workers=8)
ref = {}
for split in ("test", "validation", "train"):
    fs = sorted(glob.glob("/scratch/waxlug/data/ASR/lug/lug-%s-*.parquet" % split))
    ref[split] = set(texts(fs))
    print("  %-11s %d parquets | %d phrases uniques" % (split, len(fs), len(ref[split])), flush=True)

print("\n=== echantillon de lg_100hrs ===", flush=True)
fs = [s.rfilename for s in api.dataset_info("KasuleTrevor/lg_100hrs").siblings
      if s.rfilename.endswith(".parquet")]
print("  %d parquets au total" % len(fs), flush=True)
sel = fs[:8]
paths = [hf_hub_download("KasuleTrevor/lg_100hrs", f, repo_type="dataset",
                         token=tok, local_dir="/scratch/dllg") for f in sel]
lg = texts(paths)
print("  %d parquets lus | %d lignes" % (len(paths), len(lg)), flush=True)
names = pq.ParquetFile(paths[0]).schema_arrow.names
print("  colonnes : %s" % names, flush=True)
for c in ("audio", "path", "id", "audio_id"):
    if c in names:
        try:
            v = pq.read_table(paths[0], columns=[c]).column(c).to_pylist()[:2]
            print("  %s -> %s" % (c, str(v)[:160]), flush=True)
        except Exception:
            pass

print("\n=== VERDICT ===", flush=True)
for split in ("test", "validation", "train"):
    n = sum(1 for t in lg if t in ref[split])
    flag = ""
    if split == "test":
        flag = "   <<< CONTIENT DU TEST -- PROBLEME" if n else "   <<< aucune trace de test : OK"
    print("  recouvrement avec WAXAL lug %-11s : %d/%d (%.1f %%)%s"
          % (split, n, len(lg), 100 * n / max(len(lg), 1), flag), flush=True)
for t in lg[:3]:
    print("  ex : %s" % t[:110], flush=True)
print("LG100_CHECK_DONE", flush=True)