code/check_leak100.py

#!/usr/bin/env python3
"""`KasuleTrevor/Lingala_100hrs` contient-il des transcriptions du SPLIT TEST de
WAXAL ? Si oui, l'utiliser serait s'entrainer sur du test = disqualification.

On a mesure qu'il recoupe WAXAL a 49 %. La question qui reste est : QUELLE
partie de WAXAL ? train/validation (acceptable, c'est deja nos donnees) ou test
(interdit) ?

Methode : telecharger les transcriptions du split TEST lin de google/WaxalNLP
(colonne texte seulement, pas l'audio) et mesurer le recouvrement avec un
echantillon large de Lingala_100hrs (plusieurs parquets, pas un seul).
"""
import glob, json, os, re

os.environ.setdefault("HF_HUB_DISABLE_XET", "1")
os.environ.setdefault("HF_HOME", "/scratch/hf_home")
import pyarrow.parquet as pq
from huggingface_hub import HfApi, hf_hub_download, snapshot_download

tok = open(os.path.expanduser("~/.cache/huggingface/token")).read().strip()
api = HfApi(token=tok)


def norm(s):
    return " ".join(re.sub(r"[^\w ]", " ", str(s).lower()).split())


def texts_from_parquets(paths, cands=("text", "sentence", "transcription", "transcript")):
    out = []
    for p in paths:
        names = pq.ParquetFile(p).schema_arrow.names
        c = next((x for x in cands if x in names), None)
        if not c:
            continue
        out += [norm(v) for v in pq.read_table(p, columns=[c]).column(c).to_pylist() if v]
    return out


print("=== 1) transcriptions du split TEST lin de WAXAL ===", flush=True)
snapshot_download("google/WaxalNLP", repo_type="dataset",
                  allow_patterns=["data/ASR/lin/lin-test-*.parquet"],
                  local_dir="/scratch/waxtest", token=tok, max_workers=8)
tf = sorted(glob.glob("/scratch/waxtest/data/ASR/lin/lin-test-*.parquet"))
test_txt = set(texts_from_parquets(tf))
print("  %d parquets | %d phrases uniques dans le TEST lin" % (len(tf), len(test_txt)), flush=True)

print("\n=== 2) transcriptions du TRAIN+VAL lin de WAXAL (reference de comparaison) ===", flush=True)
tv = set()
for p in ("waxal_lin_train", "waxal_lin_validation"):
    f = "/scratch/prep/manifests/%s.jsonl" % p
    if os.path.exists(f):
        for l in open(f, encoding="utf-8"):
            t = norm(json.loads(l).get("text", ""))
            if t:
                tv.add(t)
print("  %d phrases uniques" % len(tv), flush=True)
print("  (controle : test inter train+val = %d, doit etre ~0)" % len(test_txt & tv), flush=True)

print("\n=== 3) echantillon LARGE de Lingala_100hrs ===", flush=True)
fs = [s.rfilename for s in api.dataset_info("KasuleTrevor/Lingala_100hrs").siblings
      if s.rfilename.endswith(".parquet")]
sel = fs[:8]
paths = [hf_hub_download("KasuleTrevor/Lingala_100hrs", f, repo_type="dataset",
                         token=tok, local_dir="/scratch/dl100") for f in sel]
k100 = texts_from_parquets(paths)
print("  %d parquets lus | %d lignes" % (len(paths), len(k100)), flush=True)

inter_test = sum(1 for t in k100 if t in test_txt)
inter_tv = sum(1 for t in k100 if t in tv)
print("\n=== VERDICT ===", flush=True)
print("  recouvrement avec TRAIN+VAL WAXAL : %d/%d (%.1f %%)" % (inter_tv, len(k100), 100 * inter_tv / max(len(k100), 1)), flush=True)
print("  recouvrement avec TEST WAXAL      : %d/%d (%.1f %%)  %s"
      % (inter_test, len(k100), 100 * inter_test / max(len(k100), 1),
         "<<< CONTIENT DU TEST -- INTERDIT" if inter_test > 0 else "<<< aucune trace de test"), flush=True)
if inter_test:
    ex = [t for t in k100 if t in test_txt][:3]
    for e in ex:
        print("     ex de fuite : %s" % e[:110], flush=True)
print("LEAK_CHECK_DONE", flush=True)