File size: 3,565 Bytes
6eed659 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 | #!/usr/bin/env python3
"""`KasuleTrevor/Lingala_100hrs` contient-il des transcriptions du SPLIT TEST de
WAXAL ? Si oui, l'utiliser serait s'entrainer sur du test = disqualification.
On a mesure qu'il recoupe WAXAL a 49 %. La question qui reste est : QUELLE
partie de WAXAL ? train/validation (acceptable, c'est deja nos donnees) ou test
(interdit) ?
Methode : telecharger les transcriptions du split TEST lin de google/WaxalNLP
(colonne texte seulement, pas l'audio) et mesurer le recouvrement avec un
echantillon large de Lingala_100hrs (plusieurs parquets, pas un seul).
"""
import glob, json, os, re
os.environ.setdefault("HF_HUB_DISABLE_XET", "1")
os.environ.setdefault("HF_HOME", "/scratch/hf_home")
import pyarrow.parquet as pq
from huggingface_hub import HfApi, hf_hub_download, snapshot_download
tok = open(os.path.expanduser("~/.cache/huggingface/token")).read().strip()
api = HfApi(token=tok)
def norm(s):
return " ".join(re.sub(r"[^\w ]", " ", str(s).lower()).split())
def texts_from_parquets(paths, cands=("text", "sentence", "transcription", "transcript")):
out = []
for p in paths:
names = pq.ParquetFile(p).schema_arrow.names
c = next((x for x in cands if x in names), None)
if not c:
continue
out += [norm(v) for v in pq.read_table(p, columns=[c]).column(c).to_pylist() if v]
return out
print("=== 1) transcriptions du split TEST lin de WAXAL ===", flush=True)
snapshot_download("google/WaxalNLP", repo_type="dataset",
allow_patterns=["data/ASR/lin/lin-test-*.parquet"],
local_dir="/scratch/waxtest", token=tok, max_workers=8)
tf = sorted(glob.glob("/scratch/waxtest/data/ASR/lin/lin-test-*.parquet"))
test_txt = set(texts_from_parquets(tf))
print(" %d parquets | %d phrases uniques dans le TEST lin" % (len(tf), len(test_txt)), flush=True)
print("\n=== 2) transcriptions du TRAIN+VAL lin de WAXAL (reference de comparaison) ===", flush=True)
tv = set()
for p in ("waxal_lin_train", "waxal_lin_validation"):
f = "/scratch/prep/manifests/%s.jsonl" % p
if os.path.exists(f):
for l in open(f, encoding="utf-8"):
t = norm(json.loads(l).get("text", ""))
if t:
tv.add(t)
print(" %d phrases uniques" % len(tv), flush=True)
print(" (controle : test inter train+val = %d, doit etre ~0)" % len(test_txt & tv), flush=True)
print("\n=== 3) echantillon LARGE de Lingala_100hrs ===", flush=True)
fs = [s.rfilename for s in api.dataset_info("KasuleTrevor/Lingala_100hrs").siblings
if s.rfilename.endswith(".parquet")]
sel = fs[:8]
paths = [hf_hub_download("KasuleTrevor/Lingala_100hrs", f, repo_type="dataset",
token=tok, local_dir="/scratch/dl100") for f in sel]
k100 = texts_from_parquets(paths)
print(" %d parquets lus | %d lignes" % (len(paths), len(k100)), flush=True)
inter_test = sum(1 for t in k100 if t in test_txt)
inter_tv = sum(1 for t in k100 if t in tv)
print("\n=== VERDICT ===", flush=True)
print(" recouvrement avec TRAIN+VAL WAXAL : %d/%d (%.1f %%)" % (inter_tv, len(k100), 100 * inter_tv / max(len(k100), 1)), flush=True)
print(" recouvrement avec TEST WAXAL : %d/%d (%.1f %%) %s"
% (inter_test, len(k100), 100 * inter_test / max(len(k100), 1),
"<<< CONTIENT DU TEST -- INTERDIT" if inter_test > 0 else "<<< aucune trace de test"), flush=True)
if inter_test:
ex = [t for t in k100 if t in test_txt][:3]
for e in ex:
print(" ex de fuite : %s" % e[:110], flush=True)
print("LEAK_CHECK_DONE", flush=True)
|