File size: 3,329 Bytes
6eed659 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 | #!/usr/bin/env python3
"""URGENT — `KasuleTrevor/lg_100hrs` est DANS NOTRE CHAMPION `joint_cont` et
declare dans COMPLIANCE.md comme derive de Mozilla Common Voice.
Or le MEME auteur publie `Lingala_100hrs`, dont on vient de mesurer qu'il
contient 5,4 % de transcriptions du SPLIT TEST de WAXAL. Si lg_100hrs est
construit de la meme facon, notre propre systeme serait entraine sur du test.
Test : comparer les transcriptions de lg_100hrs aux splits test / train /
validation LUGANDA de google/WaxalNLP.
"""
import glob, os, re
os.environ.setdefault("HF_HUB_DISABLE_XET", "1")
os.environ.setdefault("HF_HOME", "/scratch/hf_home")
import pyarrow.parquet as pq
from huggingface_hub import HfApi, hf_hub_download, snapshot_download
tok = open(os.path.expanduser("~/.cache/huggingface/token")).read().strip()
api = HfApi(token=tok)
def norm(s):
return " ".join(re.sub(r"[^\w ]", " ", str(s).lower()).split())
def texts(paths, cands=("text", "sentence", "transcription", "transcript")):
out = []
for p in paths:
names = pq.ParquetFile(p).schema_arrow.names
c = next((x for x in cands if x in names), None)
if not c:
continue
out += [norm(v) for v in pq.read_table(p, columns=[c]).column(c).to_pylist() if v]
return out
print("=== splits LUGANDA de WAXAL ===", flush=True)
snapshot_download("google/WaxalNLP", repo_type="dataset",
allow_patterns=["data/ASR/lug/lug-test-*.parquet",
"data/ASR/lug/lug-validation-*.parquet",
"data/ASR/lug/lug-train-00000*.parquet"],
local_dir="/scratch/waxlug", token=tok, max_workers=8)
ref = {}
for split in ("test", "validation", "train"):
fs = sorted(glob.glob("/scratch/waxlug/data/ASR/lug/lug-%s-*.parquet" % split))
ref[split] = set(texts(fs))
print(" %-11s %d parquets | %d phrases uniques" % (split, len(fs), len(ref[split])), flush=True)
print("\n=== echantillon de lg_100hrs ===", flush=True)
fs = [s.rfilename for s in api.dataset_info("KasuleTrevor/lg_100hrs").siblings
if s.rfilename.endswith(".parquet")]
print(" %d parquets au total" % len(fs), flush=True)
sel = fs[:8]
paths = [hf_hub_download("KasuleTrevor/lg_100hrs", f, repo_type="dataset",
token=tok, local_dir="/scratch/dllg") for f in sel]
lg = texts(paths)
print(" %d parquets lus | %d lignes" % (len(paths), len(lg)), flush=True)
names = pq.ParquetFile(paths[0]).schema_arrow.names
print(" colonnes : %s" % names, flush=True)
for c in ("audio", "path", "id", "audio_id"):
if c in names:
try:
v = pq.read_table(paths[0], columns=[c]).column(c).to_pylist()[:2]
print(" %s -> %s" % (c, str(v)[:160]), flush=True)
except Exception:
pass
print("\n=== VERDICT ===", flush=True)
for split in ("test", "validation", "train"):
n = sum(1 for t in lg if t in ref[split])
flag = ""
if split == "test":
flag = " <<< CONTIENT DU TEST -- PROBLEME" if n else " <<< aucune trace de test : OK"
print(" recouvrement avec WAXAL lug %-11s : %d/%d (%.1f %%)%s"
% (split, n, len(lg), 100 * n / max(len(lg), 1), flag), flush=True)
for t in lg[:3]:
print(" ex : %s" % t[:110], flush=True)
print("LG100_CHECK_DONE", flush=True)
|