| |
| """URGENT — `KasuleTrevor/lg_100hrs` est DANS NOTRE CHAMPION `joint_cont` et |
| declare dans COMPLIANCE.md comme derive de Mozilla Common Voice. |
| |
| Or le MEME auteur publie `Lingala_100hrs`, dont on vient de mesurer qu'il |
| contient 5,4 % de transcriptions du SPLIT TEST de WAXAL. Si lg_100hrs est |
| construit de la meme facon, notre propre systeme serait entraine sur du test. |
| |
| Test : comparer les transcriptions de lg_100hrs aux splits test / train / |
| validation LUGANDA de google/WaxalNLP. |
| """ |
| import glob, os, re |
|
|
| os.environ.setdefault("HF_HUB_DISABLE_XET", "1") |
| os.environ.setdefault("HF_HOME", "/scratch/hf_home") |
| import pyarrow.parquet as pq |
| from huggingface_hub import HfApi, hf_hub_download, snapshot_download |
|
|
| tok = open(os.path.expanduser("~/.cache/huggingface/token")).read().strip() |
| api = HfApi(token=tok) |
|
|
|
|
| def norm(s): |
| return " ".join(re.sub(r"[^\w ]", " ", str(s).lower()).split()) |
|
|
|
|
| def texts(paths, cands=("text", "sentence", "transcription", "transcript")): |
| out = [] |
| for p in paths: |
| names = pq.ParquetFile(p).schema_arrow.names |
| c = next((x for x in cands if x in names), None) |
| if not c: |
| continue |
| out += [norm(v) for v in pq.read_table(p, columns=[c]).column(c).to_pylist() if v] |
| return out |
|
|
|
|
| print("=== splits LUGANDA de WAXAL ===", flush=True) |
| snapshot_download("google/WaxalNLP", repo_type="dataset", |
| allow_patterns=["data/ASR/lug/lug-test-*.parquet", |
| "data/ASR/lug/lug-validation-*.parquet", |
| "data/ASR/lug/lug-train-00000*.parquet"], |
| local_dir="/scratch/waxlug", token=tok, max_workers=8) |
| ref = {} |
| for split in ("test", "validation", "train"): |
| fs = sorted(glob.glob("/scratch/waxlug/data/ASR/lug/lug-%s-*.parquet" % split)) |
| ref[split] = set(texts(fs)) |
| print(" %-11s %d parquets | %d phrases uniques" % (split, len(fs), len(ref[split])), flush=True) |
|
|
| print("\n=== echantillon de lg_100hrs ===", flush=True) |
| fs = [s.rfilename for s in api.dataset_info("KasuleTrevor/lg_100hrs").siblings |
| if s.rfilename.endswith(".parquet")] |
| print(" %d parquets au total" % len(fs), flush=True) |
| sel = fs[:8] |
| paths = [hf_hub_download("KasuleTrevor/lg_100hrs", f, repo_type="dataset", |
| token=tok, local_dir="/scratch/dllg") for f in sel] |
| lg = texts(paths) |
| print(" %d parquets lus | %d lignes" % (len(paths), len(lg)), flush=True) |
| names = pq.ParquetFile(paths[0]).schema_arrow.names |
| print(" colonnes : %s" % names, flush=True) |
| for c in ("audio", "path", "id", "audio_id"): |
| if c in names: |
| try: |
| v = pq.read_table(paths[0], columns=[c]).column(c).to_pylist()[:2] |
| print(" %s -> %s" % (c, str(v)[:160]), flush=True) |
| except Exception: |
| pass |
|
|
| print("\n=== VERDICT ===", flush=True) |
| for split in ("test", "validation", "train"): |
| n = sum(1 for t in lg if t in ref[split]) |
| flag = "" |
| if split == "test": |
| flag = " <<< CONTIENT DU TEST -- PROBLEME" if n else " <<< aucune trace de test : OK" |
| print(" recouvrement avec WAXAL lug %-11s : %d/%d (%.1f %%)%s" |
| % (split, n, len(lg), 100 * n / max(len(lg), 1), flag), flush=True) |
| for t in lg[:3]: |
| print(" ex : %s" % t[:110], flush=True) |
| print("LG100_CHECK_DONE", flush=True) |
|
|
