File size: 1,473 Bytes
6eed659 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 | import json, random, collections, os
random.seed(1234)
MANI="/scratch/prep/manifests"
OUT="/root/devhard"; os.makedirs(OUT, exist_ok=True)
def load(p): return [json.loads(l) for l in open(p,encoding="utf-8")]
dev_all=[]; trainmin={}
for L in ["lin","lug","sna"]:
rows=load(f"{MANI}/waxal_{L}_train.jsonl")
by=collections.defaultdict(list)
for r in rows: by[r.get("speaker")].append(r)
# candidate holdout speakers: middle band (avoid gutting training), 8..70 utts
cand=[s for s,v in by.items() if 8<=len(v)<=70]
random.shuffle(cand)
target=min(400, int(0.05*len(rows)))
hold=set(); n=0
for s in cand:
if n>=target: break
hold.add(s); n+=len(by[s])
dev=[r for r in rows if r.get("speaker") in hold]
tmin=[r for r in rows if r.get("speaker") not in hold]
for r in dev: r["lang"]=L
dev_all+=dev; trainmin[L]=tmin
json.dump if False else None
with open(f"{OUT}/devhard_{L}.jsonl","w",encoding="utf-8") as f:
for r in dev: f.write(json.dumps(r,ensure_ascii=False)+"\n")
with open(f"{OUT}/train_{L}_min.jsonl","w",encoding="utf-8") as f:
for r in tmin: f.write(json.dumps(r,ensure_ascii=False)+"\n")
print(f"{L}: holdout_spk={len(hold)} devhard_utts={len(dev)} train_min_utts={len(tmin)} (was {len(rows)})")
with open(f"{OUT}/devhard_all.jsonl","w",encoding="utf-8") as f:
for r in dev_all: f.write(json.dumps(r,ensure_ascii=False)+"\n")
print("TOTAL devhard:",len(dev_all))
|