File size: 1,550 Bytes
e4c97d2 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 | """Merge synth parts, split held-out synthetic test (every 25th text id), and (after eval_asr.py --hyp_only) filter TTS failures.
python3 build_synth.py merge -> /scratch/asr/synth/all.jsonl
python3 build_synth.py filter -> /scratch/asr/synth/{manifest.jsonl (train), test.jsonl} (keep clips the stock v2 model reads at CER <= 0.35)"""
import json, glob, sys, collections
D = "/scratch/asr/synth"
if sys.argv[1] == "merge":
R = [json.loads(l) for f in sorted(glob.glob(f"{D}/parts/part_*.jsonl")) for l in open(f)]
R = [r for r in R if 1.0 <= r["dur"] <= 20 and r["text"]]
with open(f"{D}/all.jsonl", "w") as f:
for r in R: f.write(json.dumps(r, ensure_ascii=False) + "\n")
print("merged", len(R), "hours", round(sum(r["dur"] for r in R) / 3600, 2))
else:
R = [json.loads(l) for l in open(f"{D}/all.jsonl")]; H = {json.loads(l)["id"]: json.loads(l) for l in open(f"{D}/all.jsonl.hyp.jsonl")}
tr, te, drop = [], [], collections.Counter()
for r in R:
c = H[r["id"]]["cer"]; r["base_cer"] = c
if c > 0.35: drop[r["pair"]] += 1; continue
(te if int(r["id"][2:]) % 25 == 0 else tr).append(r)
for name, X in (("manifest", tr), ("test", te)):
with open(f"{D}/{name}.jsonl", "w") as f:
for r in X: f.write(json.dumps(r, ensure_ascii=False) + "\n")
print("train", len(tr), round(sum(r["dur"] for r in tr) / 3600, 2), "h; test", len(te), "; dropped (base CER>0.35)", sum(drop.values()), dict(drop))
print("pairs train", collections.Counter(r["pair"] for r in tr))
|