# SPDX-License-Identifier: Apache-2.0 # Copyright 2026 Kale """Final per-voice training list (the release path's last data step): filter.py's kept takes (keep.py rules) minus every multi-sentence take that strict.py flags (missing / not alignable / final or any inner sentence end >= 100 ms rough). 1 in 100 takes (md5 of the clip id) -> split "val". Writes, under /work/ (root = env KALE_TTS_ROOT or "."): final-.jsonl kept rows that passed (clip, voice, text, seconds, sim, wpm, f0, path on the data volume) rows-.jsonl the train step's prep input: clip, split, text, path (+ ids if work/ids__all.json exists) ids_.json {"rows": {clip: token ids}} for the final takes (only if the ids file exists) and /results/final-summary-.json (kept, final, strict drops, hours, passages, median wpm). python3 -I final.py [--kept work/kept-.jsonl] [--strict work/strict-.jsonl] (default group = the voice) Trimmed from the original: it also spawned the train app's feature prep on these rows; that call now lives in the train step.""" import argparse import hashlib import json import os import re from pathlib import Path ap = argparse.ArgumentParser() ap.add_argument("voice"); ap.add_argument("--kept", default=""); ap.add_argument("--strict", default="") A = ap.parse_args() V = Path(os.environ.get("KALE_TTS_ROOT", ".")) v = A.voice idp = V / f"work/ids_{v}_all.json" K = json.loads(idp.read_text())["rows"] if idp.exists() else None kept = [json.loads(l) for l in Path(A.kept or V / f"work/kept-{v}.jsonl").read_text().splitlines() if l.strip()] kept = [k for k in kept if k["clip"].split("_")[0] == v] # a group render (several voices in one call): this voice's takes only st = {json.loads(l)["clip"]: json.loads(l) for l in Path(A.strict or V / f"work/strict-{v}.jsonl").read_text().splitlines() if l.strip()} multi = lambda t: len([s for s in re.split(r"(?<=[.!?])\s+", t.strip()) if s]) > 1 # noqa: E731 final, why = [], {} for k in kept: s = st.get(k["clip"]) if multi(k["text"]): bad = ("strict check missing" if s is None or "why" in s else "not alignable" if not s.get("aligned") else "rough final end (strict)" if s.get("end_ms", 0) >= 100 else "rough inner sentence end" if any(x >= 100 for x in s.get("inner_ms", [])) else "") if bad: why[bad] = why.get(bad, 0) + 1; continue final.append(k) split = lambda c: "val" if int(hashlib.md5(c.encode()).hexdigest(), 16) % 100 == 0 else "train" # noqa: E731 rows = [dict(clip=k["clip"], split=split(k["clip"]), text=k["text"], path=k["path"]) | ({"ids": K[k["clip"]]} if K else {}) for k in final] (V / "work").mkdir(parents=True, exist_ok=True) if K: (V / f"work/ids_{v}.json").write_text(json.dumps(dict(rows={r["clip"]: r["ids"] for r in rows}))) (V / f"work/final-{v}.jsonl").write_text("".join(json.dumps(k) + "\n" for k in final)) (V / f"work/rows-{v}.jsonl").write_text("".join(json.dumps(r) + "\n" for r in rows)) summ = dict(kept_filter=len(kept), final=len(rows), strict_drops=why, hours=round(sum(k["seconds"] for k in final) / 3600, 2), val=sum(r["split"] == "val" for r in rows), passages=sum(multi(k["text"]) for k in final), wpm_median=sorted(k["wpm"] for k in final)[len(final) // 2] if final else None) (V / "results").mkdir(parents=True, exist_ok=True) (V / f"results/final-summary-{v}.json").write_text(json.dumps(summ, indent=1)) print(json.dumps(summ))