NERDDISCO's picture
kale-tts-1
f21d6eb
Raw History Blame Contribute Delete
3.5 kB
# SPDX-License-Identifier: Apache-2.0
# Copyright 2026 Kale
"""Final per-voice training list (the release path's last data step): filter.py's kept takes (keep.py rules) minus every multi-sentence
take that strict.py flags (missing / not alignable / final or any inner sentence end >= 100 ms rough). 1 in 100 takes (md5 of the clip
id) -> split "val". Writes, under <root>/work/ (root = env KALE_TTS_ROOT or "."):
final-<voice>.jsonl kept rows that passed (clip, voice, text, seconds, sim, wpm, f0, path on the data volume)
rows-<voice>.jsonl the train step's prep input: clip, split, text, path (+ ids if work/ids_<voice>_all.json exists)
ids_<voice>.json {"rows": {clip: token ids}} for the final takes (only if the ids file exists)
and <root>/results/final-summary-<voice>.json (kept, final, strict drops, hours, passages, median wpm).
python3 -I final.py <voice> [--kept work/kept-<group>.jsonl] [--strict work/strict-<group>.jsonl] (default group = the voice)
Trimmed from the original: it also spawned the train app's feature prep on these rows; that call now lives in the train step."""
import argparse
import hashlib
import json
import os
import re
from pathlib import Path
ap = argparse.ArgumentParser()
ap.add_argument("voice"); ap.add_argument("--kept", default=""); ap.add_argument("--strict", default="")
A = ap.parse_args()
V = Path(os.environ.get("KALE_TTS_ROOT", "."))
v = A.voice
idp = V / f"work/ids_{v}_all.json"
K = json.loads(idp.read_text())["rows"] if idp.exists() else None
kept = [json.loads(l) for l in Path(A.kept or V / f"work/kept-{v}.jsonl").read_text().splitlines() if l.strip()]
kept = [k for k in kept if k["clip"].split("_")[0] == v] # a group render (several voices in one call): this voice's takes only
st = {json.loads(l)["clip"]: json.loads(l) for l in Path(A.strict or V / f"work/strict-{v}.jsonl").read_text().splitlines() if l.strip()}
multi = lambda t: len([s for s in re.split(r"(?<=[.!?])\s+", t.strip()) if s]) > 1 # noqa: E731
final, why = [], {}
for k in kept:
s = st.get(k["clip"])
if multi(k["text"]):
bad = ("strict check missing" if s is None or "why" in s else "not alignable" if not s.get("aligned") else
"rough final end (strict)" if s.get("end_ms", 0) >= 100 else "rough inner sentence end" if any(x >= 100 for x in s.get("inner_ms", [])) else "")
if bad:
why[bad] = why.get(bad, 0) + 1; continue
final.append(k)
split = lambda c: "val" if int(hashlib.md5(c.encode()).hexdigest(), 16) % 100 == 0 else "train" # noqa: E731
rows = [dict(clip=k["clip"], split=split(k["clip"]), text=k["text"], path=k["path"]) | ({"ids": K[k["clip"]]} if K else {}) for k in final]
(V / "work").mkdir(parents=True, exist_ok=True)
if K:
(V / f"work/ids_{v}.json").write_text(json.dumps(dict(rows={r["clip"]: r["ids"] for r in rows})))
(V / f"work/final-{v}.jsonl").write_text("".join(json.dumps(k) + "\n" for k in final))
(V / f"work/rows-{v}.jsonl").write_text("".join(json.dumps(r) + "\n" for r in rows))
summ = dict(kept_filter=len(kept), final=len(rows), strict_drops=why, hours=round(sum(k["seconds"] for k in final) / 3600, 2),
val=sum(r["split"] == "val" for r in rows), passages=sum(multi(k["text"]) for k in final),
wpm_median=sorted(k["wpm"] for k in final)[len(final) // 2] if final else None)
(V / "results").mkdir(parents=True, exist_ok=True)
(V / f"results/final-summary-{v}.json").write_text(json.dumps(summ, indent=1))
print(json.dumps(summ))