# SPDX-License-Identifier: Apache-2.0 # Copyright 2026 Kale """Per-voice data prep on the deployed training app (app.py), one voice at a time or many in parallel shells: 1. final takes = the filter's kept takes, minus passages that fail the strict every-sentence-end check (unaligned, a rough final end >= 100 ms or any rough inner sentence end >= 100 ms); 1 in 100 takes (clip md5) -> split "val" 2. features: app.prep on chunks of 6,000 rows -> /data/feat-/shard_NNN.pt 3. ids file up (-> /data/feat/ids_.json), teacher-aligner durations (app.durations_fn -> /data/feat/dur_.pt) 4. pace window (config/voices.json "pace_window_rule") -> /data/feat/dur_keep_.json Voice packs come from ../teacher (its style encoder over 200 of the voice's own takes >= 3 s), not from here. usage: python prep.py [--ids-suffix gb] [--kept ../filter/work/kept-.jsonl --strict ../filter/work/strict-.jsonl] [--dry-run] Inputs (local, relative to train/, set in config/paths.json): /kept-.jsonl filter output: {clip, text, path (wav on the data volume), seconds, wpm, ...} per kept take /strict-.jsonl strict check: {clip, aligned, end_ms, inner_ms[]} (or {clip, why}) per passage /ids__all.json ../render/make_ids.py output: {"rows": {clip: [Kokoro token ids]}} (en-GB voices: the British ids) (step 1 is the same rule as ../filter/final.py, which only writes the list + summary; this script repeats it and continues) Outputs (local /): final-.jsonl, ids_.json, dur_keep_.json, pace-window-.json, final-summary-.json.""" import argparse import hashlib import json import os import re import statistics as S from pathlib import Path HERE = Path(__file__).resolve().parent def cfg(name): return json.loads((HERE / "config" / name).read_text()) def modal_names(): for p in (os.environ.get("KALE_TTS_MODAL_CONFIG"), HERE.parent / "config" / "modal.json"): if p and Path(p).is_file(): return {"train_app": "kale-tts-train", "runs_volume": "kale-tts-runs", **json.loads(Path(p).read_text())} return {"train_app": "kale-tts-train", "runs_volume": "kale-tts-runs"} def jl(p): return [json.loads(x) for x in Path(p).read_text().splitlines() if x.strip()] def pace_window(fin, voice_cfg, rule, recorded=True): """the duration head's takes: a wpm window centred where this voice's book-text pace maps to its natural volcano-line pace. recorded=True (default) reuses the bounds the release used (config/voices.json pace_window.lo / hi).""" pw = voice_cfg["pace_window"] if recorded or "maya_volcano_wpm" not in pw: keep = [r["clip"] for r in fin if pw["lo"] <= r["wpm"] <= pw["hi"]] return keep, dict(lo=pw["lo"], hi=pw["hi"], center=round((pw["lo"] + pw["hi"]) / 2, 1), half_width=round((pw["hi"] - pw["lo"]) / 2, 2), kept=len(keep), of=len(fin), rule="recorded bounds") volc = float(voice_cfg["pace_window"]["maya_volcano_wpm"]) # Maya's own volcano-line pace for this design (40 words / clip s) target = min(200.0, max(160.0, volc)) # 160-200 wpm; each voice around its own pace med = S.median(r["wpm"] for r in fin) # speech-only wpm of the kept book takes c = med * target / volc w = rule["half_width_start"] while True: keep = [r["clip"] for r in fin if abs(r["wpm"] - c) <= w] if len(keep) >= rule["min_takes"] or w > rule["half_width_max"]: break w += rule["half_width_step"] return keep, dict(target_wpm=target, volcano_wpm_maya=volc, book_wpm_median=med, center=round(c, 1), half_width=w, kept=len(keep), of=len(fin)) def main(): ap = argparse.ArgumentParser() ap.add_argument("voice") ap.add_argument("--ids-suffix", default="", help="gb for the en-GB voices (ids_gb.json, dur_gb.pt)") ap.add_argument("--recompute-window", action="store_true", help="apply pace_window_rule instead of the recorded release bounds") ap.add_argument("--kept", default="", help="kept list (default /kept-.jsonl; a group file is fine)") ap.add_argument("--strict", default="", help="strict list (default /strict-.jsonl; a group file is fine)") ap.add_argument("--dry-run", action="store_true") A = ap.parse_args() v = A.voice P = cfg("paths.json") vc = {x["voice"]: x for x in cfg("voices.json")["voices"]}[v] rule = cfg("voices.json")["pace_window_rule"] work = HERE / P["work_dir"] work.mkdir(parents=True, exist_ok=True) K = json.loads((HERE / P["ids_dir"] / f"ids_{v}_all.json").read_text())["rows"] kept = jl(A.kept or HERE / P["kept_dir"] / f"kept-{v}.jsonl") kept = [k for k in kept if k["clip"].split("_")[0] == v] # a group render (several voices in one call): this voice's takes only st = {r["clip"]: r for r in jl(A.strict or HERE / P["kept_dir"] / f"strict-{v}.jsonl")} multi = lambda t: len([s for s in re.split(r"(?<=[.!?])\s+", t.strip()) if s]) > 1 # noqa: E731 final, why = [], {} for k in kept: s = st.get(k["clip"]) if multi(k["text"]): bad = ("strict check missing" if s is None or "why" in s else "not alignable" if not s.get("aligned") else "rough final end (strict)" if s.get("end_ms", 0) >= 100 else "rough inner sentence end" if any(x >= 100 for x in s.get("inner_ms", [])) else "") if bad: why[bad] = why.get(bad, 0) + 1 continue final.append(k) split = lambda c: "val" if int(hashlib.md5(c.encode()).hexdigest(), 16) % 100 == 0 else "train" # noqa: E731 rows = [dict(clip=k["clip"], split=split(k["clip"]), ids=K[k["clip"]], text=k["text"], src=f"{P['renders_root']}/{k['path']}") for k in final] tag = v + A.ids_suffix (work / f"ids_{tag}.json").write_text(json.dumps(dict(rows={r["clip"]: r["ids"] for r in rows}))) (work / f"final-{v}.jsonl").write_text("".join(json.dumps(k) + "\n" for k in final)) summ = dict(kept_filter=len(kept), final=len(rows), strict_drops=why, hours=round(sum(k["seconds"] for k in final) / 3600, 2), val=sum(r["split"] == "val" for r in rows), passages=sum(multi(k["text"]) for k in final), wpm_median=sorted(k["wpm"] for k in final)[len(final) // 2] if final else None) (work / f"final-summary-{v}.json").write_text(json.dumps(summ, indent=1)) print(json.dumps(summ)) keep, info = pace_window(final, vc, rule, recorded=not A.recompute_window) (work / f"dur_keep_{v}.json").write_text(json.dumps(keep)) (work / f"pace-window-{v}.json").write_text(json.dumps(dict(voice=v, **info), indent=1)) print(v, "window", round(info["center"] - info["half_width"], 1), "-", round(info["center"] + info["half_width"], 1), len(keep), "of", len(final)) if A.dry_run: return import modal N = modal_names() prep = modal.Function.from_name(N["train_app"], "prep") calls = [prep.spawn(rows[i:i + 6000], P["renders_root"], v, i // 500) for i in range(0, len(rows), 6000)] vol = modal.Volume.from_name(N["runs_volume"]) with vol.batch_upload(force=True) as b: b.put_file(str(work / f"ids_{tag}.json"), f"/feat/ids_{tag}.json") b.put_file(str(work / f"dur_keep_{v}.json"), f"/feat/dur_keep_{v}.json") for c in calls: print(json.dumps({k: x for k, x in c.get().items() if k != "dropped"})) dur = modal.Function.from_name(N["train_app"], "durations_fn") print(json.dumps(dur.remote(f"/data/feat/ids_{tag}.json", P["teacher_ckpt"], P["teacher_aligner_cfg"], f"dur_{tag}", f"/data/feat-{v}/shard_*.pt"), indent=1)) print(f"PREP-{v}-DONE") if __name__ == "__main__": main()