Download recipe/filter/final.py from KaleComputer/kale-tts-1-tiny: direct link, hf CLI and curl.
- Browser
- Download file 3.5 kB
-
https://huggingface.co/KaleComputer/kale-tts-1-tiny/resolve/main/recipe/filter/final.py
- Command line
-
hf download hf://KaleComputer/kale-tts-1-tiny/recipe/filter/final.py
-
curl -L -o final.py https://huggingface.co/KaleComputer/kale-tts-1-tiny/resolve/main/recipe/filter/final.py
3.5 kB
| # SPDX-License-Identifier: Apache-2.0 | |
| # Copyright 2026 Kale | |
| """Final per-voice training list (the release path's last data step): filter.py's kept takes (keep.py rules) minus every multi-sentence | |
| take that strict.py flags (missing / not alignable / final or any inner sentence end >= 100 ms rough). 1 in 100 takes (md5 of the clip | |
| id) -> split "val". Writes, under <root>/work/ (root = env KALE_TTS_ROOT or "."): | |
| final-<voice>.jsonl kept rows that passed (clip, voice, text, seconds, sim, wpm, f0, path on the data volume) | |
| rows-<voice>.jsonl the train step's prep input: clip, split, text, path (+ ids if work/ids_<voice>_all.json exists) | |
| ids_<voice>.json {"rows": {clip: token ids}} for the final takes (only if the ids file exists) | |
| and <root>/results/final-summary-<voice>.json (kept, final, strict drops, hours, passages, median wpm). | |
| python3 -I final.py <voice> [--kept work/kept-<group>.jsonl] [--strict work/strict-<group>.jsonl] (default group = the voice) | |
| Trimmed from the original: it also spawned the train app's feature prep on these rows; that call now lives in the train step.""" | |
| import argparse | |
| import hashlib | |
| import json | |
| import os | |
| import re | |
| from pathlib import Path | |
| ap = argparse.ArgumentParser() | |
| ap.add_argument("voice"); ap.add_argument("--kept", default=""); ap.add_argument("--strict", default="") | |
| A = ap.parse_args() | |
| V = Path(os.environ.get("KALE_TTS_ROOT", ".")) | |
| v = A.voice | |
| idp = V / f"work/ids_{v}_all.json" | |
| K = json.loads(idp.read_text())["rows"] if idp.exists() else None | |
| kept = [json.loads(l) for l in Path(A.kept or V / f"work/kept-{v}.jsonl").read_text().splitlines() if l.strip()] | |
| kept = [k for k in kept if k["clip"].split("_")[0] == v] # a group render (several voices in one call): this voice's takes only | |
| st = {json.loads(l)["clip"]: json.loads(l) for l in Path(A.strict or V / f"work/strict-{v}.jsonl").read_text().splitlines() if l.strip()} | |
| multi = lambda t: len([s for s in re.split(r"(?<=[.!?])\s+", t.strip()) if s]) > 1 # noqa: E731 | |
| final, why = [], {} | |
| for k in kept: | |
| s = st.get(k["clip"]) | |
| if multi(k["text"]): | |
| bad = ("strict check missing" if s is None or "why" in s else "not alignable" if not s.get("aligned") else | |
| "rough final end (strict)" if s.get("end_ms", 0) >= 100 else "rough inner sentence end" if any(x >= 100 for x in s.get("inner_ms", [])) else "") | |
| if bad: | |
| why[bad] = why.get(bad, 0) + 1; continue | |
| final.append(k) | |
| split = lambda c: "val" if int(hashlib.md5(c.encode()).hexdigest(), 16) % 100 == 0 else "train" # noqa: E731 | |
| rows = [dict(clip=k["clip"], split=split(k["clip"]), text=k["text"], path=k["path"]) | ({"ids": K[k["clip"]]} if K else {}) for k in final] | |
| (V / "work").mkdir(parents=True, exist_ok=True) | |
| if K: | |
| (V / f"work/ids_{v}.json").write_text(json.dumps(dict(rows={r["clip"]: r["ids"] for r in rows}))) | |
| (V / f"work/final-{v}.jsonl").write_text("".join(json.dumps(k) + "\n" for k in final)) | |
| (V / f"work/rows-{v}.jsonl").write_text("".join(json.dumps(r) + "\n" for r in rows)) | |
| summ = dict(kept_filter=len(kept), final=len(rows), strict_drops=why, hours=round(sum(k["seconds"] for k in final) / 3600, 2), | |
| val=sum(r["split"] == "val" for r in rows), passages=sum(multi(k["text"]) for k in final), | |
| wpm_median=sorted(k["wpm"] for k in final)[len(final) // 2] if final else None) | |
| (V / "results").mkdir(parents=True, exist_ok=True) | |
| (V / f"results/final-summary-{v}.json").write_text(json.dumps(summ, indent=1)) | |
| print(json.dumps(summ)) | |