NERDDISCO's picture
kale-tts-1
f21d6eb
Raw History Blame Contribute Delete
10 kB
# SPDX-License-Identifier: Apache-2.0
# Copyright 2026 Kale
"""Keep rules for rendered Maya-1 takes (the rules that kept the release data), from the filter's measures -> kept-<group>.jsonl + a summary.
Thresholds: config/filter.json "keep" (defaults below = the release values).
Every take: complete (Maya's end token), read-back exact (0 missed, no extra words; tag takes: laugh sounds heard as "haha" etc. are
allowed, the tag word itself is not), starts after 20 ms-0.8 s of silence and ends within 0.8 s, not cut off at the end (end level
> -15 dB under the 90th-percentile frame), no full-scale sample, pace 100-230 spoken words a minute (per-voice ceiling for naturally
fast voices: s057b 270), pitch within 15 % of its voice's median, a rough-ending measure on the final end and every inner pause-end
< 100 ms (a take without an ending measure is dropped).
Voice match: one Maya description can give several voices across seeds, so each (voice, mood) gets its own centroid: mean of all
ECAPA-TDNN embeddings (speechbrain/spkrec-ecapa-voxceleb, Apache-2.0), then the mean of the closest 60 %, and a take stays at
cosine >= 0.46 (whisper mood: 0.42). The release's own keep pass used WavLM-sv at 0.88 / 0.80; WavLM-sv is replaced because its
pretraining data (GigaSpeech) is non-commercial. ECAPA cosines run lower: on the release takes WavLM-sv kept, ECAPA to the voice centre
has a median of 0.80-0.88 and a 1st percentile of 0.46-0.59 (4 voices measured), so 0.46 keeps at least ~99 % of those takes. How many
wrong-voice takes it lets through was not measured; tighten it per voice if you see drift.
Whisper mood: voiced share under 0.25. Tag takes: the gap where the tag sits must be >= 0.3 s long and hold >= 0.15 s of sound.
Male voices (70 Hz pitch floor): a second, bench-style detector is measured; in this version it is RECORDED ONLY (bench_flag), never a
drop reason: it flags nearly every normal low male sentence end in Maya-1 audio, clean singles included. The per-voice "bench_exempt"
test below (does the detector shift the voice's kept median pitch up by > 5 %?) is still computed and reported, but has no effect.
Run by filter.py::keepall on the volume; locally: python3 keep.py <group> ... (reads work/measure-<group>.jsonl with embeddings)."""
import collections
import json
import os
import statistics as S
import sys
from pathlib import Path
import numpy as np
H = Path(os.environ.get("KALE_TTS_ROOT", ".")) # folder holding work/ and results/
KEEP = {"sim": 0.46, "sim_whisper": 0.42, "pace_min": 100, "pace_max": 230, "pace_max_per_voice": {"s057b": 270}, "pitch_tol": 0.15,
"lead_min_s": 0.02, "lead_max_s": 0.8, "tail_max_s": 0.8, "end_level_max_db": -15, "peak_max": 0.999, "rough_ms": 100,
"whisper_voiced_max": 0.25, "centroid_top": 0.6, "bench_exempt_shift": 0.05}
for _p in (os.environ.get("KALE_TTS_FILTER_CONFIG"), Path(__file__).resolve().parent / "config" / "filter.json", "/root/recipe/filter.json"):
if _p and Path(_p).is_file():
KEEP.update(json.loads(Path(_p).read_text()).get("keep", {}))
break
SIM, SIM_W = KEEP["sim"], KEEP["sim_whisper"]
OK_EXTRA = {"ha", "haha", "hahaha", "hah", "heh", "hehe", "hehehe", "oh", "ah", "aah", "ahh", "hm", "hmm", "huh", "whoa", "wow", "ooh", "oh-oh", "uh", "phew", "mm"}
TAGWORDS = {"laugh", "laughs", "laughing", "giggle", "giggles", "sigh", "sighs", "gasp", "gasps", "chuckle", "chuckles", "whisper", "curious"}
def rules(r):
why = []
if r.get("ok") is False or "seconds" not in r:
return ["unreadable"]
if not r.get("eos"):
why.append("no end token")
extra = [w for w in r.get("extra", [])]
if r.get("missed", 1):
why.append("read-back: missed words")
if r.get("tagname") and r.get("mood") not in ("whisper", "curious"):
if any(w in TAGWORDS for w in extra):
why.append("tag spoken as a word")
elif any(w not in OK_EXTRA for w in extra):
why.append("read-back: extra words")
elif any(w in TAGWORDS for w in extra):
why.append("tag spoken as a word")
elif extra:
why.append("read-back: extra words")
if not (KEEP["lead_min_s"] <= r["lead_s"] <= KEEP["lead_max_s"]):
why.append("start edge")
if r["tail_s"] > KEEP["tail_max_s"]:
why.append("end edge")
if r["end_level"] > KEEP["end_level_max_db"]:
why.append("cut off")
if r["peak"] >= KEEP["peak_max"]:
why.append("full scale")
PACE_MAX = KEEP["pace_max_per_voice"] # s057b: the esports caster's natural pace sits above 230 (median 225 speech wpm); keep her own window
if not (KEEP["pace_min"] <= r["wpm"] <= PACE_MAX.get(r.get("voice"), KEEP["pace_max"])) and r.get("mood") != "whisper":
why.append("pace")
# the clean-ending filter gates every kept take, no exceptions: no ending measure -> dropped
if r.get("end_ms") is None and r.get("mood") != "whisper" and not r.get("tagname"):
why.append("rough ending (not measured)")
# whispers and tag sounds (sigh, laugh) are unvoiced by design: the rough-ending rule would drop them all
if (r.get("end_ms") is not None and r.get("mood") != "whisper" and not r.get("tagname")
and (r["end_ms"] >= KEEP["rough_ms"] or any(x >= KEEP["rough_ms"] for x in r.get("inner_ms", [])))):
why.append("rough ending")
# male voices (pitch floor 70 Hz): the bench-style check is recorded only (an earlier version dropped if either detector flagged)
elif (r.get("fmin") == 70 and r.get("bench_end_ms") is not None and not r.get("bench_exempt") and r.get("mood") != "whisper"
and not r.get("tagname") and (r["bench_end_ms"] >= 100 or any(x >= 100 for x in r.get("bench_inner_ms", [])))):
r["bench_flag"] = True # recorded, not dropped (flags ~all male ends in Maya-1 audio, incl. clean singles); the male rule is the
# rough-ending measure with the 70 Hz floor on every sentence end + strict.py's every-end check
return why
def main(groups, src=None, out_dir=None, paths=None):
summ = {}
out_dir = out_dir or H / "work"
paths = paths or {}
for g in groups:
rs = src(g) if src else [json.loads(l) for l in (H / f"work/measure-{g}.jsonl").read_text().splitlines()]
by = collections.defaultdict(list)
for r in rs:
by[(r.get("voice"), r.get("mood") or "default")].append(r)
kept, drops = [], collections.Counter()
per_exempt = []
per = {}
for (v, m), xs in by.items():
good = [r for r in xs if "emb" in r]
if not good:
continue
E = np.array([r["emb"] for r in good], dtype=np.float32)
c = E.mean(0); c /= np.linalg.norm(c)
s = E @ c
top = E[np.argsort(-s)[:max(1, int(KEEP["centroid_top"] * len(good)))]]
c = top.mean(0); c /= np.linalg.norm(c)
sims = E @ c
f0s = [r["f0"] for r in good if r.get("f0")]
f0m = S.median(f0s) if f0s else None
if any(r.get("fmin") == 70 and r.get("bench_end_ms") is not None for r in good):
# if the bench-style check would shift a man's kept median pitch up by > 5 %, mark the voice exempt (informational now)
allf = [r["f0"] for r in good if r.get("f0")]
keptf = [r["f0"] for r in good if r.get("f0") and not ((r.get("bench_end_ms") or 0) >= 100 or any(x >= 100 for x in r.get("bench_inner_ms", [])))]
if allf and keptf and S.median(keptf) > (1 + KEEP["bench_exempt_shift"]) * S.median(allf):
for r in good:
r["bench_exempt"] = True
per_exempt.append(v)
nk, rough_n = 0, 0
for r, sim in zip(good, sims):
r["sim"] = round(float(sim), 4)
why = rules(r)
if sim < (SIM_W if m == "whisper" else SIM):
why.append("voice drift")
if m == "whisper":
if r.get("voiced") is None or r["voiced"] >= KEEP["whisper_voiced_max"]:
why.append("not whispered")
elif f0m and r.get("f0") and not ((1 - KEEP["pitch_tol"]) * f0m <= r["f0"] <= (1 + KEEP["pitch_tol"]) * f0m):
why.append("pitch")
if r.get("tagname") and m not in ("whisper", "curious"):
if r.get("tag_pos") in (None, "unaligned"):
why.append("tag not located")
elif r.get("gap_s", 0) < 0.3 or r.get("gap_active_s", 0) < 0.15:
why.append("no sound at the tag")
rough_n += any(w.startswith("rough ending") for w in why)
for w in why:
drops[w] += 1
if not why:
kept.append({k: r[k] for k in ("clip", "voice", "mood", "tagname", "text", "seconds", "sim", "wpm", "f0", "voiced") if k in r}
| {"path": paths.get(r["clip"])})
nk += 1
per[f"{v}/{m}"] = dict(takes=len(xs), kept=nk, kept_h=round(sum(r["seconds"] for r in kept[-nk:]) / 3600, 2) if nk else 0.0,
f0=f0m, sim_median=round(float(np.median(sims)), 3), rough_ending_pct=round(100 * rough_n / max(1, len(good)), 1))
out_dir.mkdir(parents=True, exist_ok=True)
(out_dir / f"kept-{g}.jsonl").write_text("".join(json.dumps(r) + "\n" for r in kept))
summ[g] = dict(bench_exempt_voices=per_exempt, takes=len(rs), kept=len(kept), hours=round(sum(r["seconds"] for r in kept) / 3600, 2), drops=dict(drops.most_common()), per=per)
print(g, summ[g]["takes"], "->", summ[g]["kept"], f"{summ[g]['hours']} h", dict(drops.most_common(8)))
return summ
if __name__ == "__main__":
s_ = main(sys.argv[1:])
p = H / "results/kept-summary.json"
old = json.loads(p.read_text()) if p.exists() else {}
old.update(s_)
p.write_text(json.dumps(old, indent=1))