NERDDISCO's picture
kale-tts-1
f21d6eb
Raw History Blame Contribute Delete
23.7 kB
# SPDX-License-Identifier: Apache-2.0
# Copyright 2026 Kale
"""Batch filter, the path that kept every release voice's training takes (16 Nov 2026). Three Modal entry points, run after render.py:
main per take, on one GPU per call: length, peak, edge silence, end level (cut off?), pitch, voiced share, pace (spoken words / speech
seconds), ECAPA-TDNN embedding (voice match is computed later against each voice's OWN centroid), Whisper large-v3-turbo
read-back of the text (numbers spoken on both sides, `spoken`), and for whisper / tag takes a CLAP zero-shot score
(laion/clap-htsat-unfused, Apache-2.0; evaluation only; unused by the release voices, which have no moods or tags).
ends_main the rough-ending measure on CPUs: longest run of rough frames (above -30 dB, spectral centroid < 2.5 kHz, no pyin pitch
fmin-700 Hz) in the last ~300 ms before the take's end and before every inner pause >= 150 ms; fmin 70 Hz for male voices
(description says " male voice"), 110 Hz otherwise. Also records a second, bench-style detector for male voices (not used to drop).
keepall keep.py's rules on the full measures -> /data/kept/kept-<group>.jsonl (+ results/kept-summary.json locally).
Then strict.py checks every sentence end of the kept passages via forced alignment, and final.py writes the final per-voice list.
modal run filter.py::main --group <g> --per-call 400
modal run filter.py::ends_main --group <g> --per-call 150
modal run filter.py::keepall --groups <g>
Reads <root>/work/jobs-<g>.jsonl and <root>/work/render-<g>/chunk-*.json (render.py outputs); root = env KALE_TTS_ROOT or ".".
Writes <root>/work/measure-<g>.slim.jsonl (no embeddings; the full rows stay on the volume under /data/measure/<g>/)."""
import json
import os
import re
from pathlib import Path
import modal
def _cfg(name, default):
"""recipe config: env KALE_TTS_<NAME>_CONFIG, then ../config/<name>.json (modal.json) or ./config/<name>.json, then the copy in the image"""
here = Path(__file__).resolve().parent
for p in (os.environ.get(f"KALE_TTS_{name.upper()}_CONFIG"), here.parent / "config" / f"{name}.json", here / "config" / f"{name}.json",
Path("/root/recipe") / f"{name}.json"):
if p and Path(p).is_file():
return {**default, **json.loads(Path(p).read_text())}
return default
NAMES = _cfg("modal", {"filter_app": "kale-tts-filter", "data_volume": "kale-tts-data"})
CFG = _cfg("filter", {"measure_gpus": 8, "endings_cpus": 64})
app = modal.App(NAMES["filter_app"])
vol = modal.Volume.from_name(NAMES["data_volume"], create_if_missing=False) # render.py's volume (renders under /data/onevoice/)
def _cfg_files(image, names=("modal", "filter")):
"""add the config files to an image so the containers read the same names and thresholds"""
for n in names:
for p in (Path(__file__).resolve().parent.parent / "config" / f"{n}.json", Path(__file__).resolve().parent / "config" / f"{n}.json"):
if p.is_file():
image = image.add_local_file(str(p), f"/root/recipe/{n}.json")
break
return image
def _p(rel):
return f"/data/{rel}"
ASR = "openai/whisper-large-v3-turbo"
CLAP = "laion/clap-htsat-unfused"
img = (modal.Image.debian_slim(python_version="3.11").apt_install("ffmpeg", "libsndfile1")
.pip_install("torch==2.4.1", "torchaudio==2.4.1", "transformers==4.46.3", "accelerate", "soundfile", "librosa==0.10.2", "numpy<2", "scipy", "num2words")
.run_commands(f"python -c \"from transformers import WhisperForConditionalGeneration as W, WhisperProcessor as P; P.from_pretrained('{ASR}'); W.from_pretrained('{ASR}')\"",
"pip install speechbrain==1.0.3 'huggingface_hub<0.26'",
"python -c \"from speechbrain.inference.speaker import EncoderClassifier as E; E.from_hparams(source='speechbrain/spkrec-ecapa-voxceleb', savedir='/opt/ecapa')\"",
f"python -c \"from transformers import ClapModel, ClapProcessor; ClapModel.from_pretrained('{CLAP}'); ClapProcessor.from_pretrained('{CLAP}')\""))
img = _cfg_files(img)
img_keep = _cfg_files(modal.Image.debian_slim(python_version="3.11").pip_install("numpy<2")
.add_local_file(str(Path(__file__).resolve().parent / "keep.py"), "/opt/keep.py", copy=True))
TAG_PROMPTS = {"laugh": "a person laughing", "giggle": "a person giggling", "sigh": "a person sighing", "gasp": "a person gasping",
"chuckle": "a person chuckling", "speech": "a person talking calmly", "whisper": "a person whispering"}
def spoken(word):
from num2words import num2words
t = re.sub(r"(\d+),(\d{3})", r"\1\2", word)
t = t.replace("%", " percent ").replace("$", " dollars ").replace("&", " and ")
t = re.sub(r"\d+(\.\d+)?", lambda m: " " + num2words(float(m.group()) if "." in m.group() else int(m.group())) + " ", t)
t = t.lower().replace("-", " ").replace("’", "'")
return [w.strip("'") for w in re.sub(r"[^a-z' ]", " ", t).split() if w.strip("'")]
@app.function(image=img, gpu=["H100", "L40S"], cpu=8, memory=24576, timeout=3600, max_containers=int(CFG["measure_gpus"]),
scaledown_window=60, volumes={"/data": vol})
def measure(items: list):
import difflib
import librosa
import numpy as np
import soundfile as sf
import torch
from speechbrain.inference.speaker import EncoderClassifier # ECAPA-TDNN (Apache-2.0) for voice match
from transformers import ClapModel, ClapProcessor, WhisperForConditionalGeneration, WhisperProcessor
vol.reload()
dev = "cuda"
proc = WhisperProcessor.from_pretrained(ASR)
wm = WhisperForConditionalGeneration.from_pretrained(ASR, torch_dtype=torch.float16).to(dev).eval()
ec = EncoderClassifier.from_hparams(source="speechbrain/spkrec-ecapa-voxceleb", savedir="/opt/ecapa", run_opts={"device": dev})
cm, cp = ClapModel.from_pretrained(CLAP).to(dev).eval(), ClapProcessor.from_pretrained(CLAP)
from transformers import pipeline
asr_ts = pipeline("automatic-speech-recognition", model=wm, tokenizer=proc.tokenizer, feature_extractor=proc.feature_extractor,
torch_dtype=torch.float16, device=dev)
names = list(TAG_PROMPTS)
def clap(seg):
y48 = librosa.resample(seg, orig_sr=16000, target_sr=48000, res_type="soxr_hq")
with torch.no_grad():
ae = cm.get_audio_features(**cp(audios=[y48], sampling_rate=48000, return_tensors="pt").to(dev))
ae = torch.nn.functional.normalize(ae, dim=-1)
return {n_: round(x, 4) for n_, x in zip(names, (ae @ te.T)[0].cpu().tolist())}
def tag_gap(y16, text):
"""where the tag sits, the gap between the aligned words around it: its length, energy, voiced share, CLAP."""
toks = text.split()
ti = next((i for i, w in enumerate(toks) if re.fullmatch(r"<[a-z_]+>", w)), None)
if ti is None:
return {}
words = [w for w in toks if not re.fullmatch(r"<[a-z_]+>", w)]
o = asr_ts({"raw": y16, "sampling_rate": 16000}, return_timestamps="word", generate_kwargs=dict(language="en", task="transcribe"))
hw = [(c["text"], c["timestamp"]) for c in o["chunks"]]
a = [" ".join(spoken(w)) for w in words]
b = [" ".join(spoken(w)) for w, _ in hw]
idx = {}
for op, i0, i1, j0, j1 in difflib.SequenceMatcher(None, a, b, autojunk=False).get_opcodes():
if op == "equal":
for k in range(i1 - i0):
idx[i0 + k] = j0 + k
dur_ = len(y16) / 16000
if ti == 0:
before = 0.0
after = hw[idx[0]][1][0] if 0 in idx else None
elif ti >= len(words):
before = hw[idx[len(words) - 1]][1][1] if len(words) - 1 in idx else None
after = dur_
else:
before = hw[idx[ti - 1]][1][1] if ti - 1 in idx else None
after = hw[idx[ti]][1][0] if ti in idx else None
if before is None or after is None:
return dict(tag_pos="unaligned")
after = after if after is not None else dur_
g0, g1 = before, after
seg = y16[int(g0 * 16000):int(g1 * 16000)]
res = dict(tag_pos="start" if ti == 0 else ("end" if ti >= len(words) else "mid"), gap_s=round(g1 - g0, 3))
if len(seg) > 1600:
ef = 10 * np.log10(np.convolve(seg ** 2, np.ones(320) / 320, "valid")[::160] + 1e-10)
sp_ = np.percentile(10 * np.log10(np.convolve(y16 ** 2, np.ones(320) / 320, "valid")[::160] + 1e-10), 90)
res.update(gap_active_s=round(float((ef > sp_ - 30).sum() * 0.01), 3), gap_level=round(float(np.percentile(ef, 90) - sp_), 1))
if len(seg) >= 4000:
res["gap_clap"] = clap(seg)
return res
with torch.no_grad():
te = cm.get_text_features(**cp(text=[TAG_PROMPTS[n] for n in names], return_tensors="pt", padding=True).to(dev))
te = torch.nn.functional.normalize(te, dim=-1)
def emb(yy):
n = 16000 * 8
cs = [yy[i:i + n] for i in range(0, len(yy), n) if len(yy[i:i + n]) > 16000] or [yy]
es = []
for c in cs:
with torch.no_grad():
es.append(torch.nn.functional.normalize(ec.encode_batch(torch.from_numpy(np.ascontiguousarray(c, dtype=np.float32))[None].to(dev))[0, 0].float(), dim=-1))
return torch.nn.functional.normalize(torch.stack(es).mean(0), dim=-1)
out, live = [], []
for it in items:
try:
y, sr = sf.read(_p(it['path']), dtype="float32", always_2d=True)
y = y.mean(1)
dur = len(y) / sr
y16 = librosa.resample(y, orig_sr=sr, target_sr=16000, res_type="soxr_hq")
hop = sr // 200
n = len(y) // hop
e = 10 * np.log10((y[:n * hop].reshape(n, hop) ** 2).mean(1) + 1e-10)
sp = np.percentile(e, 90)
loud = np.flatnonzero(e > sp - 35)
lead = float(loud[0] * hop / sr) if len(loud) else dur
tail = float(dur - (loud[-1] + 1) * hop / sr) if len(loud) else dur
f0, vshare = None, None
h16 = 80
nf = max(0, (len(y16) - 480) // h16 + 1)
if nf > 0:
idx = np.arange(480)[None, :] + h16 * np.arange(nf)[:, None]
F = y16[idx] * np.hanning(480)[None, :]
F = F - F.mean(1, keepdims=True)
ac = np.fft.irfft(np.abs(np.fft.rfft(F, 1024, axis=1)) ** 2, axis=1)[:, :240]
nac = ac[:, 40:200].max(1) / (ac[:, 0] + 1e-9)
en = 10 * np.log10((F ** 2).mean(1) + 1e-10)
act = en > np.percentile(en, 90) - 35
v = (nac > 0.5) & act
vshare = float(v.sum() / max(1, act.sum()))
if v.sum() >= 10:
f0 = float(np.median(16000 / (ac[:, 40:200].argmax(1) + 40)[v]))
plain = re.sub(r"<[a-z_]+>", " ", it["text"]).strip()
sw = [spoken(w) for w in plain.split()]
n_sp = sum(len(x) for x in sw)
r = dict(clip=it["clip"], voice=it["voice"], mood=it.get("mood"), tagname=it.get("tagname"), text=it["text"], seconds=round(dur, 2),
peak=round(float(np.abs(y).max()), 4), lead_s=round(lead, 3), tail_s=round(tail, 3), end_level=round(float(e[-6:].mean() - sp), 1),
f0=None if f0 is None else round(f0, 1), voiced=None if vshare is None else round(vshare, 3), spoken_words=n_sp,
wpm=round(n_sp / max(0.1, dur - lead - tail) * 60, 1), emb=[round(float(x), 4) for x in emb(y16).cpu()])
if it.get("mood") == "whisper":
r["clap"] = clap(y16)
if it.get("tagname") and it.get("mood") != "whisper" and "<" in it["text"]:
try:
r.update(tag_gap(y16, it["text"]))
except Exception as ex: # noqa: BLE001
r["gap_why"] = repr(ex)[:200]
out.append(r)
live.append((r, y16, sw))
except Exception as ex: # noqa: BLE001
out.append(dict(clip=it["clip"], voice=it.get("voice"), ok=False, why=repr(ex)[:300]))
for b in range(0, len(live), 24):
part = live[b:b + 24]
feats = proc([x[1] for x in part], sampling_rate=16000, return_tensors="pt").input_features.to(dev, torch.float16)
with torch.no_grad():
g = wm.generate(feats, language="en", task="transcribe", num_beams=1, do_sample=False)
for (r, _, sw), h in zip(part, proc.tokenizer.batch_decode(g, skip_special_tokens=True)):
a = [t for ws in sw for t in ws]
hb = [t for w in h.split() for t in spoken(w)]
miss, extra = 0, []
for op, i0, i1, j0, j1 in difflib.SequenceMatcher(None, a, hb, autojunk=False).get_opcodes():
if op in ("replace", "delete"):
miss += i1 - i0
if op in ("replace", "insert"):
extra += hb[j0:j1]
r.update(heard=h.strip(), missed=miss, extra=extra)
# the full measures (voice embeddings) stay on the volume; only slim rows come back to the local machine
d = Path(f"/data/measure/{items[0]['group']}")
d.mkdir(parents=True, exist_ok=True)
(d / f"{items[0]['clip']}.jsonl").write_text("".join(json.dumps(r) + "\n" for r in out))
vol.commit()
return [{k: v for k, v in r.items() if k != "emb"} for r in out]
@app.function(image=img_keep, cpu=4, memory=16384, timeout=1800, max_containers=1, volumes={"/data": vol})
def keep_remote(groups: list, out: str = "kept", ends_tag: str = ""):
"""keep.py on the volume's full measures -> /data/kept/kept-<group>.jsonl (with the wav path) + summary (returned)"""
import sys
sys.path.insert(0, "/opt")
import keep
vol.reload()
paths, eos = {}, {}
for g in groups:
for f in list(Path("/data/onevoice/_vllm_chunks").glob(f"{g}/*.json")): # render.py's chunk markers (--sub <g>)
for t in json.loads(f.read_text()).get("out", []) if f.read_text().startswith("{") else json.loads(f.read_text()):
if t.get("path"):
paths[t["clip"]] = t["path"]
eos[t["clip"]] = t.get("eos")
def src(g): # full measures + Maya's end-token flag from the render's own chunk records
rows = {}
for f in sorted(Path(f"/data/measure/{g}").glob("*.jsonl")):
for l in f.read_text().splitlines():
x = json.loads(l); rows[x["clip"]] = x
rows = list(rows.values())
ends = {}
ed = Path(f"/data/endings/{g}{ends_tag}") if Path(f"/data/endings/{g}{ends_tag}").exists() else Path(f"/data/endings/{g}")
for f in ed.glob("*.jsonl") if ed.exists() else []:
for l in f.read_text().splitlines():
x = json.loads(l)
ends[x["clip"]] = x
for r in rows:
r["eos"] = eos.get(r["clip"])
if r["clip"] in ends and "end_ms" in ends[r["clip"]]:
e_ = ends[r["clip"]]
r["end_ms"], r["inner_ms"] = e_["end_ms"], e_["inner_ms"]
r["fmin"], r["bench_end_ms"], r["bench_inner_ms"] = e_.get("fmin", 110), e_.get("bench_end_ms"), e_.get("bench_inner_ms", [])
return rows
summ = keep.main(groups, src=src, out_dir=Path(f"/data/{out}"), paths=paths)
vol.commit()
return summ
@app.local_entrypoint()
def main(group: str, per_call: int = 250):
root = Path(os.environ.get("KALE_TTS_ROOT", "."))
rows = [json.loads(x) for x in (root / f"work/jobs-{group}.jsonl").read_text().splitlines() if x.strip()]
done = {}
for f in sorted((root / f"work/render-{group}").glob("chunk-*.json")):
for t in json.loads(f.read_text()):
if t.get("path"):
done[t["clip"]] = t
items = [dict(r, group=group, path=done[r["clip"]]["path"], eos=done[r["clip"]].get("eos")) for r in rows if r["clip"] in done]
print(group, len(items), "rendered takes of", len(rows), flush=True)
od = root / f"work/measure-{group}.slim.jsonl" # slim: no embeddings (full rows on the volume, /measure/<group>/)
have = {json.loads(l)["clip"] for l in od.read_text().splitlines() if l.strip()} if od.exists() else set() # resume: skip measured
items = [it for it in items if it["clip"] not in have]
print(group, len(items), "still to measure", flush=True)
with od.open("a") as f:
for res in measure.map([items[i:i + per_call] for i in range(0, len(items), per_call)], order_outputs=False, return_exceptions=True):
if isinstance(res, Exception):
print("FAILED", repr(res)[:300], flush=True)
continue
for r in res:
r["eos"] = done.get(r["clip"], {}).get("eos")
f.write(json.dumps(r) + "\n")
f.flush()
print("measured", len(res), flush=True)
@app.local_entrypoint()
def keepall(groups: str, out: str = "kept", ends_tag: str = ""):
summ = keep_remote.remote(groups.split(","), out, ends_tag)
root = Path(os.environ.get("KALE_TTS_ROOT", "."))
p = root / ("results/kept-summary.json" if out == "kept" else f"results/{out}-summary.json")
old = json.loads(p.read_text()) if p.exists() else {}
old.update(summ)
p.write_text(json.dumps(old, indent=1))
for g, v in summ.items():
print(g, v["takes"], "->", v["kept"], v["hours"], "h", v["drops"])
# ===================================================================================================================== rough endings
# The rough-ending ("T's") measure, unchanged maths: the longest run of rough frames (above -30 dB, centroid under 2.5 kHz, no pitch
# fmin-700 Hz) in the last 300 ms before the take's end and, for takes with more than one sentence, before every inner pause of >= 150 ms.
# keep.py drops a take when any ending has >= 100 ms. Pause-found ends miss sentence ends without a pause: strict.py checks those.
img_end = _cfg_files(modal.Image.debian_slim(python_version="3.10").apt_install("libsndfile1").pip_install("librosa==0.10.2", "soundfile", "numpy<2"))
@app.function(image=img_end, cpu=4, memory=4096, timeout=1800, max_containers=int(CFG["endings_cpus"]), scaledown_window=60, volumes={"/data": vol})
def endings(rows: list):
import librosa
import numpy as np
import soundfile as sf
HOP = 256
out = []
for r in rows:
try:
y, sr = sf.read(_p(r['path']), dtype="float32")
hop = int(round(HOP * sr / 22050)); nf = 4 * hop
rms = librosa.feature.rms(y=y, frame_length=nf, hop_length=hop)[0]
db = 20 * np.log10(rms / max(rms.max(), 1e-9) + 1e-9)
live = np.where(db > -35)[0]
ends = [int(live[-1]) + 1]
if r.get("multi"):
quiet = db <= -35; t = int(live[0])
while t < ends[0]:
if quiet[t]:
u = t
while u < ends[0] and quiet[u]: u += 1
if u - t >= 13: ends.append(t)
t = u
else: t += 1
res, bres = [], []
for e in ends:
a = max(0, e - 26); a0 = max(0, a - 12); e0 = min(len(db), e + 12)
seg = y[a0 * hop: e0 * hop + nf].astype(np.float64)
# the 110 Hz floor was fitted on a ~200 Hz female voice; for male voices it would read low pitch as creak (70 Hz floor)
fmin = r.get("fmin", 110)
f0, _, _ = librosa.pyin(seg, fmin=fmin, fmax=700, sr=sr, frame_length=nf, hop_length=hop, fill_na=np.nan)
cen = librosa.feature.spectral_centroid(y=seg.astype(np.float32), sr=sr, n_fft=nf, hop_length=hop)[0]
n = min(len(f0), len(cen), e0 - a0); f0 = f0[:n].copy(); cen = cen[:n]; d = db[a0:a0 + n]
f0[d < -35] = np.nan
rough = ((d > -30) & (cen < 2500) & np.isnan(f0))[a - a0: e - a0]
run = best = 0
for q in rough:
run = run + 1 if q else 0; best = max(best, run)
res.append(round(best * 11.61))
# second, bench-style check (male voices; recorded only, keep.py does not drop on it): on the same last-300-ms window at 16 kHz, 5 ms hops,
# loud vowel-band sound (80-1000 Hz within 20 dB of the window's loudest) where autocorrelation finds no pitch
# (nac <= 0.5 over lags 40-200 = 80-400 Hz) -> longest run in ms
w16 = librosa.resample(y[a * hop: e * hop + nf].astype(np.float32), orig_sr=sr, target_sr=16000)
bm = 0
nfr = max(0, (len(w16) - 480) // 80 + 1)
if nfr > 0:
idx = np.arange(480)[None, :] + 80 * np.arange(nfr)[:, None]
F = w16[idx] * np.hanning(480)[None, :]
F = F - F.mean(1, keepdims=True)
spec = np.abs(np.fft.rfft(F, 1024, axis=1)) ** 2
ac = np.fft.irfft(spec, axis=1)[:, :240]
nac = ac[:, 40:200].max(1) / (ac[:, 0] + 1e-9)
fq = np.fft.rfftfreq(1024, 1 / 16000)
low = 10 * np.log10(spec[:, (fq >= 80) & (fq <= 1000)].sum(1) + 1e-10)
snd = low > low.max() - 20
rb = snd & ~(nac > 0.5)
run = 0
for q in rb:
run = run + 1 if q else 0; bm = max(bm, run)
bres.append(bm * 5)
out.append(dict(clip=r["clip"], end_ms=res[0], inner_ms=sorted(res[1:], reverse=True), fmin=r.get("fmin", 110),
bench_end_ms=bres[0], bench_inner_ms=sorted(bres[1:], reverse=True)))
except Exception as ex: # noqa: BLE001
out.append(dict(clip=r["clip"], why=repr(ex)[:200]))
d = Path(f"/data/endings/{rows[0]['group']}")
d.mkdir(parents=True, exist_ok=True)
(d / f"{rows[0]['clip']}.jsonl").write_text("".join(json.dumps(x) + "\n" for x in out))
vol.commit()
return len(out)
@app.local_entrypoint()
def ends_main(group: str, per_call: int = 150, tag: str = ""):
root = Path(os.environ.get("KALE_TTS_ROOT", "."))
done = {}
for f in sorted((root / f"work/render-{group}").glob("chunk-*.json")):
for t in json.loads(f.read_text()):
if t.get("path"):
done[t["clip"]] = t
desc = {json.loads(l)["clip"]: json.loads(l)["description"] for l in (root / f"work/jobs-{group}.jsonl").read_text().splitlines() if l.strip()}
male = lambda c: " male voice" in desc.get(c, "") and "female" not in desc.get(c, "")
have = set(endings_rows.remote([group + tag])) # resume: skip takes already measured (v2 volume)
rows = [dict(clip=c, path=t["path"], group=group + tag, multi=len(re.findall(r"[.!?](\s|$)", t["text"])) > 1, fmin=70 if male(c) else 110)
for c, t in done.items() if c not in have]
print(group, len(rows), "still to check,", len(have), "done", flush=True)
n = sum(endings.map([rows[i:i + per_call] for i in range(0, len(rows), per_call)], order_outputs=False))
print(group, n, "takes measured", flush=True)
@app.function(image=img_keep, cpu=1, memory=4096, timeout=600, volumes={"/data": vol})
def endings_rows(groups: list):
vol.reload()
out = {}
for g in groups:
for f in Path(f"/data/endings/{g}").glob("*.jsonl") if Path(f"/data/endings/{g}").exists() else []:
for l in f.read_text().splitlines():
x = json.loads(l)
out[x["clip"]] = x
return out