kale-tts-1-tiny / recipe /filter /stream_filter.py
NERDDISCO's picture
kale-tts-1
f21d6eb
Raw History Blame Contribute Delete
20.9 kB
# SPDX-License-Identifier: Apache-2.0
# Copyright 2026 Kale
"""Streaming render + screen (16 Nov 2026), the RECOMMENDED pipeline shape: one warm container per GPU loads Maya-1 (vLLM 0.33.0 bf16,
same prompt and sampling as render.py), SNAC, Whisper large-v3-turbo, UTMOS22 strong and ECAPA ONCE, then takes calls of up to ~25
designs: renders their takes in one vLLM batch and scores every take in the same container, so nothing waits for a whole round:
read-back = Whisper large-v3-turbo, missed / extra words on the `spoken` normalisation (copy of filter.py::measure)
creak = the rough-ending measure on EVERY sentence end (copy of filter.py::endings; 70 Hz pitch floor for men, 110 for women)
utmos, ECAPA-TDNN embedding (speechbrain/spkrec-ecapa-voxceleb, eval only), seconds, peak, wpm = words / full clip seconds
Clean take = Maya end token, <= 1 missed word, peak < 0.999, every sentence end < 100 ms rough (bench-style check not used: it flags
every Maya male end). Returns per design: scores (unclear, weakest_tenth, utmos, creak), clean count, ECAPA centroid, the clean take
nearest its target wpm as float WAV bytes. Every take's wav is kept on the data volume as ONE tar per call (/stream/<tag>/<call>.tar).
Two lines per design: the fixed test line (config/stream.json "test_line", the volcano line; comparable across every voice; creak rule,
pick nearest the target wpm) and the design's own line. ~100 designs screened end to end in ~5 minutes on 12 H100s.
As written, this screens voice DESIGNS (step 1 of the guide; then select_stream.py). The release's training data went through the
batch path (../render/render.py -> filter.py -> keep.py -> strict.py -> final.py), which applies the same checks one after another;
streaming those checks per training chunk is the recommended shape for new data (see README, known gaps).
modal deploy stream_filter.py
python stream_filter.py <group> [--tv 6] [--tl 6] [--per-call 12] [--seed0 7000000]
Input: <root>/design/candidates-<group>.json (list of designs: voice, description, gender, line, target_wpm, wpm_lo, wpm_hi,
creak_exempt, label, ...; example: config/stream_designs.example.json). Output: <root>/design/stream-<group>.jsonl (one row per design
as each call lands) + pick wavs <root>/design/raw/<voice>.wav and <voice>_line.wav. root = env KALE_TTS_ROOT or "."."""
import io
import json
import os
import re
import tarfile
import time
from pathlib import Path
import modal
MAYA, MAYA_REV = "maya-research/maya1", "b1b9b38ca9decbfcfd6c98ea93dd502bfd0f9572"
SNAC_ID, SNAC_REV = "hubertsiuzdak/snac_24khz", "d73ad176a12188fcf4f360ba3bf2c2fbbe8f58ec"
ASR = "openai/whisper-large-v3-turbo"
CODE_START, CODE_END, CODE_OFFSET = 128257, 128258, 128266
SNAC_MIN, SNAC_MAX = 128266, 156937
SOH, EOH, SOA, EOT = 128259, 128260, 128261, 128009
WORDS = 40
def _cfg(name, default):
"""recipe config: env KALE_TTS_<NAME>_CONFIG, then ../config/<name>.json (modal.json) or ./config/<name>.json, then the copy in the image"""
here = Path(__file__).resolve().parent
for p in (os.environ.get(f"KALE_TTS_{name.upper()}_CONFIG"), here.parent / "config" / f"{name}.json", here / "config" / f"{name}.json",
Path("/root/recipe") / f"{name}.json"):
if p and Path(p).is_file():
return {**default, **json.loads(Path(p).read_text())}
return default
NAMES = _cfg("modal", {"stream_app": "kale-tts-stream", "data_volume": "kale-tts-data"})
CFG = _cfg("stream", {"gpus": 12})
app = modal.App(NAMES["stream_app"])
vol = modal.Volume.from_name(NAMES["data_volume"], create_if_missing=True)
def _fetch():
from huggingface_hub import snapshot_download
snapshot_download(MAYA, revision=MAYA_REV, allow_patterns=["*.json", "*.safetensors", "*.jinja", "*.txt"])
snapshot_download(SNAC_ID, revision=SNAC_REV)
from transformers import WhisperForConditionalGeneration as W, WhisperProcessor as P
P.from_pretrained(ASR); W.from_pretrained(ASR)
import torch
torch.hub.load("tarepan/SpeechMOS:v1.2.0", "utmos22_strong", trust_repo=True)
import torchaudio
if not hasattr(torchaudio, "list_audio_backends"):
torchaudio.list_audio_backends = lambda: ["soundfile"] # speechbrain 1.0.3 on the torch vLLM 0.33 pulls in
import huggingface_hub as _hh
_o = _hh.hf_hub_download
_hh.hf_hub_download = lambda *a, use_auth_token=None, **k: _o(*a, **k) # speechbrain 1.0.3 + new hf hub
from speechbrain.inference.speaker import EncoderClassifier as E
snapshot_download("speechbrain/spkrec-ecapa-voxceleb", local_dir="/opt/ecapa_src") # local source: speechbrain 1.0.3 + new hf hub
E.from_hparams(source="/opt/ecapa_src", savedir="/opt/ecapa")
img = (modal.Image.debian_slim(python_version="3.11").apt_install("libsndfile1", "git")
.pip_install("vllm==0.33.0", "snac==1.2.1", "soundfile", "huggingface_hub", "ninja", "nvidia-cuda-nvcc==13.0.*", "librosa==0.10.2",
"speechbrain==1.0.3", "num2words", "scipy")
.env({"VLLM_WORKER_MULTIPROC_METHOD": "spawn", "VLLM_USE_FLASHINFER_SAMPLER": "0", "MAYA_VLLM_ATTN": "FLASH_ATTN", "HF_HOME": "/opt/hf", "TORCH_HOME": "/opt/torch",
"CUDA_HOME": "/usr/local/lib/python3.11/site-packages/nvidia/cu13",
"PATH": "/usr/local/lib/python3.11/site-packages/nvidia/cu13/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"})
.run_function(_fetch))
for _n in ("modal", "stream"): # the containers read the same names and settings
for _p in (Path(__file__).resolve().parent.parent / "config" / f"{_n}.json", Path(__file__).resolve().parent / "config" / f"{_n}.json"):
if _p.is_file():
img = img.add_local_file(str(_p), f"/root/recipe/{_n}.json")
break
def spoken(word):
from num2words import num2words
t = re.sub(r"(\d+),(\d{3})", r"\1\2", word)
t = t.replace("%", " percent ").replace("$", " dollars ").replace("&", " and ")
t = re.sub(r"\d+(\.\d+)?", lambda m: " " + num2words(float(m.group()) if "." in m.group() else int(m.group())) + " ", t)
t = t.lower().replace("-", " ").replace("’", "'")
return [w.strip("'") for w in re.sub(r"[^a-z' ]", " ", t).split() if w.strip("'")]
def _unpack(tok):
n = len(tok) // 7
l1, l2, l3 = [], [], []
for i in range(n):
s = [(t - CODE_OFFSET) % 4096 for t in tok[i * 7:(i + 1) * 7]]
l1.append(s[0]); l2 += [s[1], s[4]]; l3 += [s[2], s[3], s[5], s[6]]
return [l1, l2, l3]
def endings(y, sr, fmin, multi=True):
"""filter.py::endings, the rough-ending measure only, on one array: rough ms per sentence end (last end first)."""
import librosa
import numpy as np
HOP = 256
hop = int(round(HOP * sr / 22050)); nf = 4 * hop
rms = librosa.feature.rms(y=y, frame_length=nf, hop_length=hop)[0]
db = 20 * np.log10(rms / max(rms.max(), 1e-9) + 1e-9)
live = np.where(db > -35)[0]
ends = [int(live[-1]) + 1]
if multi:
quiet = db <= -35; t = int(live[0])
while t < ends[0]:
if quiet[t]:
u = t
while u < ends[0] and quiet[u]: u += 1
if u - t >= 13: ends.append(t)
t = u
else: t += 1
res = []
for e in ends:
a = max(0, e - 26); a0 = max(0, a - 12); e0 = min(len(db), e + 12)
seg = y[a0 * hop: e0 * hop + nf].astype(np.float64)
f0, _, _ = librosa.pyin(seg, fmin=fmin, fmax=700, sr=sr, frame_length=nf, hop_length=hop, fill_na=np.nan)
cen = librosa.feature.spectral_centroid(y=seg.astype(np.float32), sr=sr, n_fft=nf, hop_length=hop)[0]
n = min(len(f0), len(cen), e0 - a0); f0 = f0[:n].copy(); cen = cen[:n]; d = db[a0:a0 + n]
f0[d < -35] = np.nan
rough = ((d > -30) & (cen < 2500) & np.isnan(f0))[a - a0: e - a0]
run = best = 0
for q in rough:
run = run + 1 if q else 0; best = max(best, run)
res.append(round(best * 11.61))
return res
@app.cls(image=img, gpu="H100", cpu=8, memory=65536, timeout=2400, max_containers=int(CFG["gpus"]), scaledown_window=120,
retries=modal.Retries(max_retries=3, initial_delay=2.0), volumes={"/data": vol})
class Stream:
@modal.enter()
def load(self):
t0 = time.time()
from transformers import AutoTokenizer
from vllm import LLM
self.tok = AutoTokenizer.from_pretrained(MAYA, revision=MAYA_REV)
self.llm = LLM(MAYA, revision=MAYA_REV, dtype="bfloat16", gpu_memory_utilization=0.55, max_model_len=2400, seed=1,
attention_backend=os.environ.get("MAYA_VLLM_ATTN", "FLASH_ATTN")) # first, before anything touches CUDA
import torch
from snac import SNAC
import torchaudio
if not hasattr(torchaudio, "list_audio_backends"):
torchaudio.list_audio_backends = lambda: ["soundfile"] # speechbrain 1.0.3 on the torch vLLM 0.33 pulls in
import huggingface_hub as _hh
_o = _hh.hf_hub_download
_hh.hf_hub_download = lambda *a, use_auth_token=None, **k: _o(*a, **k)
from speechbrain.inference.speaker import EncoderClassifier
from transformers import AutoTokenizer, WhisperForConditionalGeneration, WhisperProcessor
from vllm import LLM
self.snac = SNAC.from_pretrained(SNAC_ID, revision=SNAC_REV).eval().to("cuda")
self.proc = WhisperProcessor.from_pretrained(ASR)
self.wm = WhisperForConditionalGeneration.from_pretrained(ASR, torch_dtype=torch.float16).to("cuda").eval()
self.um = torch.hub.load("tarepan/SpeechMOS:v1.2.0", "utmos22_strong", trust_repo=True).cuda().eval()
self.ec = EncoderClassifier.from_hparams(source="/opt/ecapa_src", savedir="/opt/ecapa", run_opts={"device": "cuda"})
self.load_s = round(time.time() - t0, 1)
def _render(self, jobs, seed0):
import numpy as np
import torch
from vllm import SamplingParams
d = lambda i: self.tok.decode([i])
prompt = lambda j: d(SOH) + self.tok.bos_token + f'<description="{j["description"]}"> {j["text"]}' + d(EOT) + d(EOH) + d(SOA) + d(CODE_START)
sps = [SamplingParams(temperature=0.4, top_p=0.9, repetition_penalty=1.1, max_tokens=min(2048, 260 + 75 * len(j["text"].split())),
min_tokens=28, stop_token_ids=[CODE_END], seed=seed0 + i) for i, j in enumerate(jobs)]
ps = [{"prompt_token_ids": self.tok(prompt(j), add_special_tokens=False)["input_ids"]} for j in jobs]
outs = self.llm.generate(ps, sps, use_tqdm=False)
# on a warm engine the outputs did not come back in prompt order (lines landed on other designs). Match each output to a
# job by its prompt token ids; takes of one design + line share a prompt, so they are interchangeable (seed = that job's seed).
pool = {}
for o in outs:
pool.setdefault(tuple(o.prompt_token_ids), []).append(o)
outs = [pool[tuple(p["prompt_token_ids"])].pop() for p in ps]
res = []
for i, (j, o) in enumerate(zip(jobs, outs)):
row = list(o.outputs[0].token_ids)
eos = o.outputs[0].finish_reason == "stop" or CODE_END in row
row = row[:row.index(CODE_END)] if CODE_END in row else row
codes = [x for x in row if SNAC_MIN <= x <= SNAC_MAX]
if len(codes) < 28:
res.append((j, None, False, seed0 + i)); continue
lv = [torch.tensor(l, dtype=torch.long, device="cuda").unsqueeze(0) for l in _unpack(codes)]
with torch.inference_mode():
a = self.snac.decoder(self.snac.quantizer.from_codes(lv))[0, 0].float().cpu().numpy()
res.append((j, a.astype(np.float32), bool(eos), seed0 + i))
return res
def _score(self, designs, takes, tag, call):
"""takes: list of (job, audio 24 kHz float32 or None, eos, seed); job["line"] = "V" (volcano line) or "L" (the voice's own line)"""
import difflib
import librosa
import numpy as np
import soundfile as sf
import torch
unit = lambda a: a / np.linalg.norm(a)
rows = []
live = [(j, a, eos, s) for j, a, eos, s in takes if a is not None]
plain = lambda t: re.sub(r"<[a-z_]+>", " ", t)
heard = []
for b in range(0, len(live), 32):
part = live[b:b + 32]
y16s = [librosa.resample(a, orig_sr=24000, target_sr=16000, res_type="soxr_hq") for _, a, _, _ in part]
feats = self.proc(y16s, sampling_rate=16000, return_tensors="pt").input_features.to("cuda", torch.float16)
with torch.no_grad():
g = self.wm.generate(feats, language="en", task="transcribe", num_beams=1, do_sample=False)
hs = self.proc.tokenizer.batch_decode(g, skip_special_tokens=True)
heard += hs
for (j, a, eos, s), y16 in zip(part, y16s):
with torch.inference_mode():
t = torch.from_numpy(y16).cuda().unsqueeze(0)
mos = float(self.um(t, 16000)[0])
e = unit(self.ec.encode_batch(t)[0, 0].float().cpu().numpy())
rows.append(dict(job=j, a=a, eos=eos, seed=s, utmos=round(mos, 3), emb=e))
dz = {d["voice"]: d for d in designs}
for r, h in zip(rows, heard):
j = r["job"]; d = dz[j["voice"]]
a_ = [t for w in plain(j["text"]).split() for t in spoken(w)]
hb = [t for w in h.split() for t in spoken(w)]
miss, extra = 0, 0
for op, i0, i1, j0, j1 in difflib.SequenceMatcher(None, a_, hb, autojunk=False).get_opcodes():
if op in ("replace", "delete"): miss += i1 - i0
if op in ("replace", "insert"): extra += j1 - j0
try:
ends = endings(r["a"], 24000, 70 if d["gender"] == "male" else 110)
except Exception: # noqa: BLE001
ends = [999]
sec = len(r["a"]) / 24000
r.update(missed=miss, extra=extra, words=len(a_), heard=h.strip(), ends=len(ends), creak_ends=sum(x >= 100 for x in ends), rough_ms=max(ends),
seconds=round(sec, 2), wpm_full=round(len(a_) / sec * 60, 1), peak=round(float(np.abs(r["a"]).max()), 4))
ok = bool(r["eos"]) and r["peak"] < 0.999 and miss <= (1 if j["line"] == "V" else max(2, len(a_) // 12))
# creak rule on the volcano line, every sentence end; whispered / breathy-by-design voices have no pitch anywhere -> exempt
r["clean"] = ok and (d.get("creak_exempt") or j["line"] == "L" or r["rough_ms"] < 100)
tp = Path(f"/data/stream/{tag}/{call:04d}.tar"); tp.parent.mkdir(parents=True, exist_ok=True)
with tarfile.open(tp, "w") as tf: # every take kept, one tar per call (volumes have a file-count limit)
for r in rows:
buf = io.BytesIO(); sf.write(buf, r["a"], 24000, format="WAV", subtype="FLOAT"); b_ = buf.getvalue()
ti = tarfile.TarInfo(f"{r['job']['clip']}.wav"); ti.size = len(b_); tf.addfile(ti, io.BytesIO(b_))
meta = json.dumps([{k: v for k, v in r.items() if k not in ("a", "emb", "job")} | dict(clip=r["job"]["clip"]) for r in rows]).encode()
ti = tarfile.TarInfo("takes.json"); ti.size = len(meta); tf.addfile(ti, io.BytesIO(meta))
vol.commit()
wav = lambda r: (lambda b: (sf.write(b, r["a"], 24000, format="WAV", subtype="FLOAT"), b.getvalue())[1])(io.BytesIO())
slim = lambda r: {k: v for k, v in r.items() if k not in ("a", "emb", "job")} | dict(clip=r["job"]["clip"])
out = []
for dsg in designs:
RV = [r for r in rows if r["job"]["voice"] == dsg["voice"] and r["job"]["line"] == "V"]
RL = [r for r in rows if r["job"]["voice"] == dsg["voice"] and r["job"]["line"] == "L"]
if not RV:
continue
cv, cl = [r for r in RV if r["clean"]], [r for r in RL if r["clean"]]
ev = [r for r in RV if r["eos"]] or RV
us = sorted(r["utmos"] for r in RV)
C = unit(np.mean([r["emb"] for r in RV], 0))
lo, hi = dsg.get("wpm_lo", 0), dsg.get("wpm_hi", 999) # prefer clean takes inside the design's pace window (we aim at 160-200)
inw = lambda r: lo <= r["wpm_full"] <= hi
bv = min([r for r in cv if inw(r)] or cv or RV, key=lambda r: (abs(r["wpm_full"] - dsg["target_wpm"]) / 10 - r["utmos"]))
bl = max([r for r in cl if inw(r)] or cl or RL, key=lambda r: r["utmos"] - abs(r["wpm_full"] - dsg["target_wpm"]) / 25) if RL else None
wpms = sorted(r["wpm_full"] for r in ev)
out.append(dict(dsg, takes=len(RV), clean_takes=len(cv), line_takes=len(RL), line_clean=len(cl), cent=C.tolist(),
scores=dict(unclear=round(100 * sum(r["missed"] + r["extra"] for r in ev) / max(1, sum(r["words"] for r in ev)), 2),
weakest_tenth=round(float(np.mean(us[:max(1, round(len(us) * 0.1))])), 3), utmos=round(float(np.mean(us)), 3),
pitch_step=None, creak=round(100 * sum(r["creak_ends"] for r in RV) / max(1, sum(r["ends"] for r in RV)), 1)),
clean_in_window=sum(inw(r) for r in cv), line_clean_in_window=sum(inw(r) for r in cl), wpm_median=wpms[len(wpms) // 2], wpm_min=wpms[0], wpm_max=wpms[-1], pick=slim(bv), wav=wav(bv),
line_pick=slim(bl) if bl else None, line_wav=wav(bl) if bl else None, tar=str(tp)))
return out
@modal.method()
def run(self, designs: list, tv: int, tl: int, seed0: int, tag: str, call: int, text: str):
t0 = time.time()
jobs = [dict(clip=f"{d['voice']}_V{i:02d}", voice=d["voice"], line="V", text=text, description=d["description"]) for d in designs for i in range(tv)]
jobs += [dict(clip=f"{d['voice']}_L{i:02d}", voice=d["voice"], line="L", text=d["line"], description=d["description"]) for d in designs for i in range(tl)]
rend = self._render(jobs, seed0)
t1 = time.time()
out = self._score(designs, rend, tag, call)
return dict(call=call, voices=out, load_s=self.load_s, render_s=round(t1 - t0, 1), score_s=round(time.time() - t1, 1))
T = CFG.get("test_line", "A volcano is a place where very hot, melted rock comes up out of the ground. Ash and smoky gases can come out too. "
"Deep under the ground, the melted rock waits in a big pool called a magma chamber.")
def drive(group: str, tv: int = 6, tl: int = 6, per_call: int = 12, seed0: int = 7_000_000):
"""client side (against the deployed app): streams one row per design to design/stream-<group>.jsonl (pick wavs in
design/raw/<v>.wav and <v>_line.wav) as each call lands"""
root = Path(os.environ.get("KALE_TTS_ROOT", "."))
designs = json.loads((root / f"design/candidates-{group}.json").read_text())
designs = designs["designs"] if isinstance(designs, dict) else designs # {"_license", "designs": [...]} or a plain list
outp = root / f"design/stream-{group}.jsonl"
done = {json.loads(l)["voice"] for l in outp.read_text().splitlines()} if outp.exists() else set()
designs = [d for d in designs if d["voice"] not in done]
parts = [designs[i:i + per_call] for i in range(0, len(designs), per_call)]
S = modal.Cls.from_name(NAMES["stream_app"], "Stream")()
calls = S.run.starmap([(p, tv, tl, seed0 + 1000 * k, group, k, T) for k, p in enumerate(parts)], order_outputs=False, return_exceptions=True)
(root / "design/raw").mkdir(parents=True, exist_ok=True)
for r in calls:
if isinstance(r, Exception):
print("CALL FAILED", repr(r)[:300], flush=True); continue
with outp.open("a") as f:
for o in r["voices"]:
p = root / f"design/raw/{o['voice']}.wav"; p.write_bytes(o.pop("wav")); o["raw"] = str(p)
lw = o.pop("line_wav")
if lw:
p = root / f"design/raw/{o['voice']}_line.wav"; p.write_bytes(lw); o["raw_line"] = str(p)
f.write(json.dumps(o) + "\n")
print(f"{time.strftime('%H:%M')} call {r['call']}: {len(r['voices'])} voices, load {r['load_s']} s, render {r['render_s']} s, score {r['score_s']} s", flush=True)
if __name__ == "__main__": # was scripts/drive.py
import argparse
ap = argparse.ArgumentParser(); ap.add_argument("group"); ap.add_argument("--tv", type=int, default=CFG.get("takes_test_line", 6))
ap.add_argument("--tl", type=int, default=CFG.get("takes_own_line", 6)); ap.add_argument("--per-call", type=int, default=CFG.get("designs_per_call", 12))
ap.add_argument("--seed0", type=int, default=CFG.get("seed0", 7_000_000))
a = ap.parse_args()
drive(a.group, a.tv, a.tl, a.per_call, a.seed0)
print("DRIVE-DONE", a.group, flush=True)