# SPDX-License-Identifier: Apache-2.0 # Copyright 2026 Kale """Streaming render + screen (16 Nov 2026), the RECOMMENDED pipeline shape: one warm container per GPU loads Maya-1 (vLLM 0.33.0 bf16, same prompt and sampling as render.py), SNAC, Whisper large-v3-turbo, UTMOS22 strong and ECAPA ONCE, then takes calls of up to ~25 designs: renders their takes in one vLLM batch and scores every take in the same container, so nothing waits for a whole round: read-back = Whisper large-v3-turbo, missed / extra words on the `spoken` normalisation (copy of filter.py::measure) creak = the rough-ending measure on EVERY sentence end (copy of filter.py::endings; 70 Hz pitch floor for men, 110 for women) utmos, ECAPA-TDNN embedding (speechbrain/spkrec-ecapa-voxceleb, eval only), seconds, peak, wpm = words / full clip seconds Clean take = Maya end token, <= 1 missed word, peak < 0.999, every sentence end < 100 ms rough (bench-style check not used: it flags every Maya male end). Returns per design: scores (unclear, weakest_tenth, utmos, creak), clean count, ECAPA centroid, the clean take nearest its target wpm as float WAV bytes. Every take's wav is kept on the data volume as ONE tar per call (/stream//.tar). Two lines per design: the fixed test line (config/stream.json "test_line", the volcano line; comparable across every voice; creak rule, pick nearest the target wpm) and the design's own line. ~100 designs screened end to end in ~5 minutes on 12 H100s. As written, this screens voice DESIGNS (step 1 of the guide; then select_stream.py). The release's training data went through the batch path (../render/render.py -> filter.py -> keep.py -> strict.py -> final.py), which applies the same checks one after another; streaming those checks per training chunk is the recommended shape for new data (see README, known gaps). modal deploy stream_filter.py python stream_filter.py [--tv 6] [--tl 6] [--per-call 12] [--seed0 7000000] Input: /design/candidates-.json (list of designs: voice, description, gender, line, target_wpm, wpm_lo, wpm_hi, creak_exempt, label, ...; example: config/stream_designs.example.json). Output: /design/stream-.jsonl (one row per design as each call lands) + pick wavs /design/raw/.wav and _line.wav. root = env KALE_TTS_ROOT or ".".""" import io import json import os import re import tarfile import time from pathlib import Path import modal MAYA, MAYA_REV = "maya-research/maya1", "b1b9b38ca9decbfcfd6c98ea93dd502bfd0f9572" SNAC_ID, SNAC_REV = "hubertsiuzdak/snac_24khz", "d73ad176a12188fcf4f360ba3bf2c2fbbe8f58ec" ASR = "openai/whisper-large-v3-turbo" CODE_START, CODE_END, CODE_OFFSET = 128257, 128258, 128266 SNAC_MIN, SNAC_MAX = 128266, 156937 SOH, EOH, SOA, EOT = 128259, 128260, 128261, 128009 WORDS = 40 def _cfg(name, default): """recipe config: env KALE_TTS__CONFIG, then ../config/.json (modal.json) or ./config/.json, then the copy in the image""" here = Path(__file__).resolve().parent for p in (os.environ.get(f"KALE_TTS_{name.upper()}_CONFIG"), here.parent / "config" / f"{name}.json", here / "config" / f"{name}.json", Path("/root/recipe") / f"{name}.json"): if p and Path(p).is_file(): return {**default, **json.loads(Path(p).read_text())} return default NAMES = _cfg("modal", {"stream_app": "kale-tts-stream", "data_volume": "kale-tts-data"}) CFG = _cfg("stream", {"gpus": 12}) app = modal.App(NAMES["stream_app"]) vol = modal.Volume.from_name(NAMES["data_volume"], create_if_missing=True) def _fetch(): from huggingface_hub import snapshot_download snapshot_download(MAYA, revision=MAYA_REV, allow_patterns=["*.json", "*.safetensors", "*.jinja", "*.txt"]) snapshot_download(SNAC_ID, revision=SNAC_REV) from transformers import WhisperForConditionalGeneration as W, WhisperProcessor as P P.from_pretrained(ASR); W.from_pretrained(ASR) import torch torch.hub.load("tarepan/SpeechMOS:v1.2.0", "utmos22_strong", trust_repo=True) import torchaudio if not hasattr(torchaudio, "list_audio_backends"): torchaudio.list_audio_backends = lambda: ["soundfile"] # speechbrain 1.0.3 on the torch vLLM 0.33 pulls in import huggingface_hub as _hh _o = _hh.hf_hub_download _hh.hf_hub_download = lambda *a, use_auth_token=None, **k: _o(*a, **k) # speechbrain 1.0.3 + new hf hub from speechbrain.inference.speaker import EncoderClassifier as E snapshot_download("speechbrain/spkrec-ecapa-voxceleb", local_dir="/opt/ecapa_src") # local source: speechbrain 1.0.3 + new hf hub E.from_hparams(source="/opt/ecapa_src", savedir="/opt/ecapa") img = (modal.Image.debian_slim(python_version="3.11").apt_install("libsndfile1", "git") .pip_install("vllm==0.33.0", "snac==1.2.1", "soundfile", "huggingface_hub", "ninja", "nvidia-cuda-nvcc==13.0.*", "librosa==0.10.2", "speechbrain==1.0.3", "num2words", "scipy") .env({"VLLM_WORKER_MULTIPROC_METHOD": "spawn", "VLLM_USE_FLASHINFER_SAMPLER": "0", "MAYA_VLLM_ATTN": "FLASH_ATTN", "HF_HOME": "/opt/hf", "TORCH_HOME": "/opt/torch", "CUDA_HOME": "/usr/local/lib/python3.11/site-packages/nvidia/cu13", "PATH": "/usr/local/lib/python3.11/site-packages/nvidia/cu13/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"}) .run_function(_fetch)) for _n in ("modal", "stream"): # the containers read the same names and settings for _p in (Path(__file__).resolve().parent.parent / "config" / f"{_n}.json", Path(__file__).resolve().parent / "config" / f"{_n}.json"): if _p.is_file(): img = img.add_local_file(str(_p), f"/root/recipe/{_n}.json") break def spoken(word): from num2words import num2words t = re.sub(r"(\d+),(\d{3})", r"\1\2", word) t = t.replace("%", " percent ").replace("$", " dollars ").replace("&", " and ") t = re.sub(r"\d+(\.\d+)?", lambda m: " " + num2words(float(m.group()) if "." in m.group() else int(m.group())) + " ", t) t = t.lower().replace("-", " ").replace("’", "'") return [w.strip("'") for w in re.sub(r"[^a-z' ]", " ", t).split() if w.strip("'")] def _unpack(tok): n = len(tok) // 7 l1, l2, l3 = [], [], [] for i in range(n): s = [(t - CODE_OFFSET) % 4096 for t in tok[i * 7:(i + 1) * 7]] l1.append(s[0]); l2 += [s[1], s[4]]; l3 += [s[2], s[3], s[5], s[6]] return [l1, l2, l3] def endings(y, sr, fmin, multi=True): """filter.py::endings, the rough-ending measure only, on one array: rough ms per sentence end (last end first).""" import librosa import numpy as np HOP = 256 hop = int(round(HOP * sr / 22050)); nf = 4 * hop rms = librosa.feature.rms(y=y, frame_length=nf, hop_length=hop)[0] db = 20 * np.log10(rms / max(rms.max(), 1e-9) + 1e-9) live = np.where(db > -35)[0] ends = [int(live[-1]) + 1] if multi: quiet = db <= -35; t = int(live[0]) while t < ends[0]: if quiet[t]: u = t while u < ends[0] and quiet[u]: u += 1 if u - t >= 13: ends.append(t) t = u else: t += 1 res = [] for e in ends: a = max(0, e - 26); a0 = max(0, a - 12); e0 = min(len(db), e + 12) seg = y[a0 * hop: e0 * hop + nf].astype(np.float64) f0, _, _ = librosa.pyin(seg, fmin=fmin, fmax=700, sr=sr, frame_length=nf, hop_length=hop, fill_na=np.nan) cen = librosa.feature.spectral_centroid(y=seg.astype(np.float32), sr=sr, n_fft=nf, hop_length=hop)[0] n = min(len(f0), len(cen), e0 - a0); f0 = f0[:n].copy(); cen = cen[:n]; d = db[a0:a0 + n] f0[d < -35] = np.nan rough = ((d > -30) & (cen < 2500) & np.isnan(f0))[a - a0: e - a0] run = best = 0 for q in rough: run = run + 1 if q else 0; best = max(best, run) res.append(round(best * 11.61)) return res @app.cls(image=img, gpu="H100", cpu=8, memory=65536, timeout=2400, max_containers=int(CFG["gpus"]), scaledown_window=120, retries=modal.Retries(max_retries=3, initial_delay=2.0), volumes={"/data": vol}) class Stream: @modal.enter() def load(self): t0 = time.time() from transformers import AutoTokenizer from vllm import LLM self.tok = AutoTokenizer.from_pretrained(MAYA, revision=MAYA_REV) self.llm = LLM(MAYA, revision=MAYA_REV, dtype="bfloat16", gpu_memory_utilization=0.55, max_model_len=2400, seed=1, attention_backend=os.environ.get("MAYA_VLLM_ATTN", "FLASH_ATTN")) # first, before anything touches CUDA import torch from snac import SNAC import torchaudio if not hasattr(torchaudio, "list_audio_backends"): torchaudio.list_audio_backends = lambda: ["soundfile"] # speechbrain 1.0.3 on the torch vLLM 0.33 pulls in import huggingface_hub as _hh _o = _hh.hf_hub_download _hh.hf_hub_download = lambda *a, use_auth_token=None, **k: _o(*a, **k) from speechbrain.inference.speaker import EncoderClassifier from transformers import AutoTokenizer, WhisperForConditionalGeneration, WhisperProcessor from vllm import LLM self.snac = SNAC.from_pretrained(SNAC_ID, revision=SNAC_REV).eval().to("cuda") self.proc = WhisperProcessor.from_pretrained(ASR) self.wm = WhisperForConditionalGeneration.from_pretrained(ASR, torch_dtype=torch.float16).to("cuda").eval() self.um = torch.hub.load("tarepan/SpeechMOS:v1.2.0", "utmos22_strong", trust_repo=True).cuda().eval() self.ec = EncoderClassifier.from_hparams(source="/opt/ecapa_src", savedir="/opt/ecapa", run_opts={"device": "cuda"}) self.load_s = round(time.time() - t0, 1) def _render(self, jobs, seed0): import numpy as np import torch from vllm import SamplingParams d = lambda i: self.tok.decode([i]) prompt = lambda j: d(SOH) + self.tok.bos_token + f' {j["text"]}' + d(EOT) + d(EOH) + d(SOA) + d(CODE_START) sps = [SamplingParams(temperature=0.4, top_p=0.9, repetition_penalty=1.1, max_tokens=min(2048, 260 + 75 * len(j["text"].split())), min_tokens=28, stop_token_ids=[CODE_END], seed=seed0 + i) for i, j in enumerate(jobs)] ps = [{"prompt_token_ids": self.tok(prompt(j), add_special_tokens=False)["input_ids"]} for j in jobs] outs = self.llm.generate(ps, sps, use_tqdm=False) # on a warm engine the outputs did not come back in prompt order (lines landed on other designs). Match each output to a # job by its prompt token ids; takes of one design + line share a prompt, so they are interchangeable (seed = that job's seed). pool = {} for o in outs: pool.setdefault(tuple(o.prompt_token_ids), []).append(o) outs = [pool[tuple(p["prompt_token_ids"])].pop() for p in ps] res = [] for i, (j, o) in enumerate(zip(jobs, outs)): row = list(o.outputs[0].token_ids) eos = o.outputs[0].finish_reason == "stop" or CODE_END in row row = row[:row.index(CODE_END)] if CODE_END in row else row codes = [x for x in row if SNAC_MIN <= x <= SNAC_MAX] if len(codes) < 28: res.append((j, None, False, seed0 + i)); continue lv = [torch.tensor(l, dtype=torch.long, device="cuda").unsqueeze(0) for l in _unpack(codes)] with torch.inference_mode(): a = self.snac.decoder(self.snac.quantizer.from_codes(lv))[0, 0].float().cpu().numpy() res.append((j, a.astype(np.float32), bool(eos), seed0 + i)) return res def _score(self, designs, takes, tag, call): """takes: list of (job, audio 24 kHz float32 or None, eos, seed); job["line"] = "V" (volcano line) or "L" (the voice's own line)""" import difflib import librosa import numpy as np import soundfile as sf import torch unit = lambda a: a / np.linalg.norm(a) rows = [] live = [(j, a, eos, s) for j, a, eos, s in takes if a is not None] plain = lambda t: re.sub(r"<[a-z_]+>", " ", t) heard = [] for b in range(0, len(live), 32): part = live[b:b + 32] y16s = [librosa.resample(a, orig_sr=24000, target_sr=16000, res_type="soxr_hq") for _, a, _, _ in part] feats = self.proc(y16s, sampling_rate=16000, return_tensors="pt").input_features.to("cuda", torch.float16) with torch.no_grad(): g = self.wm.generate(feats, language="en", task="transcribe", num_beams=1, do_sample=False) hs = self.proc.tokenizer.batch_decode(g, skip_special_tokens=True) heard += hs for (j, a, eos, s), y16 in zip(part, y16s): with torch.inference_mode(): t = torch.from_numpy(y16).cuda().unsqueeze(0) mos = float(self.um(t, 16000)[0]) e = unit(self.ec.encode_batch(t)[0, 0].float().cpu().numpy()) rows.append(dict(job=j, a=a, eos=eos, seed=s, utmos=round(mos, 3), emb=e)) dz = {d["voice"]: d for d in designs} for r, h in zip(rows, heard): j = r["job"]; d = dz[j["voice"]] a_ = [t for w in plain(j["text"]).split() for t in spoken(w)] hb = [t for w in h.split() for t in spoken(w)] miss, extra = 0, 0 for op, i0, i1, j0, j1 in difflib.SequenceMatcher(None, a_, hb, autojunk=False).get_opcodes(): if op in ("replace", "delete"): miss += i1 - i0 if op in ("replace", "insert"): extra += j1 - j0 try: ends = endings(r["a"], 24000, 70 if d["gender"] == "male" else 110) except Exception: # noqa: BLE001 ends = [999] sec = len(r["a"]) / 24000 r.update(missed=miss, extra=extra, words=len(a_), heard=h.strip(), ends=len(ends), creak_ends=sum(x >= 100 for x in ends), rough_ms=max(ends), seconds=round(sec, 2), wpm_full=round(len(a_) / sec * 60, 1), peak=round(float(np.abs(r["a"]).max()), 4)) ok = bool(r["eos"]) and r["peak"] < 0.999 and miss <= (1 if j["line"] == "V" else max(2, len(a_) // 12)) # creak rule on the volcano line, every sentence end; whispered / breathy-by-design voices have no pitch anywhere -> exempt r["clean"] = ok and (d.get("creak_exempt") or j["line"] == "L" or r["rough_ms"] < 100) tp = Path(f"/data/stream/{tag}/{call:04d}.tar"); tp.parent.mkdir(parents=True, exist_ok=True) with tarfile.open(tp, "w") as tf: # every take kept, one tar per call (volumes have a file-count limit) for r in rows: buf = io.BytesIO(); sf.write(buf, r["a"], 24000, format="WAV", subtype="FLOAT"); b_ = buf.getvalue() ti = tarfile.TarInfo(f"{r['job']['clip']}.wav"); ti.size = len(b_); tf.addfile(ti, io.BytesIO(b_)) meta = json.dumps([{k: v for k, v in r.items() if k not in ("a", "emb", "job")} | dict(clip=r["job"]["clip"]) for r in rows]).encode() ti = tarfile.TarInfo("takes.json"); ti.size = len(meta); tf.addfile(ti, io.BytesIO(meta)) vol.commit() wav = lambda r: (lambda b: (sf.write(b, r["a"], 24000, format="WAV", subtype="FLOAT"), b.getvalue())[1])(io.BytesIO()) slim = lambda r: {k: v for k, v in r.items() if k not in ("a", "emb", "job")} | dict(clip=r["job"]["clip"]) out = [] for dsg in designs: RV = [r for r in rows if r["job"]["voice"] == dsg["voice"] and r["job"]["line"] == "V"] RL = [r for r in rows if r["job"]["voice"] == dsg["voice"] and r["job"]["line"] == "L"] if not RV: continue cv, cl = [r for r in RV if r["clean"]], [r for r in RL if r["clean"]] ev = [r for r in RV if r["eos"]] or RV us = sorted(r["utmos"] for r in RV) C = unit(np.mean([r["emb"] for r in RV], 0)) lo, hi = dsg.get("wpm_lo", 0), dsg.get("wpm_hi", 999) # prefer clean takes inside the design's pace window (we aim at 160-200) inw = lambda r: lo <= r["wpm_full"] <= hi bv = min([r for r in cv if inw(r)] or cv or RV, key=lambda r: (abs(r["wpm_full"] - dsg["target_wpm"]) / 10 - r["utmos"])) bl = max([r for r in cl if inw(r)] or cl or RL, key=lambda r: r["utmos"] - abs(r["wpm_full"] - dsg["target_wpm"]) / 25) if RL else None wpms = sorted(r["wpm_full"] for r in ev) out.append(dict(dsg, takes=len(RV), clean_takes=len(cv), line_takes=len(RL), line_clean=len(cl), cent=C.tolist(), scores=dict(unclear=round(100 * sum(r["missed"] + r["extra"] for r in ev) / max(1, sum(r["words"] for r in ev)), 2), weakest_tenth=round(float(np.mean(us[:max(1, round(len(us) * 0.1))])), 3), utmos=round(float(np.mean(us)), 3), pitch_step=None, creak=round(100 * sum(r["creak_ends"] for r in RV) / max(1, sum(r["ends"] for r in RV)), 1)), clean_in_window=sum(inw(r) for r in cv), line_clean_in_window=sum(inw(r) for r in cl), wpm_median=wpms[len(wpms) // 2], wpm_min=wpms[0], wpm_max=wpms[-1], pick=slim(bv), wav=wav(bv), line_pick=slim(bl) if bl else None, line_wav=wav(bl) if bl else None, tar=str(tp))) return out @modal.method() def run(self, designs: list, tv: int, tl: int, seed0: int, tag: str, call: int, text: str): t0 = time.time() jobs = [dict(clip=f"{d['voice']}_V{i:02d}", voice=d["voice"], line="V", text=text, description=d["description"]) for d in designs for i in range(tv)] jobs += [dict(clip=f"{d['voice']}_L{i:02d}", voice=d["voice"], line="L", text=d["line"], description=d["description"]) for d in designs for i in range(tl)] rend = self._render(jobs, seed0) t1 = time.time() out = self._score(designs, rend, tag, call) return dict(call=call, voices=out, load_s=self.load_s, render_s=round(t1 - t0, 1), score_s=round(time.time() - t1, 1)) T = CFG.get("test_line", "A volcano is a place where very hot, melted rock comes up out of the ground. Ash and smoky gases can come out too. " "Deep under the ground, the melted rock waits in a big pool called a magma chamber.") def drive(group: str, tv: int = 6, tl: int = 6, per_call: int = 12, seed0: int = 7_000_000): """client side (against the deployed app): streams one row per design to design/stream-.jsonl (pick wavs in design/raw/.wav and _line.wav) as each call lands""" root = Path(os.environ.get("KALE_TTS_ROOT", ".")) designs = json.loads((root / f"design/candidates-{group}.json").read_text()) designs = designs["designs"] if isinstance(designs, dict) else designs # {"_license", "designs": [...]} or a plain list outp = root / f"design/stream-{group}.jsonl" done = {json.loads(l)["voice"] for l in outp.read_text().splitlines()} if outp.exists() else set() designs = [d for d in designs if d["voice"] not in done] parts = [designs[i:i + per_call] for i in range(0, len(designs), per_call)] S = modal.Cls.from_name(NAMES["stream_app"], "Stream")() calls = S.run.starmap([(p, tv, tl, seed0 + 1000 * k, group, k, T) for k, p in enumerate(parts)], order_outputs=False, return_exceptions=True) (root / "design/raw").mkdir(parents=True, exist_ok=True) for r in calls: if isinstance(r, Exception): print("CALL FAILED", repr(r)[:300], flush=True); continue with outp.open("a") as f: for o in r["voices"]: p = root / f"design/raw/{o['voice']}.wav"; p.write_bytes(o.pop("wav")); o["raw"] = str(p) lw = o.pop("line_wav") if lw: p = root / f"design/raw/{o['voice']}_line.wav"; p.write_bytes(lw); o["raw_line"] = str(p) f.write(json.dumps(o) + "\n") print(f"{time.strftime('%H:%M')} call {r['call']}: {len(r['voices'])} voices, load {r['load_s']} s, render {r['render_s']} s, score {r['score_s']} s", flush=True) if __name__ == "__main__": # was scripts/drive.py import argparse ap = argparse.ArgumentParser(); ap.add_argument("group"); ap.add_argument("--tv", type=int, default=CFG.get("takes_test_line", 6)) ap.add_argument("--tl", type=int, default=CFG.get("takes_own_line", 6)); ap.add_argument("--per-call", type=int, default=CFG.get("designs_per_call", 12)) ap.add_argument("--seed0", type=int, default=CFG.get("seed0", 7_000_000)) a = ap.parse_args() drive(a.group, a.tv, a.tl, a.per_call, a.seed0) print("DRIVE-DONE", a.group, flush=True)