NERDDISCO's picture
kale-tts-1
f21d6eb
Raw History Blame Contribute Delete
8.88 kB
# SPDX-License-Identifier: Apache-2.0
# Copyright 2026 Kale
"""Maya-1 training-data renderer: the script that rendered every release voice's training takes (16 Nov 2026 build; the same file
rendered the earlier voices v7 and m3).
Input: a jobs.jsonl (rows: clip, voice, text, description[, mood, tagname]; built by make_jobs.py). One Maya-1 prompt per row,
sampling from config/render.json (temperature 0.4, top_p 0.9, repetition penalty 1.1, min 28 / max min(2048, 220 + 60*words) tokens),
one seed per take (seed0 + row index), vLLM 0.33.0 bf16 (`--quant fp8` = vLLM online fp8 weights; the release used bf16).
Outputs on the data volume: /data/onevoice/<voice>/<sub>/<clip>.wav (float32 24 kHz, exactly as generated, uncut) and next to it
<clip>.snac.npy (the take's raw Maya code tokens, uint16, shape (frames, 7), value = token id - 128266; SNAC code = value % 4096; slot
order per frame = Maya's [l1, l2, l3, l3, l2, l3, l3], see _unpack); chunk markers /data/onevoice/_vllm_chunks/<sub>/; locally
<root>/work/render-<group or sub>/chunk-*.json with eos/seconds/seed/raw_peak/path per take (the filter step reads these).
modal run render.py::main --jobs work/jobs-<voice>.jsonl --sub <voice> --group <voice> --per-call 10000 --seed0 <seed0>
Env inside the image: VLLM_USE_FLASHINFER_SAMPLER=0, MAYA_VLLM_ATTN=FLASH_ATTN, CUDA_HOME/PATH to the pip CUDA 13 nvcc. GPU H100,
8 cores, 64 GiB, timeout 40 min per chunk.
Known trap: on a warm vLLM engine outputs may come back out of prompt order. This script makes one fresh engine per call and zips
outputs with jobs (as the release did); stream_filter.py in ../filter matches outputs to jobs by prompt tokens instead."""
import json
import os
import time
from pathlib import Path
import modal
MAYA, MAYA_REV = "maya-research/maya1", "b1b9b38ca9decbfcfd6c98ea93dd502bfd0f9572"
SNAC_ID, SNAC_REV = "hubertsiuzdak/snac_24khz", "d73ad176a12188fcf4f360ba3bf2c2fbbe8f58ec"
CODE_START, CODE_END, CODE_OFFSET = 128257, 128258, 128266
SNAC_MIN, SNAC_MAX = 128266, 156937
SOH, EOH, SOA, BOS, EOT = 128259, 128260, 128261, 128000, 128009
def _cfg(name, default):
"""recipe config: env KALE_TTS_<NAME>_CONFIG, then ../config/<name>.json (modal.json) or ./config/<name>.json, then the copy in the image"""
here = Path(__file__).resolve().parent
for p in (os.environ.get(f"KALE_TTS_{name.upper()}_CONFIG"), here.parent / "config" / f"{name}.json", here / "config" / f"{name}.json",
Path("/root/recipe") / f"{name}.json"):
if p and Path(p).is_file():
return {**default, **json.loads(Path(p).read_text())}
return default
NAMES = _cfg("modal", {"render_app": "kale-tts-render", "data_volume": "kale-tts-data"})
CFG = _cfg("render", {"render_gpus": 4})
SAMPLING = CFG.get("sampling", {"temperature": 0.4, "top_p": 0.9, "repetition_penalty": 1.1, "min_tokens": 28, "max_tokens_cap": 2048,
"max_tokens_base": 220, "max_tokens_per_word": 60})
app = modal.App(NAMES["render_app"])
vol = modal.Volume.from_name(NAMES["data_volume"], create_if_missing=True)
def _fetch():
from huggingface_hub import snapshot_download
snapshot_download(MAYA, revision=MAYA_REV, allow_patterns=["*.json", "*.safetensors", "*.jinja", "*.txt"])
snapshot_download(SNAC_ID, revision=SNAC_REV)
img = (modal.Image.debian_slim(python_version="3.11").apt_install("libsndfile1")
.pip_install("vllm==0.33.0", "snac==1.2.1", "soundfile", "huggingface_hub", "ninja", "nvidia-cuda-nvcc==13.0.*")
.env({"VLLM_USE_FLASHINFER_SAMPLER": "0", "MAYA_VLLM_ATTN": "FLASH_ATTN", "HF_HOME": "/opt/hf", "CUDA_HOME": "/usr/local/lib/python3.11/site-packages/nvidia/cu13",
"PATH": "/usr/local/lib/python3.11/site-packages/nvidia/cu13/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"})
.run_function(_fetch))
for _n in ("modal", "render"): # the containers read the same names and settings
for _p in (Path(__file__).resolve().parent.parent / "config" / f"{_n}.json", Path(__file__).resolve().parent / "config" / f"{_n}.json"):
if _p.is_file():
img = img.add_local_file(str(_p), f"/root/recipe/{_n}.json")
break
def _unpack(tok):
n = len(tok) // 7
l1, l2, l3 = [], [], []
for i in range(n):
s = [(t - CODE_OFFSET) % 4096 for t in tok[i * 7:(i + 1) * 7]]
l1.append(s[0]); l2 += [s[1], s[4]]; l3 += [s[2], s[3], s[5], s[6]]
return [l1, l2, l3]
@app.function(image=img, gpu="H100", cpu=8, memory=65536, timeout=2400, max_containers=int(CFG["render_gpus"]), scaledown_window=60,
volumes={"/data": vol})
def render_v(jobs: list, sub: str, seed0: int, chunk: int, quant: str = ""):
import numpy as np
import soundfile as sf
import torch
from snac import SNAC
from transformers import AutoTokenizer
from vllm import LLM, SamplingParams
vol.reload()
marker = Path(f"/data/onevoice/_vllm_chunks/{sub}/{chunk:04d}.json")
if marker.exists():
return dict(chunk=chunk, out=json.loads(marker.read_text()), cached=True)
t0 = time.time()
tok = AutoTokenizer.from_pretrained(MAYA, revision=MAYA_REV)
d = lambda i: tok.decode([i])
snac = SNAC.from_pretrained(SNAC_ID, revision=SNAC_REV).eval().to("cuda")
llm = LLM(MAYA, revision=MAYA_REV, dtype="bfloat16", gpu_memory_utilization=0.80, max_model_len=2400, seed=seed0,
attention_backend=os.environ.get("MAYA_VLLM_ATTN", "FLASH_ATTN"), **({"quantization": quant} if quant else {}))
t_load = time.time() - t0
prompt = lambda j: d(SOH) + tok.bos_token + f'<description="{j["description"]}"> {j["text"]}' + d(EOT) + d(EOH) + d(SOA) + d(CODE_START)
S_ = SAMPLING
sps = [SamplingParams(temperature=S_["temperature"], top_p=S_["top_p"], repetition_penalty=S_["repetition_penalty"],
max_tokens=min(S_["max_tokens_cap"], S_["max_tokens_base"] + S_["max_tokens_per_word"] * len(j["text"].split())),
min_tokens=S_["min_tokens"], stop_token_ids=[CODE_END], seed=seed0 + i) for i, j in enumerate(jobs)]
ps = [{"prompt_token_ids": tok(prompt(j), add_special_tokens=False)["input_ids"]} for j in jobs]
outs = llm.generate(ps, sps, use_tqdm=False)
res = []
for i, (j, o) in enumerate(zip(jobs, outs)):
row = list(o.outputs[0].token_ids)
eos = o.outputs[0].finish_reason == "stop" or CODE_END in row
row = row[:row.index(CODE_END)] if CODE_END in row else row
codes = [x for x in row if SNAC_MIN <= x <= SNAC_MAX]
if len(codes) < 28:
res.append(dict(clip=j["clip"], ok=False, why="no audio")); continue
lv = [torch.tensor(l, dtype=torch.long, device="cuda").unsqueeze(0) for l in _unpack(codes)]
with torch.inference_mode():
a = snac.decoder(snac.quantizer.from_codes(lv))[0, 0].float().cpu().numpy()
out = Path(f"/data/onevoice/{j['voice']}/{sub}/{j['clip']}.wav")
out.parent.mkdir(parents=True, exist_ok=True)
sf.write(out, a.astype("float32"), 24000, subtype="FLOAT")
n7 = len(codes) // 7 * 7
np.save(out.with_suffix(".snac.npy"), (np.array(codes[:n7], dtype=np.int64) - CODE_OFFSET).astype(np.uint16).reshape(-1, 7))
sec = len(a) / 24000
res.append(dict(clip=j["clip"], ok=bool(eos), eos=bool(eos), seconds=round(sec, 3), seed=seed0 + i, engine="vllm-0.33.0", quant=quant or "bf16",
raw_peak=round(float(np.abs(a).max()), 4), text=j["text"], path=f"onevoice/{j['voice']}/{sub}/{j['clip']}.wav",
snac=f"onevoice/{j['voice']}/{sub}/{j['clip']}.snac.npy"))
marker.parent.mkdir(parents=True, exist_ok=True)
marker.write_text(json.dumps(res))
vol.commit()
audio = sum(r.get("seconds", 0) for r in res)
return dict(chunk=chunk, out=res, gpu_s=round(time.time() - t0, 1), load_s=round(t_load, 1), audio_s=round(audio, 1))
@app.local_entrypoint()
def main(jobs: str, sub: str, per_call: int = 2000, seed0: int = 1_000_000, group: str = "", quant: str = ""):
root = Path(os.environ.get("KALE_TTS_ROOT", ".")) # folder holding work/
rows = [json.loads(x) for x in (root / jobs).read_text().splitlines() if x.strip()]
args = [(rows[i:i + per_call], sub, seed0 + i, k, quant) for k, i in enumerate(range(0, len(rows), per_call))]
od = root / f"work/render-{group or sub}"
od.mkdir(parents=True, exist_ok=True)
for r in render_v.starmap(args, order_outputs=False, return_exceptions=True):
if isinstance(r, Exception):
print("CHUNK FAILED", repr(r)[:300], flush=True)
continue
(od / f"chunk-{r['chunk']:04d}.json").write_text(json.dumps(r["out"]))
print(f"chunk {r['chunk']}: {len(r['out'])} takes, {r.get('audio_s', 0) / 3600:.2f} h audio in {r.get('gpu_s', 0)} s (load {r.get('load_s')} s)", flush=True)