File size: 8,881 Bytes
f21d6eb | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 | # SPDX-License-Identifier: Apache-2.0
# Copyright 2026 Kale
"""Maya-1 training-data renderer: the script that rendered every release voice's training takes (16 Nov 2026 build; the same file
rendered the earlier voices v7 and m3).
Input: a jobs.jsonl (rows: clip, voice, text, description[, mood, tagname]; built by make_jobs.py). One Maya-1 prompt per row,
sampling from config/render.json (temperature 0.4, top_p 0.9, repetition penalty 1.1, min 28 / max min(2048, 220 + 60*words) tokens),
one seed per take (seed0 + row index), vLLM 0.33.0 bf16 (`--quant fp8` = vLLM online fp8 weights; the release used bf16).
Outputs on the data volume: /data/onevoice/<voice>/<sub>/<clip>.wav (float32 24 kHz, exactly as generated, uncut) and next to it
<clip>.snac.npy (the take's raw Maya code tokens, uint16, shape (frames, 7), value = token id - 128266; SNAC code = value % 4096; slot
order per frame = Maya's [l1, l2, l3, l3, l2, l3, l3], see _unpack); chunk markers /data/onevoice/_vllm_chunks/<sub>/; locally
<root>/work/render-<group or sub>/chunk-*.json with eos/seconds/seed/raw_peak/path per take (the filter step reads these).
modal run render.py::main --jobs work/jobs-<voice>.jsonl --sub <voice> --group <voice> --per-call 10000 --seed0 <seed0>
Env inside the image: VLLM_USE_FLASHINFER_SAMPLER=0, MAYA_VLLM_ATTN=FLASH_ATTN, CUDA_HOME/PATH to the pip CUDA 13 nvcc. GPU H100,
8 cores, 64 GiB, timeout 40 min per chunk.
Known trap: on a warm vLLM engine outputs may come back out of prompt order. This script makes one fresh engine per call and zips
outputs with jobs (as the release did); stream_filter.py in ../filter matches outputs to jobs by prompt tokens instead."""
import json
import os
import time
from pathlib import Path
import modal
MAYA, MAYA_REV = "maya-research/maya1", "b1b9b38ca9decbfcfd6c98ea93dd502bfd0f9572"
SNAC_ID, SNAC_REV = "hubertsiuzdak/snac_24khz", "d73ad176a12188fcf4f360ba3bf2c2fbbe8f58ec"
CODE_START, CODE_END, CODE_OFFSET = 128257, 128258, 128266
SNAC_MIN, SNAC_MAX = 128266, 156937
SOH, EOH, SOA, BOS, EOT = 128259, 128260, 128261, 128000, 128009
def _cfg(name, default):
"""recipe config: env KALE_TTS_<NAME>_CONFIG, then ../config/<name>.json (modal.json) or ./config/<name>.json, then the copy in the image"""
here = Path(__file__).resolve().parent
for p in (os.environ.get(f"KALE_TTS_{name.upper()}_CONFIG"), here.parent / "config" / f"{name}.json", here / "config" / f"{name}.json",
Path("/root/recipe") / f"{name}.json"):
if p and Path(p).is_file():
return {**default, **json.loads(Path(p).read_text())}
return default
NAMES = _cfg("modal", {"render_app": "kale-tts-render", "data_volume": "kale-tts-data"})
CFG = _cfg("render", {"render_gpus": 4})
SAMPLING = CFG.get("sampling", {"temperature": 0.4, "top_p": 0.9, "repetition_penalty": 1.1, "min_tokens": 28, "max_tokens_cap": 2048,
"max_tokens_base": 220, "max_tokens_per_word": 60})
app = modal.App(NAMES["render_app"])
vol = modal.Volume.from_name(NAMES["data_volume"], create_if_missing=True)
def _fetch():
from huggingface_hub import snapshot_download
snapshot_download(MAYA, revision=MAYA_REV, allow_patterns=["*.json", "*.safetensors", "*.jinja", "*.txt"])
snapshot_download(SNAC_ID, revision=SNAC_REV)
img = (modal.Image.debian_slim(python_version="3.11").apt_install("libsndfile1")
.pip_install("vllm==0.33.0", "snac==1.2.1", "soundfile", "huggingface_hub", "ninja", "nvidia-cuda-nvcc==13.0.*")
.env({"VLLM_USE_FLASHINFER_SAMPLER": "0", "MAYA_VLLM_ATTN": "FLASH_ATTN", "HF_HOME": "/opt/hf", "CUDA_HOME": "/usr/local/lib/python3.11/site-packages/nvidia/cu13",
"PATH": "/usr/local/lib/python3.11/site-packages/nvidia/cu13/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"})
.run_function(_fetch))
for _n in ("modal", "render"): # the containers read the same names and settings
for _p in (Path(__file__).resolve().parent.parent / "config" / f"{_n}.json", Path(__file__).resolve().parent / "config" / f"{_n}.json"):
if _p.is_file():
img = img.add_local_file(str(_p), f"/root/recipe/{_n}.json")
break
def _unpack(tok):
n = len(tok) // 7
l1, l2, l3 = [], [], []
for i in range(n):
s = [(t - CODE_OFFSET) % 4096 for t in tok[i * 7:(i + 1) * 7]]
l1.append(s[0]); l2 += [s[1], s[4]]; l3 += [s[2], s[3], s[5], s[6]]
return [l1, l2, l3]
@app.function(image=img, gpu="H100", cpu=8, memory=65536, timeout=2400, max_containers=int(CFG["render_gpus"]), scaledown_window=60,
volumes={"/data": vol})
def render_v(jobs: list, sub: str, seed0: int, chunk: int, quant: str = ""):
import numpy as np
import soundfile as sf
import torch
from snac import SNAC
from transformers import AutoTokenizer
from vllm import LLM, SamplingParams
vol.reload()
marker = Path(f"/data/onevoice/_vllm_chunks/{sub}/{chunk:04d}.json")
if marker.exists():
return dict(chunk=chunk, out=json.loads(marker.read_text()), cached=True)
t0 = time.time()
tok = AutoTokenizer.from_pretrained(MAYA, revision=MAYA_REV)
d = lambda i: tok.decode([i])
snac = SNAC.from_pretrained(SNAC_ID, revision=SNAC_REV).eval().to("cuda")
llm = LLM(MAYA, revision=MAYA_REV, dtype="bfloat16", gpu_memory_utilization=0.80, max_model_len=2400, seed=seed0,
attention_backend=os.environ.get("MAYA_VLLM_ATTN", "FLASH_ATTN"), **({"quantization": quant} if quant else {}))
t_load = time.time() - t0
prompt = lambda j: d(SOH) + tok.bos_token + f'<description="{j["description"]}"> {j["text"]}' + d(EOT) + d(EOH) + d(SOA) + d(CODE_START)
S_ = SAMPLING
sps = [SamplingParams(temperature=S_["temperature"], top_p=S_["top_p"], repetition_penalty=S_["repetition_penalty"],
max_tokens=min(S_["max_tokens_cap"], S_["max_tokens_base"] + S_["max_tokens_per_word"] * len(j["text"].split())),
min_tokens=S_["min_tokens"], stop_token_ids=[CODE_END], seed=seed0 + i) for i, j in enumerate(jobs)]
ps = [{"prompt_token_ids": tok(prompt(j), add_special_tokens=False)["input_ids"]} for j in jobs]
outs = llm.generate(ps, sps, use_tqdm=False)
res = []
for i, (j, o) in enumerate(zip(jobs, outs)):
row = list(o.outputs[0].token_ids)
eos = o.outputs[0].finish_reason == "stop" or CODE_END in row
row = row[:row.index(CODE_END)] if CODE_END in row else row
codes = [x for x in row if SNAC_MIN <= x <= SNAC_MAX]
if len(codes) < 28:
res.append(dict(clip=j["clip"], ok=False, why="no audio")); continue
lv = [torch.tensor(l, dtype=torch.long, device="cuda").unsqueeze(0) for l in _unpack(codes)]
with torch.inference_mode():
a = snac.decoder(snac.quantizer.from_codes(lv))[0, 0].float().cpu().numpy()
out = Path(f"/data/onevoice/{j['voice']}/{sub}/{j['clip']}.wav")
out.parent.mkdir(parents=True, exist_ok=True)
sf.write(out, a.astype("float32"), 24000, subtype="FLOAT")
n7 = len(codes) // 7 * 7
np.save(out.with_suffix(".snac.npy"), (np.array(codes[:n7], dtype=np.int64) - CODE_OFFSET).astype(np.uint16).reshape(-1, 7))
sec = len(a) / 24000
res.append(dict(clip=j["clip"], ok=bool(eos), eos=bool(eos), seconds=round(sec, 3), seed=seed0 + i, engine="vllm-0.33.0", quant=quant or "bf16",
raw_peak=round(float(np.abs(a).max()), 4), text=j["text"], path=f"onevoice/{j['voice']}/{sub}/{j['clip']}.wav",
snac=f"onevoice/{j['voice']}/{sub}/{j['clip']}.snac.npy"))
marker.parent.mkdir(parents=True, exist_ok=True)
marker.write_text(json.dumps(res))
vol.commit()
audio = sum(r.get("seconds", 0) for r in res)
return dict(chunk=chunk, out=res, gpu_s=round(time.time() - t0, 1), load_s=round(t_load, 1), audio_s=round(audio, 1))
@app.local_entrypoint()
def main(jobs: str, sub: str, per_call: int = 2000, seed0: int = 1_000_000, group: str = "", quant: str = ""):
root = Path(os.environ.get("KALE_TTS_ROOT", ".")) # folder holding work/
rows = [json.loads(x) for x in (root / jobs).read_text().splitlines() if x.strip()]
args = [(rows[i:i + per_call], sub, seed0 + i, k, quant) for k, i in enumerate(range(0, len(rows), per_call))]
od = root / f"work/render-{group or sub}"
od.mkdir(parents=True, exist_ok=True)
for r in render_v.starmap(args, order_outputs=False, return_exceptions=True):
if isinstance(r, Exception):
print("CHUNK FAILED", repr(r)[:300], flush=True)
continue
(od / f"chunk-{r['chunk']:04d}.json").write_text(json.dumps(r["out"]))
print(f"chunk {r['chunk']}: {len(r['out'])} takes, {r.get('audio_s', 0) / 3600:.2f} h audio in {r.get('gpu_s', 0)} s (load {r.get('load_s')} s)", flush=True)
|