NERDDISCO's picture
kale-tts-1
f21d6eb
Raw History Blame Contribute Delete
27.5 kB
# SPDX-License-Identifier: Apache-2.0
# Copyright 2026 Kale
# Uses jhu-clsp/ettin-encoder-17m (MIT) and its tokenizer through transformers (Apache-2.0); downloaded into the image, not copied.
# Uses Kokoro-82M / kokoro (Apache-2.0, Copyright hexgrad) through ../train/kline.py; ../train/onnx_export.py carries paradee portions
# (Apache-2.0, ../train/third_party/paradee/LICENSE and NOTICE).
"""kale-tts-1-tiny's Ettin text side (Modal app): Ettin-encoder-17m word features (frozen) + a small phoneme encoder replace Kokoro's
ALBERT (+ bert_encoder). Distilled from the release teacher's own text side on text only, so the predictor input (d_en), durations,
pitch and loudness match; the teacher's predictor stays frozen and is reused, as are the text encoder and the t10k decoder.
upload : modal run textside.py::upload_entry work/sents_us.jsonl + work/sents_gb.jsonl -> runs volume textside/
train : modal run --detach textside.py::train_entry (settings: config/textside.json "train"; 1 x H100, ~72 min for 20k steps)
export : modal run textside.py::export_entry (CPU) -> runs volume textside/export/<name>/model.fp32.onnx, copied to work/
App and volume names come from ../config/modal.json ("textside_app", "runs_volume", "teacher_volume").
Volumes: /data = runs volume (../train's runs: duration head, pitch/loudness heads, tiny decoder; this step's data, runs and exports),
/t = teacher volume (../teacher's export + voice packs, read-only).
"""
import json
import os
import time
from datetime import datetime, timezone
from pathlib import Path
import modal
HERE = Path(__file__).resolve().parent
def _cfg(name, local, remote):
for p in (os.environ.get(name) if name else None, local, Path(remote)):
if p and Path(p).is_file():
return Path(p)
return None
_MC = _cfg("KALE_TTS_MODAL_CONFIG", HERE.parent / "config" / "modal.json", "/root/recipe/modal.json")
_TC = _cfg("", HERE / "config" / "textside.json", "/root/textside/textside.json")
NAMES = {"textside_app": "kale-tts-textside", "runs_volume": "kale-tts-runs", "teacher_volume": "kale-tts-teacher",
**(json.loads(_MC.read_text()) if _MC else {})}
CFG = json.loads(_TC.read_text()) if _TC else {}
app = modal.App(NAMES["textside_app"])
data = modal.Volume.from_name(NAMES["runs_volume"], create_if_missing=True)
teacher = modal.Volume.from_name(NAMES["teacher_volume"], create_if_missing=True).read_only()
VOL = {"/data": data, "/t": teacher} # HF files baked into the image (no HF volume)
TRAIN_DIR = HERE.parent / "train"
WORK = HERE / "work"
utc = lambda: datetime.now(timezone.utc).strftime("%d %b %H:%M:%S UTC") # noqa: E731
ETTIN = CFG.get("ettin_model", "jhu-clsp/ettin-encoder-17m")
ETTIN_REV = CFG.get("ettin_revision")
KOKORO_REV = CFG.get("kokoro_rev", "f3ff3571791e39611d31c381e3a41a3af07b4987")
img = (
modal.Image.debian_slim(python_version="3.12")
.apt_install("libsndfile1", "git")
.pip_install("torch==2.8.0", "torchaudio==2.8.0", "transformers==4.48.3", "kokoro==0.9.4", "soundfile", "numpy<2.3", "scipy",
"onnx", "onnxruntime==1.22.1", "onnxscript")
.run_commands(f"python -c \"from huggingface_hub import snapshot_download as s; s('{ETTIN}', local_dir='/opt/ettin'"
+ (f", revision='{ETTIN_REV}'" if ETTIN_REV else "") + ")\"",
f"python -c \"from huggingface_hub import hf_hub_download as d; d('hexgrad/Kokoro-82M', 'config.json', revision='{KOKORO_REV}')\"")
.env({"PYTHONUNBUFFERED": "1", "HF_HUB_DISABLE_TELEMETRY": "1", "PARADEE_DIR": "/opt/paradee"})
)
if (TRAIN_DIR / "third_party" / "paradee").is_dir():
img = img.add_local_dir(str(TRAIN_DIR / "third_party" / "paradee"), "/opt/paradee", copy=True, ignore=["__pycache__"])
for _n in ("model.py", "kline.py", "onnx_export.py"):
if (TRAIN_DIR / _n).is_file():
img = img.add_local_file(str(TRAIN_DIR / _n), f"/opt/recipe/{_n}", copy=True)
for _n in ("trace_us.jsonl", "trace_gb.jsonl"): # the trace line's ids (make_ids.mjs), for the export
if (WORK / _n).is_file():
img = img.add_local_file(str(WORK / _n), f"/opt/textside/{_n}", copy=True)
if _MC is not None:
img = img.add_local_file(str(_MC), "/root/recipe/modal.json", copy=True)
if _TC is not None:
img = img.add_local_file(str(_TC), "/root/textside/textside.json", copy=True)
V10 = CFG.get("voices", "s057b,f06,d03,w165,w602,m3,t071,d13,x328,h013f")
GB = tuple(CFG.get("gb_voices", ["d03", "d13"]))
CLAMP10 = CFG.get("end_clamp", {"n_frames_tail": 36, "semis": 6.0, "thr": 50.0, "min_hz": 85.0})
F0_FLOOR = float(CFG.get("f0_floor", 80.0))
VD = CFG.get("volume_dir", "/data/textside")
def _log(msg, path=None):
line = f"{utc()} {msg}"
print(line, flush=True)
if path:
with open(path, "a") as f:
f.write(line + "\n")
# ------------------------------------------------------------------ model pieces
def _mods():
import torch
import torch.nn as nn
class WordFeat(nn.Module):
"""frozen Ettin-17m: per-word features (mean of the last two layers, mean over a word's sub-tokens), previous sentence as left
context. Batched."""
def __init__(self, path="/opt/ettin"):
super().__init__()
from transformers import AutoModel, AutoTokenizer
self.tok = AutoTokenizer.from_pretrained(path)
self.model = AutoModel.from_pretrained(path, attn_implementation="eager").eval().requires_grad_(False)
@torch.no_grad()
def forward(self, texts, prevs, dev):
fulls, spans = [], []
for t, p in zip(texts, prevs):
words = t.split(); pre = (p.strip() + " ") if p and p.strip() else ""
sp, pos = [], len(pre)
for w in words:
sp.append((pos, pos + len(w))); pos += len(w) + 1
fulls.append(pre + " ".join(words)); spans.append(sp)
enc = self.tok(fulls, return_offsets_mapping=True, return_tensors="pt", padding=True, truncation=True, max_length=512)
off = enc.pop("offset_mapping").tolist()
hs = self.model(**{k: v.to(dev) for k, v in enc.items()}, output_hidden_states=True).hidden_states
h = ((hs[-1] + hs[-2]) / 2).float() # [B, Ttok, 256]
W = max(len(s) for s in spans)
out = torch.zeros(len(texts), max(W, 1), h.shape[-1], device=dev)
for b, sp in enumerate(spans):
for i, (a, e) in enumerate(sp):
idx = [t for t, (s, f) in enumerate(off[b]) if f > s and s < e and f > a]
if idx:
out[b, i] = h[b, idx].mean(0)
return out
class ConvBlock(nn.Module):
def __init__(self, c, k=5):
super().__init__()
self.conv = nn.Conv1d(c, c, k, padding=k // 2); self.norm = nn.LayerNorm(c); self.act = nn.GELU()
def forward(self, x, m): # x [B,T,C], m [B,T] True = pad
y = self.conv(x.masked_fill(m[..., None], 0).transpose(1, 2)).transpose(1, 2)
return x + self.act(self.norm(y))
class TFBlock(nn.Module):
"""pre-norm transformer block with hand-written attention (exports with dynamic length; nn.MultiheadAttention's legacy export
bakes the length)"""
def __init__(self, c, nh=4, drop=0.1):
super().__init__()
self.n1 = nn.LayerNorm(c); self.qkv = nn.Linear(c, 3 * c); self.o = nn.Linear(c, c); self.nh = nh
self.n2 = nn.LayerNorm(c); self.ff = nn.Sequential(nn.Linear(c, 4 * c), nn.GELU(), nn.Linear(4 * c, c)); self.drop = nn.Dropout(drop)
def forward(self, x, m): # x [B,T,C], m [B,T] True = pad
B, T, C = x.shape; hd = C // self.nh
q, k, v = self.qkv(self.n1(x)).reshape(B, T, 3, self.nh, hd).permute(2, 0, 3, 1, 4)
a = (q @ k.transpose(-1, -2)) * (hd ** -0.5) + m[:, None, None, :].float() * -1e4
y = (a.softmax(-1) @ v).transpose(1, 2).reshape(B, T, C)
x = x + self.drop(self.o(y))
return x + self.drop(self.ff(self.n2(x)))
class K20Text(nn.Module):
"""replaces bert + bert_encoder: phoneme embedding -> 3 conv blocks -> + Ettin word features (gathered per token) -> 2 transformer
layers -> 512-d d_en (Kokoro's predictor input)."""
def __init__(self, c=256, nconv=3, ntf=2, wdim=256, out=512):
super().__init__()
self.emb = nn.Embedding(178, c)
self.convs = nn.ModuleList([ConvBlock(c) for _ in range(nconv)])
self.wproj = nn.Sequential(nn.Linear(wdim, c), nn.GELU(), nn.Linear(c, c))
self.pos = nn.Embedding(512, c)
self.tf = nn.ModuleList([TFBlock(c) for _ in range(ntf)])
self.out = nn.Linear(c, out)
def forward(self, x, m, wfeat, widx): # x [B,T] ids, wfeat [B,W,256], widx [B,T]
h = self.emb(x)
for b in self.convs:
h = b(h, m)
w = torch.gather(wfeat, 1, widx[..., None].expand(-1, -1, wfeat.shape[-1]))
h = h + self.wproj(w) + self.pos(torch.arange(x.shape[1], device=x.device))[None]
for b in self.tf:
h = b(h, m)
return self.out(h) # [B,T,512]
return WordFeat, K20Text
def _teacher(dev):
"""the release tiny's text side -> T: .K (kline module), .km (Kokoro modules), .packs (voice pack + the voice's pitch offset),
.voices, .pred (the predictor the student feeds), .out_dim (d_en width), .d_en(x, m) -> [B,T,C] (the part the student replaces:
ALBERT + bert_encoder), .style(voice, n_ids) -> [128] (the pack row's prosody half)."""
import sys
import types
import torch
sys.path.insert(0, "/opt/recipe")
T = types.SimpleNamespace()
os.environ.update(KLINE_EXPORT=CFG["teacher_export"], KLINE_VOICES=V10, KLINE_PACKS=CFG["teacher_packs"])
import kline as K
km, _ = K.load_kmodel(dev, CFG["dur_ckpt"], CFG["pros_ckpt"])
packs = K.load_packs(dev); pk = torch.load(CFG["pros_ckpt"], map_location=dev, weights_only=False); out = {}
for v in K.VOICES:
p = packs[v].clone(); p[..., 128:] += pk["pv"][pk["voices"].index(v)].to(p.device); out[v] = p
T.K, T.km, T.packs, T.voices, T.pred, T.out_dim = K, km, out, list(out), km.predictor, 512
T.d_en = lambda x, m: km.bert_encoder(km.bert(x, attention_mask=(~m).int()))
def _st(v, n):
p = out[v][min(n, len(out[v])) - 1]
return p[0, 128:] if p.dim() == 2 else p[128:]
T.style = _st
return T
def _batch(rows, dev):
"""rows {ids, widx, text, prev} -> padded ids with Kokoro's 0 pads at both ends, mask, widx (pads -> word 0 / last word)"""
import torch
T = max(len(r["ids"]) for r in rows) + 2
x = torch.zeros(len(rows), T, dtype=torch.long); wi = torch.zeros(len(rows), T, dtype=torch.long)
L = torch.LongTensor([len(r["ids"]) + 2 for r in rows])
for b, r in enumerate(rows):
x[b, 1:1 + len(r["ids"])] = torch.LongTensor(r["ids"])
w = r["widx"]; wi[b, 1:1 + len(w)] = torch.LongTensor(w); wi[b, 1 + len(w)] = w[-1] if w else 0
m = torch.arange(T)[None] >= L[:, None]
return x.to(dev), m.to(dev), wi.to(dev), L.to(dev)
# ------------------------------------------------------------------ training
@app.function(image=img, gpu="H100", cpu=8, memory=65536, timeout=3 * 3600, max_containers=1, volumes=VOL)
def train(run: str, steps: int = 20000, bs: int = 64, lr: float = 3e-4, w_feat: float = 1.0, w_dur: float = 1.0, w_f0: float = 1.0,
w_n: float = 1.0, ckpt_every: int = 5000, resume: bool = True, f0_log: bool = True, held_n: int = 256, min_ids: int = 4,
max_ids: int = 400):
"""text-only distillation. Teacher (frozen): d_en = bert_encoder(bert(x)), durations (t10d2 head), F0 / N (t10p heads) on the
teacher's rounded durations. Student: K20Text -> d_en; the teacher's predictor (frozen) on the student d_en -> durations, F0, N under
the TEACHER's alignment. A random voice per sentence (its pack row + pitch offset as style); d03 / d13 sentences use en-GB ids + word
map. Losses: L1 d_en, L1 durations, F0 (f0_log: semitones on the teacher's voiced frames / 12 + Hz/100 x 0.5 on its unvoiced ones;
else L1 Hz / 100), L1 N."""
import random
import torch
torch.backends.cuda.matmul.allow_tf32 = True
dev = "cuda"
od = Path(f"{VD}/{run}"); od.mkdir(parents=True, exist_ok=True); lg = str(od / "train.log")
rows = [json.loads(l) for l in open(f"{VD}/{CFG.get('sents', 'sents.jsonl')}")]
gbr = {}
for l in open(f"{VD}/{CFG.get('sents_gb', 'sents_gb.jsonl')}"): # d03 / d13 read en-gb ids
r = json.loads(l); gbr[r["text"]] = r
rows = [r for r in rows if min_ids <= len(r["ids"]) <= max_ids and r["text"] in gbr]
random.Random(1).shuffle(rows); held, rows = rows[:held_n], rows[held_n:]
TT = _teacher(dev)
vnames = TT.voices
WordFeat, K20Text = _mods()
wf = WordFeat().to(dev)
st = K20Text(out=TT.out_dim).to(dev)
npar = sum(p.numel() for p in st.parameters())
pred = TT.pred
for mod in pred.modules(): # cuDNN LSTM backward needs train mode; single-layer LSTMs have no dropout, so outputs are unchanged
if isinstance(mod, torch.nn.LSTM):
assert mod.num_layers == 1; mod.train()
opt = torch.optim.AdamW(st.parameters(), lr=lr, weight_decay=0.01)
step0 = 0
if resume and (od / "last.pt").exists():
ck = torch.load(od / "last.pt", map_location=dev, weights_only=False)
st.load_state_dict(ck["model"]); opt.load_state_dict(ck["opt"]); step0 = ck["step"]
sched = torch.optim.lr_scheduler.LambdaLR(opt, lambda s: min(1.0, (s + 1) / 1000) * max(0.05, 0.5 * (1 + __import__("math").cos(3.14159 * min(1.0, s / steps)))))
for _ in range(step0):
sched.step()
_log(f"teacher: voices {vnames}", lg)
_log(f"train {run}: {len(rows)} sentences, held {len(held)}, student {npar / 1e6:.2f} M params, from step {step0}", lg)
def teacher_and_student(b, train_mode):
vs = [random.choice(vnames) for _ in b] # a random voice per sentence
b = [gbr[r["text"]] if v in GB else r for r, v in zip(b, vs)] # British voices: en-gb ids + word map
x, m, wi, L = _batch(b, dev)
s = torch.stack([TT.style(v, len(r["ids"])) for v, r in zip(vs, b)]).to(dev).float()
with torch.no_grad():
d_en_t = TT.d_en(x, m) # [B,T,C]
d_t = pred.text_encoder(d_en_t.transpose(-1, -2), s, L, m)
xx, _ = pred.lstm(d_t)
dur_t = torch.sigmoid(pred.duration_proj(xx)).sum(-1).masked_fill(m, 0) # [B,T]
n = torch.round(dur_t).clamp(min=1).long().masked_fill(m, 0)
Fr = int(n.sum(1).max())
aln = torch.zeros(x.shape[0], x.shape[1], Fr, device=dev)
fm = torch.zeros(x.shape[0], Fr, dtype=torch.bool, device=dev)
for bi in range(x.shape[0]):
idx = torch.repeat_interleave(torch.arange(x.shape[1], device=dev), n[bi])
aln[bi, idx, torch.arange(len(idx), device=dev)] = 1; fm[bi, :len(idx)] = True
F0_t, N_t = pred.F0Ntrain(d_t.transpose(-1, -2) @ aln, s)
wfeat = wf([r["text"] for r in b], [r["prev"] for r in b], dev)
with torch.set_grad_enabled(train_mode):
d_en_s = st(x, m, wfeat, wi)
d_s = pred.text_encoder(d_en_s.transpose(-1, -2), s, L, m)
xs, _ = pred.lstm(d_s)
dur_s = torch.sigmoid(pred.duration_proj(xs)).sum(-1)
F0_s, N_s = pred.F0Ntrain(d_s.transpose(-1, -2) @ aln, s)
tm = ~m
l_feat = (d_en_s - d_en_t).abs().mean(-1)[tm].mean()
l_dur = (dur_s - dur_t).abs()[tm].mean()
ff = fm.repeat_interleave(F0_s.shape[1] // fm.shape[1], dim=1)[:, :F0_s.shape[1]] # F0 / N run at 2x the frame rate
if f0_log: # semitones on the teacher's voiced frames (a 5 Hz miss is 1 st at 90 Hz, 0.4 st at 220 Hz), Hz/100 on its unvoiced ones
vo = ff & (F0_t > 40)
l_f0 = (12 * torch.log2(F0_s.clamp(min=20) / F0_t.clamp(min=20))).abs()[vo].mean() / 12 + (F0_s.abs() / 100)[ff & ~vo].mean() * 0.5
else:
l_f0 = ((F0_s - F0_t).abs() / 100)[ff].mean()
l_n = (N_s - N_t).abs()[ff].mean()
return l_feat, l_dur, l_f0, l_n
t0 = time.time(); acc = []
for step in range(step0, steps):
st.train()
b = random.sample(rows, bs)
l_feat, l_dur, l_f0, l_n = teacher_and_student(b, True)
loss = w_feat * l_feat + w_dur * l_dur + w_f0 * l_f0 + w_n * l_n
opt.zero_grad(set_to_none=True); loss.backward()
torch.nn.utils.clip_grad_norm_(st.parameters(), 1.0); opt.step(); sched.step()
acc.append([l_feat.item(), l_dur.item(), l_f0.item(), l_n.item()])
if (step + 1) % 200 == 0:
a = [sum(c) / len(c) for c in zip(*acc)]; acc = []
st.eval(); hv = []
if (step + 1) % 1000 == 0:
with torch.no_grad():
for i in range(0, len(held), 64):
hv.append([v.item() for v in teacher_and_student(held[i:i + 64], False)])
hv = [round(sum(c) / len(c), 4) for c in zip(*hv)]
_log(f"step {step + 1} feat {a[0]:.4f} dur {a[1]:.4f} f0 {a[2]:.4f} N {a[3]:.4f}" + (f" | held {hv}" if hv else "") +
f" | {(time.time() - t0) / 60:.1f} min", lg)
if (step + 1) % ckpt_every == 0 or step + 1 == steps:
ck = dict(model=st.state_dict(), opt=opt.state_dict(), step=step + 1, pred=None, params=npar)
torch.save(ck, od / f"step{step + 1:06d}.pt"); torch.save(ck, od / "last.pt"); data.commit()
_log(f"done {run} {steps} steps {(time.time() - t0) / 60:.1f} min", lg)
data.commit()
return dict(run=run, steps=steps, params=npar)
@app.local_entrypoint()
def train_entry(run: str = "", steps: int = 0):
"""defaults from config/textside.json "train" (the release run: t10e, 20k steps, f0_log)"""
t = dict(CFG["train"]); t = {k: v for k, v in t.items() if not k.startswith("_")}
print(train.remote(run or t["run"], steps or t["steps"], t["bs"], t["lr"], t["w_feat"], t["w_dur"], t["w_f0"], t["w_n"],
ckpt_every=t["ckpt_every"], f0_log=t["f0_log"], held_n=t["held"], min_ids=t["min_ids"], max_ids=t["max_ids"]))
@app.local_entrypoint()
def upload_entry(us: str = str(WORK / "sents_us.jsonl"), gb: str = str(WORK / "sents_gb.jsonl")):
"""make_ids.mjs outputs -> runs volume <volume_dir>/sents.jsonl, sents_gb.jsonl"""
sub = VD[len("/data/"):] if VD.startswith("/data/") else VD.lstrip("/")
with data.batch_upload(force=True) as b:
b.put_file(us, f"/{sub}/{CFG.get('sents', 'sents.jsonl')}"); b.put_file(gb, f"/{sub}/{CFG.get('sents_gb', 'sents_gb.jsonl')}")
print("uploaded to", sub)
# ------------------------------------------------------------------ export: one graph = Ettin + student + the predictor / text encoder / decoder
def clamp_onnx(F0, n_frames_tail=36, semis=6.0, thr=50.0, min_hz=85.0):
"""kline end_clamp with graph ops only (same result): lower median of the voiced F0 via sort, last voiced frame via a masked max."""
import torch
f = F0[0]; v = f > thr; cnt = v.sum()
srt = torch.sort(torch.where(v, f, torch.full_like(f, -1e9)), descending=True).values
k = cnt - 1 - torch.div(cnt - 1, 2, rounding_mode="floor") # lower median (ascending (n-1)//2) in descending order
med = srt.gather(0, k.clamp(min=0).reshape(1))[0]
floor = torch.clamp(med * 2 ** (-semis / 12), min=min_hz)
ar = torch.arange(f.shape[0], device=f.device)
last = torch.where(v, ar, torch.zeros_like(ar)).max()
tail = (ar >= last - n_frames_tail) & (ar <= last)
return torch.where(v & tail & (cnt >= 5), torch.maximum(f, floor), f)[None]
@app.function(image=img, cpu=8.0, memory=32768, timeout=3600, max_containers=1, volumes=VOL)
def export10(ck: str, name: str, voice: str = "s057b"):
"""one fp32 graph for all voices: inputs input_ids [1,T], style [1,256], speed [1], ettin_ids [1,S], ettin_mask [1,S], wavg [1,W,S]
(mean over each word's sub-tokens), widx [1,T] (word per phoneme token), voice_sound [1,16] (the decoder's style row) -> waveform.
The end clamp (per-sentence floor) and the 80 Hz voicing floor are in the graph. `voice` only picks the trace / test inputs."""
import sys
import numpy as np
import torch
import torch.nn as nn
sys.path.insert(0, "/opt/recipe")
import onnx_export as X
torch.set_num_threads(8); torch.backends.mha.set_fastpath_enabled(False)
TT = _teacher("cpu"); K = TT.K; vid = K.VOICES.index(voice)
WordFeat, K20Text = _mods()
c = torch.load(ck, map_location="cpu", weights_only=False); st = K20Text(out=TT.out_dim).eval(); st.load_state_dict(c["model"])
wf = WordFeat()
kd = K.KDec("k").eval(); kd.load_state_dict(torch.load(CFG["kdec_ckpt"], map_location="cpu", weights_only=False)["model"]); X.strip(kd); X.strip(TT.km)
dec, sty, te = kd.dec, kd.style[vid:vid + 1].detach(), TT.km.text_encoder
pred = TT.pred; X.strip(pred)
class G(nn.Module):
def __init__(self):
super().__init__(); self.pred, self.st, self.dec, self.te = pred, st, dec, te; self.head = X.HeadX(dec.head)
self.et = wf.model; self.register_buffer("sty", sty)
def textside(self, input_ids, m, ettin_ids, ettin_mask, wavg, widx):
hs = self.et(input_ids=ettin_ids, attention_mask=ettin_mask, output_hidden_states=True).hidden_states; h = (hs[-1] + hs[-2]) / 2
return self.st(input_ids, m, wavg @ h, widx)
def body(self, input_ids, s, speed, d_en):
p = self.pred; d_en = d_en.transpose(-1, -2)
T = input_ids.shape[1]; sx = s[:, :, None].expand(-1, -1, T); x = torch.cat([d_en, sx], 1)
for b in p.text_encoder.lstms:
if isinstance(b, nn.LSTM): x = b(x.transpose(-1, -2))[0].transpose(-1, -2)
else: x = torch.cat([b(x.transpose(-1, -2), s).transpose(-1, -2), sx], 1)
d = x.transpose(-1, -2)
dur = torch.sigmoid(p.duration_proj(p.lstm(d)[0])).sum(-1)[0]
n = torch.clamp(torch.round(dur / speed), min=1).long()
end = torch.cumsum(n, 0); start = end - n; f = torch.arange(end[-1])[None]
aln = ((f >= start[:, None]) & (f < end[:, None])).float()[None]
F0, N = p.F0Ntrain(d.transpose(-1, -2) @ aln, s)
F0 = clamp_onnx(F0, **CLAMP10); F0 = torch.where(F0 < F0_FLOOR, torch.zeros_like(F0), F0)
e = self.te; hh = e.embedding(input_ids).transpose(1, 2)
for c_ in e.cnn: hh = c_(hh)
t_en = e.lstm(hh.transpose(1, 2))[0].transpose(-1, -2)
sty = getattr(self, "sty_in", None)
return self.head(self.dec.trunk(t_en @ aln, F0, N, self.sty if sty is None else sty), F0)
class GS(G): # one file for all voices: the decoder's style row is an input (voice_sound [1,16])
def forward(self, input_ids, style, speed, ettin_ids, ettin_mask, wavg, widx, voice_sound):
m = torch.zeros_like(input_ids, dtype=torch.bool)
self.sty_in = voice_sound
return self.body(input_ids, style[:, 128:], speed, self.textside(input_ids, m, ettin_ids, ettin_mask, wavg, widx))
g = GS().eval()
pack = TT.packs[voice]
vol = [json.loads(l) for l in open("/opt/textside/trace_gb.jsonl" if voice in GB else "/opt/textside/trace_us.jsonl") if l.strip()]
tok = wf.tok
od = Path(f"{VD}/export/{name}"); od.mkdir(parents=True, exist_ok=True)
man = []
for k, r in enumerate(vol): # the trace line's inputs, one set per sentence (in<k>_<name>.bin + raw_inputs.json)
words = (r["text"] or "").split(); pre = (r["prev"] + " ") if r.get("prev") else ""
enc = tok(pre + " ".join(words), return_offsets_mapping=True); off, pos = enc["offset_mapping"], len(pre); S = len(enc["input_ids"])
A = np.zeros((1, max(1, len(words)), S), np.float32)
for i, w in enumerate(words):
a, b = pos, pos + len(w); pos += len(w) + 1
idx = [t for t, (s_, e_) in enumerate(off) if e_ > s_ and s_ < b and e_ > a]
A[0, i, idx] = 1.0 / max(1, len(idx))
arr = dict(input_ids=np.array([[0] + r["ids"] + [0]], np.int64), style=pack[len(r["ids"]) - 1].float().reshape(1, 256).numpy(),
speed=np.ones(1, np.float32))
arr.update(ettin_ids=np.array([enc["input_ids"]], np.int64), ettin_mask=np.ones((1, S), np.int64), wavg=A,
widx=np.array([[r["widx"][0]] + r["widx"] + [r["widx"][-1]]], np.int64))
arr["voice_sound"] = sty.float().numpy().reshape(1, 16)
ent = []
for n_, a_ in arr.items():
a_.tofile(od / f"in{k}_{n_}.bin"); ent.append(dict(name=n_, dtype=str(a_.dtype), shape=list(a_.shape)))
man.append(ent)
(od / "raw_inputs.json").write_text(json.dumps(man))
f0 = {e["name"]: torch.from_numpy(np.fromfile(od / f"in0_{e['name']}.bin", dtype=e["dtype"]).reshape(e["shape"])) for e in man[0]}
names = [e["name"] for e in man[0]]
path = str(od / "model.fp32.onnx")
with torch.no_grad():
yt = g(*[f0[n_] for n_ in names])
dyn = {"input_ids": {1: "tokens"}, "waveform": {1: "samples"}}
dyn.update({"ettin_ids": {1: "sub"}, "ettin_mask": {1: "sub"}, "wavg": {1: "words", 2: "sub"}, "widx": {1: "tokens"}})
torch.onnx.export(g, tuple(f0[n_] for n_ in names), path, input_names=names, output_names=["waveform"], dynamic_axes=dyn, opset_version=17, dynamo=False)
import onnxruntime as ort
so = ort.SessionOptions(); so.intra_op_num_threads = 4; res = {}
s_ = ort.InferenceSession(path, so, providers=["CPUExecutionProvider"])
y = s_.run(None, {n_: f0[n_].numpy() for n_ in names})[0]
res["fp32"] = dict(mb=round(os.path.getsize(path) / 1e6, 2), samples=int(y.size), max_diff_vs_torch=float(np.abs(y.ravel()[:yt.numel()] - yt.numpy().ravel()[:y.size]).max()))
np_ = lambda m_: round(sum(p.numel() for p in m_.parameters()) / 1e6, 3) # noqa: E731
res.update(name=name, voice=voice, ck=ck, params=dict(student=np_(st), word_encoder=np_(wf.model), predictor=np_(pred), decoder=np_(dec)))
(od / "export.json").write_text(json.dumps(res, indent=1)); data.commit()
return res
@app.local_entrypoint()
def export_entry(ck: str = "", name: str = "", voice: str = "", out: str = str(WORK)):
"""defaults from config/textside.json "export" (the release: t10e step 20000, trace voice s057b); copies model.fp32.onnx +
export.json to <out>/<name>/"""
e = CFG["export"]; name = name or e["name"]
print(json.dumps(export10.remote(ck or e["ck"], name, voice or e["voice"]), indent=1))
sub = VD[len("/data/"):] if VD.startswith("/data/") else VD.lstrip("/")
od = Path(out) / name; od.mkdir(parents=True, exist_ok=True)
for fn in ("model.fp32.onnx", "export.json"):
with open(od / fn, "wb") as f:
for chunk in data.read_file(f"/{sub}/export/{name}/{fn}"):
f.write(chunk)
print("copied to", od)