Download recipe/textside/textside.py from KaleComputer/kale-tts-1-tiny: direct link, hf CLI and curl.
- Browser
- Download file 27.5 kB
-
https://huggingface.co/KaleComputer/kale-tts-1-tiny/resolve/main/recipe/textside/textside.py
- Command line
-
hf download hf://KaleComputer/kale-tts-1-tiny/recipe/textside/textside.py
-
curl -L -o textside.py https://huggingface.co/KaleComputer/kale-tts-1-tiny/resolve/main/recipe/textside/textside.py
27.5 kB
| # SPDX-License-Identifier: Apache-2.0 | |
| # Copyright 2026 Kale | |
| # Uses jhu-clsp/ettin-encoder-17m (MIT) and its tokenizer through transformers (Apache-2.0); downloaded into the image, not copied. | |
| # Uses Kokoro-82M / kokoro (Apache-2.0, Copyright hexgrad) through ../train/kline.py; ../train/onnx_export.py carries paradee portions | |
| # (Apache-2.0, ../train/third_party/paradee/LICENSE and NOTICE). | |
| """kale-tts-1-tiny's Ettin text side (Modal app): Ettin-encoder-17m word features (frozen) + a small phoneme encoder replace Kokoro's | |
| ALBERT (+ bert_encoder). Distilled from the release teacher's own text side on text only, so the predictor input (d_en), durations, | |
| pitch and loudness match; the teacher's predictor stays frozen and is reused, as are the text encoder and the t10k decoder. | |
| upload : modal run textside.py::upload_entry work/sents_us.jsonl + work/sents_gb.jsonl -> runs volume textside/ | |
| train : modal run --detach textside.py::train_entry (settings: config/textside.json "train"; 1 x H100, ~72 min for 20k steps) | |
| export : modal run textside.py::export_entry (CPU) -> runs volume textside/export/<name>/model.fp32.onnx, copied to work/ | |
| App and volume names come from ../config/modal.json ("textside_app", "runs_volume", "teacher_volume"). | |
| Volumes: /data = runs volume (../train's runs: duration head, pitch/loudness heads, tiny decoder; this step's data, runs and exports), | |
| /t = teacher volume (../teacher's export + voice packs, read-only). | |
| """ | |
| import json | |
| import os | |
| import time | |
| from datetime import datetime, timezone | |
| from pathlib import Path | |
| import modal | |
| HERE = Path(__file__).resolve().parent | |
| def _cfg(name, local, remote): | |
| for p in (os.environ.get(name) if name else None, local, Path(remote)): | |
| if p and Path(p).is_file(): | |
| return Path(p) | |
| return None | |
| _MC = _cfg("KALE_TTS_MODAL_CONFIG", HERE.parent / "config" / "modal.json", "/root/recipe/modal.json") | |
| _TC = _cfg("", HERE / "config" / "textside.json", "/root/textside/textside.json") | |
| NAMES = {"textside_app": "kale-tts-textside", "runs_volume": "kale-tts-runs", "teacher_volume": "kale-tts-teacher", | |
| **(json.loads(_MC.read_text()) if _MC else {})} | |
| CFG = json.loads(_TC.read_text()) if _TC else {} | |
| app = modal.App(NAMES["textside_app"]) | |
| data = modal.Volume.from_name(NAMES["runs_volume"], create_if_missing=True) | |
| teacher = modal.Volume.from_name(NAMES["teacher_volume"], create_if_missing=True).read_only() | |
| VOL = {"/data": data, "/t": teacher} # HF files baked into the image (no HF volume) | |
| TRAIN_DIR = HERE.parent / "train" | |
| WORK = HERE / "work" | |
| utc = lambda: datetime.now(timezone.utc).strftime("%d %b %H:%M:%S UTC") # noqa: E731 | |
| ETTIN = CFG.get("ettin_model", "jhu-clsp/ettin-encoder-17m") | |
| ETTIN_REV = CFG.get("ettin_revision") | |
| KOKORO_REV = CFG.get("kokoro_rev", "f3ff3571791e39611d31c381e3a41a3af07b4987") | |
| img = ( | |
| modal.Image.debian_slim(python_version="3.12") | |
| .apt_install("libsndfile1", "git") | |
| .pip_install("torch==2.8.0", "torchaudio==2.8.0", "transformers==4.48.3", "kokoro==0.9.4", "soundfile", "numpy<2.3", "scipy", | |
| "onnx", "onnxruntime==1.22.1", "onnxscript") | |
| .run_commands(f"python -c \"from huggingface_hub import snapshot_download as s; s('{ETTIN}', local_dir='/opt/ettin'" | |
| + (f", revision='{ETTIN_REV}'" if ETTIN_REV else "") + ")\"", | |
| f"python -c \"from huggingface_hub import hf_hub_download as d; d('hexgrad/Kokoro-82M', 'config.json', revision='{KOKORO_REV}')\"") | |
| .env({"PYTHONUNBUFFERED": "1", "HF_HUB_DISABLE_TELEMETRY": "1", "PARADEE_DIR": "/opt/paradee"}) | |
| ) | |
| if (TRAIN_DIR / "third_party" / "paradee").is_dir(): | |
| img = img.add_local_dir(str(TRAIN_DIR / "third_party" / "paradee"), "/opt/paradee", copy=True, ignore=["__pycache__"]) | |
| for _n in ("model.py", "kline.py", "onnx_export.py"): | |
| if (TRAIN_DIR / _n).is_file(): | |
| img = img.add_local_file(str(TRAIN_DIR / _n), f"/opt/recipe/{_n}", copy=True) | |
| for _n in ("trace_us.jsonl", "trace_gb.jsonl"): # the trace line's ids (make_ids.mjs), for the export | |
| if (WORK / _n).is_file(): | |
| img = img.add_local_file(str(WORK / _n), f"/opt/textside/{_n}", copy=True) | |
| if _MC is not None: | |
| img = img.add_local_file(str(_MC), "/root/recipe/modal.json", copy=True) | |
| if _TC is not None: | |
| img = img.add_local_file(str(_TC), "/root/textside/textside.json", copy=True) | |
| V10 = CFG.get("voices", "s057b,f06,d03,w165,w602,m3,t071,d13,x328,h013f") | |
| GB = tuple(CFG.get("gb_voices", ["d03", "d13"])) | |
| CLAMP10 = CFG.get("end_clamp", {"n_frames_tail": 36, "semis": 6.0, "thr": 50.0, "min_hz": 85.0}) | |
| F0_FLOOR = float(CFG.get("f0_floor", 80.0)) | |
| VD = CFG.get("volume_dir", "/data/textside") | |
| def _log(msg, path=None): | |
| line = f"{utc()} {msg}" | |
| print(line, flush=True) | |
| if path: | |
| with open(path, "a") as f: | |
| f.write(line + "\n") | |
| # ------------------------------------------------------------------ model pieces | |
| def _mods(): | |
| import torch | |
| import torch.nn as nn | |
| class WordFeat(nn.Module): | |
| """frozen Ettin-17m: per-word features (mean of the last two layers, mean over a word's sub-tokens), previous sentence as left | |
| context. Batched.""" | |
| def __init__(self, path="/opt/ettin"): | |
| super().__init__() | |
| from transformers import AutoModel, AutoTokenizer | |
| self.tok = AutoTokenizer.from_pretrained(path) | |
| self.model = AutoModel.from_pretrained(path, attn_implementation="eager").eval().requires_grad_(False) | |
| def forward(self, texts, prevs, dev): | |
| fulls, spans = [], [] | |
| for t, p in zip(texts, prevs): | |
| words = t.split(); pre = (p.strip() + " ") if p and p.strip() else "" | |
| sp, pos = [], len(pre) | |
| for w in words: | |
| sp.append((pos, pos + len(w))); pos += len(w) + 1 | |
| fulls.append(pre + " ".join(words)); spans.append(sp) | |
| enc = self.tok(fulls, return_offsets_mapping=True, return_tensors="pt", padding=True, truncation=True, max_length=512) | |
| off = enc.pop("offset_mapping").tolist() | |
| hs = self.model(**{k: v.to(dev) for k, v in enc.items()}, output_hidden_states=True).hidden_states | |
| h = ((hs[-1] + hs[-2]) / 2).float() # [B, Ttok, 256] | |
| W = max(len(s) for s in spans) | |
| out = torch.zeros(len(texts), max(W, 1), h.shape[-1], device=dev) | |
| for b, sp in enumerate(spans): | |
| for i, (a, e) in enumerate(sp): | |
| idx = [t for t, (s, f) in enumerate(off[b]) if f > s and s < e and f > a] | |
| if idx: | |
| out[b, i] = h[b, idx].mean(0) | |
| return out | |
| class ConvBlock(nn.Module): | |
| def __init__(self, c, k=5): | |
| super().__init__() | |
| self.conv = nn.Conv1d(c, c, k, padding=k // 2); self.norm = nn.LayerNorm(c); self.act = nn.GELU() | |
| def forward(self, x, m): # x [B,T,C], m [B,T] True = pad | |
| y = self.conv(x.masked_fill(m[..., None], 0).transpose(1, 2)).transpose(1, 2) | |
| return x + self.act(self.norm(y)) | |
| class TFBlock(nn.Module): | |
| """pre-norm transformer block with hand-written attention (exports with dynamic length; nn.MultiheadAttention's legacy export | |
| bakes the length)""" | |
| def __init__(self, c, nh=4, drop=0.1): | |
| super().__init__() | |
| self.n1 = nn.LayerNorm(c); self.qkv = nn.Linear(c, 3 * c); self.o = nn.Linear(c, c); self.nh = nh | |
| self.n2 = nn.LayerNorm(c); self.ff = nn.Sequential(nn.Linear(c, 4 * c), nn.GELU(), nn.Linear(4 * c, c)); self.drop = nn.Dropout(drop) | |
| def forward(self, x, m): # x [B,T,C], m [B,T] True = pad | |
| B, T, C = x.shape; hd = C // self.nh | |
| q, k, v = self.qkv(self.n1(x)).reshape(B, T, 3, self.nh, hd).permute(2, 0, 3, 1, 4) | |
| a = (q @ k.transpose(-1, -2)) * (hd ** -0.5) + m[:, None, None, :].float() * -1e4 | |
| y = (a.softmax(-1) @ v).transpose(1, 2).reshape(B, T, C) | |
| x = x + self.drop(self.o(y)) | |
| return x + self.drop(self.ff(self.n2(x))) | |
| class K20Text(nn.Module): | |
| """replaces bert + bert_encoder: phoneme embedding -> 3 conv blocks -> + Ettin word features (gathered per token) -> 2 transformer | |
| layers -> 512-d d_en (Kokoro's predictor input).""" | |
| def __init__(self, c=256, nconv=3, ntf=2, wdim=256, out=512): | |
| super().__init__() | |
| self.emb = nn.Embedding(178, c) | |
| self.convs = nn.ModuleList([ConvBlock(c) for _ in range(nconv)]) | |
| self.wproj = nn.Sequential(nn.Linear(wdim, c), nn.GELU(), nn.Linear(c, c)) | |
| self.pos = nn.Embedding(512, c) | |
| self.tf = nn.ModuleList([TFBlock(c) for _ in range(ntf)]) | |
| self.out = nn.Linear(c, out) | |
| def forward(self, x, m, wfeat, widx): # x [B,T] ids, wfeat [B,W,256], widx [B,T] | |
| h = self.emb(x) | |
| for b in self.convs: | |
| h = b(h, m) | |
| w = torch.gather(wfeat, 1, widx[..., None].expand(-1, -1, wfeat.shape[-1])) | |
| h = h + self.wproj(w) + self.pos(torch.arange(x.shape[1], device=x.device))[None] | |
| for b in self.tf: | |
| h = b(h, m) | |
| return self.out(h) # [B,T,512] | |
| return WordFeat, K20Text | |
| def _teacher(dev): | |
| """the release tiny's text side -> T: .K (kline module), .km (Kokoro modules), .packs (voice pack + the voice's pitch offset), | |
| .voices, .pred (the predictor the student feeds), .out_dim (d_en width), .d_en(x, m) -> [B,T,C] (the part the student replaces: | |
| ALBERT + bert_encoder), .style(voice, n_ids) -> [128] (the pack row's prosody half).""" | |
| import sys | |
| import types | |
| import torch | |
| sys.path.insert(0, "/opt/recipe") | |
| T = types.SimpleNamespace() | |
| os.environ.update(KLINE_EXPORT=CFG["teacher_export"], KLINE_VOICES=V10, KLINE_PACKS=CFG["teacher_packs"]) | |
| import kline as K | |
| km, _ = K.load_kmodel(dev, CFG["dur_ckpt"], CFG["pros_ckpt"]) | |
| packs = K.load_packs(dev); pk = torch.load(CFG["pros_ckpt"], map_location=dev, weights_only=False); out = {} | |
| for v in K.VOICES: | |
| p = packs[v].clone(); p[..., 128:] += pk["pv"][pk["voices"].index(v)].to(p.device); out[v] = p | |
| T.K, T.km, T.packs, T.voices, T.pred, T.out_dim = K, km, out, list(out), km.predictor, 512 | |
| T.d_en = lambda x, m: km.bert_encoder(km.bert(x, attention_mask=(~m).int())) | |
| def _st(v, n): | |
| p = out[v][min(n, len(out[v])) - 1] | |
| return p[0, 128:] if p.dim() == 2 else p[128:] | |
| T.style = _st | |
| return T | |
| def _batch(rows, dev): | |
| """rows {ids, widx, text, prev} -> padded ids with Kokoro's 0 pads at both ends, mask, widx (pads -> word 0 / last word)""" | |
| import torch | |
| T = max(len(r["ids"]) for r in rows) + 2 | |
| x = torch.zeros(len(rows), T, dtype=torch.long); wi = torch.zeros(len(rows), T, dtype=torch.long) | |
| L = torch.LongTensor([len(r["ids"]) + 2 for r in rows]) | |
| for b, r in enumerate(rows): | |
| x[b, 1:1 + len(r["ids"])] = torch.LongTensor(r["ids"]) | |
| w = r["widx"]; wi[b, 1:1 + len(w)] = torch.LongTensor(w); wi[b, 1 + len(w)] = w[-1] if w else 0 | |
| m = torch.arange(T)[None] >= L[:, None] | |
| return x.to(dev), m.to(dev), wi.to(dev), L.to(dev) | |
| # ------------------------------------------------------------------ training | |
| def train(run: str, steps: int = 20000, bs: int = 64, lr: float = 3e-4, w_feat: float = 1.0, w_dur: float = 1.0, w_f0: float = 1.0, | |
| w_n: float = 1.0, ckpt_every: int = 5000, resume: bool = True, f0_log: bool = True, held_n: int = 256, min_ids: int = 4, | |
| max_ids: int = 400): | |
| """text-only distillation. Teacher (frozen): d_en = bert_encoder(bert(x)), durations (t10d2 head), F0 / N (t10p heads) on the | |
| teacher's rounded durations. Student: K20Text -> d_en; the teacher's predictor (frozen) on the student d_en -> durations, F0, N under | |
| the TEACHER's alignment. A random voice per sentence (its pack row + pitch offset as style); d03 / d13 sentences use en-GB ids + word | |
| map. Losses: L1 d_en, L1 durations, F0 (f0_log: semitones on the teacher's voiced frames / 12 + Hz/100 x 0.5 on its unvoiced ones; | |
| else L1 Hz / 100), L1 N.""" | |
| import random | |
| import torch | |
| torch.backends.cuda.matmul.allow_tf32 = True | |
| dev = "cuda" | |
| od = Path(f"{VD}/{run}"); od.mkdir(parents=True, exist_ok=True); lg = str(od / "train.log") | |
| rows = [json.loads(l) for l in open(f"{VD}/{CFG.get('sents', 'sents.jsonl')}")] | |
| gbr = {} | |
| for l in open(f"{VD}/{CFG.get('sents_gb', 'sents_gb.jsonl')}"): # d03 / d13 read en-gb ids | |
| r = json.loads(l); gbr[r["text"]] = r | |
| rows = [r for r in rows if min_ids <= len(r["ids"]) <= max_ids and r["text"] in gbr] | |
| random.Random(1).shuffle(rows); held, rows = rows[:held_n], rows[held_n:] | |
| TT = _teacher(dev) | |
| vnames = TT.voices | |
| WordFeat, K20Text = _mods() | |
| wf = WordFeat().to(dev) | |
| st = K20Text(out=TT.out_dim).to(dev) | |
| npar = sum(p.numel() for p in st.parameters()) | |
| pred = TT.pred | |
| for mod in pred.modules(): # cuDNN LSTM backward needs train mode; single-layer LSTMs have no dropout, so outputs are unchanged | |
| if isinstance(mod, torch.nn.LSTM): | |
| assert mod.num_layers == 1; mod.train() | |
| opt = torch.optim.AdamW(st.parameters(), lr=lr, weight_decay=0.01) | |
| step0 = 0 | |
| if resume and (od / "last.pt").exists(): | |
| ck = torch.load(od / "last.pt", map_location=dev, weights_only=False) | |
| st.load_state_dict(ck["model"]); opt.load_state_dict(ck["opt"]); step0 = ck["step"] | |
| sched = torch.optim.lr_scheduler.LambdaLR(opt, lambda s: min(1.0, (s + 1) / 1000) * max(0.05, 0.5 * (1 + __import__("math").cos(3.14159 * min(1.0, s / steps))))) | |
| for _ in range(step0): | |
| sched.step() | |
| _log(f"teacher: voices {vnames}", lg) | |
| _log(f"train {run}: {len(rows)} sentences, held {len(held)}, student {npar / 1e6:.2f} M params, from step {step0}", lg) | |
| def teacher_and_student(b, train_mode): | |
| vs = [random.choice(vnames) for _ in b] # a random voice per sentence | |
| b = [gbr[r["text"]] if v in GB else r for r, v in zip(b, vs)] # British voices: en-gb ids + word map | |
| x, m, wi, L = _batch(b, dev) | |
| s = torch.stack([TT.style(v, len(r["ids"])) for v, r in zip(vs, b)]).to(dev).float() | |
| with torch.no_grad(): | |
| d_en_t = TT.d_en(x, m) # [B,T,C] | |
| d_t = pred.text_encoder(d_en_t.transpose(-1, -2), s, L, m) | |
| xx, _ = pred.lstm(d_t) | |
| dur_t = torch.sigmoid(pred.duration_proj(xx)).sum(-1).masked_fill(m, 0) # [B,T] | |
| n = torch.round(dur_t).clamp(min=1).long().masked_fill(m, 0) | |
| Fr = int(n.sum(1).max()) | |
| aln = torch.zeros(x.shape[0], x.shape[1], Fr, device=dev) | |
| fm = torch.zeros(x.shape[0], Fr, dtype=torch.bool, device=dev) | |
| for bi in range(x.shape[0]): | |
| idx = torch.repeat_interleave(torch.arange(x.shape[1], device=dev), n[bi]) | |
| aln[bi, idx, torch.arange(len(idx), device=dev)] = 1; fm[bi, :len(idx)] = True | |
| F0_t, N_t = pred.F0Ntrain(d_t.transpose(-1, -2) @ aln, s) | |
| wfeat = wf([r["text"] for r in b], [r["prev"] for r in b], dev) | |
| with torch.set_grad_enabled(train_mode): | |
| d_en_s = st(x, m, wfeat, wi) | |
| d_s = pred.text_encoder(d_en_s.transpose(-1, -2), s, L, m) | |
| xs, _ = pred.lstm(d_s) | |
| dur_s = torch.sigmoid(pred.duration_proj(xs)).sum(-1) | |
| F0_s, N_s = pred.F0Ntrain(d_s.transpose(-1, -2) @ aln, s) | |
| tm = ~m | |
| l_feat = (d_en_s - d_en_t).abs().mean(-1)[tm].mean() | |
| l_dur = (dur_s - dur_t).abs()[tm].mean() | |
| ff = fm.repeat_interleave(F0_s.shape[1] // fm.shape[1], dim=1)[:, :F0_s.shape[1]] # F0 / N run at 2x the frame rate | |
| if f0_log: # semitones on the teacher's voiced frames (a 5 Hz miss is 1 st at 90 Hz, 0.4 st at 220 Hz), Hz/100 on its unvoiced ones | |
| vo = ff & (F0_t > 40) | |
| l_f0 = (12 * torch.log2(F0_s.clamp(min=20) / F0_t.clamp(min=20))).abs()[vo].mean() / 12 + (F0_s.abs() / 100)[ff & ~vo].mean() * 0.5 | |
| else: | |
| l_f0 = ((F0_s - F0_t).abs() / 100)[ff].mean() | |
| l_n = (N_s - N_t).abs()[ff].mean() | |
| return l_feat, l_dur, l_f0, l_n | |
| t0 = time.time(); acc = [] | |
| for step in range(step0, steps): | |
| st.train() | |
| b = random.sample(rows, bs) | |
| l_feat, l_dur, l_f0, l_n = teacher_and_student(b, True) | |
| loss = w_feat * l_feat + w_dur * l_dur + w_f0 * l_f0 + w_n * l_n | |
| opt.zero_grad(set_to_none=True); loss.backward() | |
| torch.nn.utils.clip_grad_norm_(st.parameters(), 1.0); opt.step(); sched.step() | |
| acc.append([l_feat.item(), l_dur.item(), l_f0.item(), l_n.item()]) | |
| if (step + 1) % 200 == 0: | |
| a = [sum(c) / len(c) for c in zip(*acc)]; acc = [] | |
| st.eval(); hv = [] | |
| if (step + 1) % 1000 == 0: | |
| with torch.no_grad(): | |
| for i in range(0, len(held), 64): | |
| hv.append([v.item() for v in teacher_and_student(held[i:i + 64], False)]) | |
| hv = [round(sum(c) / len(c), 4) for c in zip(*hv)] | |
| _log(f"step {step + 1} feat {a[0]:.4f} dur {a[1]:.4f} f0 {a[2]:.4f} N {a[3]:.4f}" + (f" | held {hv}" if hv else "") + | |
| f" | {(time.time() - t0) / 60:.1f} min", lg) | |
| if (step + 1) % ckpt_every == 0 or step + 1 == steps: | |
| ck = dict(model=st.state_dict(), opt=opt.state_dict(), step=step + 1, pred=None, params=npar) | |
| torch.save(ck, od / f"step{step + 1:06d}.pt"); torch.save(ck, od / "last.pt"); data.commit() | |
| _log(f"done {run} {steps} steps {(time.time() - t0) / 60:.1f} min", lg) | |
| data.commit() | |
| return dict(run=run, steps=steps, params=npar) | |
| def train_entry(run: str = "", steps: int = 0): | |
| """defaults from config/textside.json "train" (the release run: t10e, 20k steps, f0_log)""" | |
| t = dict(CFG["train"]); t = {k: v for k, v in t.items() if not k.startswith("_")} | |
| print(train.remote(run or t["run"], steps or t["steps"], t["bs"], t["lr"], t["w_feat"], t["w_dur"], t["w_f0"], t["w_n"], | |
| ckpt_every=t["ckpt_every"], f0_log=t["f0_log"], held_n=t["held"], min_ids=t["min_ids"], max_ids=t["max_ids"])) | |
| def upload_entry(us: str = str(WORK / "sents_us.jsonl"), gb: str = str(WORK / "sents_gb.jsonl")): | |
| """make_ids.mjs outputs -> runs volume <volume_dir>/sents.jsonl, sents_gb.jsonl""" | |
| sub = VD[len("/data/"):] if VD.startswith("/data/") else VD.lstrip("/") | |
| with data.batch_upload(force=True) as b: | |
| b.put_file(us, f"/{sub}/{CFG.get('sents', 'sents.jsonl')}"); b.put_file(gb, f"/{sub}/{CFG.get('sents_gb', 'sents_gb.jsonl')}") | |
| print("uploaded to", sub) | |
| # ------------------------------------------------------------------ export: one graph = Ettin + student + the predictor / text encoder / decoder | |
| def clamp_onnx(F0, n_frames_tail=36, semis=6.0, thr=50.0, min_hz=85.0): | |
| """kline end_clamp with graph ops only (same result): lower median of the voiced F0 via sort, last voiced frame via a masked max.""" | |
| import torch | |
| f = F0[0]; v = f > thr; cnt = v.sum() | |
| srt = torch.sort(torch.where(v, f, torch.full_like(f, -1e9)), descending=True).values | |
| k = cnt - 1 - torch.div(cnt - 1, 2, rounding_mode="floor") # lower median (ascending (n-1)//2) in descending order | |
| med = srt.gather(0, k.clamp(min=0).reshape(1))[0] | |
| floor = torch.clamp(med * 2 ** (-semis / 12), min=min_hz) | |
| ar = torch.arange(f.shape[0], device=f.device) | |
| last = torch.where(v, ar, torch.zeros_like(ar)).max() | |
| tail = (ar >= last - n_frames_tail) & (ar <= last) | |
| return torch.where(v & tail & (cnt >= 5), torch.maximum(f, floor), f)[None] | |
| def export10(ck: str, name: str, voice: str = "s057b"): | |
| """one fp32 graph for all voices: inputs input_ids [1,T], style [1,256], speed [1], ettin_ids [1,S], ettin_mask [1,S], wavg [1,W,S] | |
| (mean over each word's sub-tokens), widx [1,T] (word per phoneme token), voice_sound [1,16] (the decoder's style row) -> waveform. | |
| The end clamp (per-sentence floor) and the 80 Hz voicing floor are in the graph. `voice` only picks the trace / test inputs.""" | |
| import sys | |
| import numpy as np | |
| import torch | |
| import torch.nn as nn | |
| sys.path.insert(0, "/opt/recipe") | |
| import onnx_export as X | |
| torch.set_num_threads(8); torch.backends.mha.set_fastpath_enabled(False) | |
| TT = _teacher("cpu"); K = TT.K; vid = K.VOICES.index(voice) | |
| WordFeat, K20Text = _mods() | |
| c = torch.load(ck, map_location="cpu", weights_only=False); st = K20Text(out=TT.out_dim).eval(); st.load_state_dict(c["model"]) | |
| wf = WordFeat() | |
| kd = K.KDec("k").eval(); kd.load_state_dict(torch.load(CFG["kdec_ckpt"], map_location="cpu", weights_only=False)["model"]); X.strip(kd); X.strip(TT.km) | |
| dec, sty, te = kd.dec, kd.style[vid:vid + 1].detach(), TT.km.text_encoder | |
| pred = TT.pred; X.strip(pred) | |
| class G(nn.Module): | |
| def __init__(self): | |
| super().__init__(); self.pred, self.st, self.dec, self.te = pred, st, dec, te; self.head = X.HeadX(dec.head) | |
| self.et = wf.model; self.register_buffer("sty", sty) | |
| def textside(self, input_ids, m, ettin_ids, ettin_mask, wavg, widx): | |
| hs = self.et(input_ids=ettin_ids, attention_mask=ettin_mask, output_hidden_states=True).hidden_states; h = (hs[-1] + hs[-2]) / 2 | |
| return self.st(input_ids, m, wavg @ h, widx) | |
| def body(self, input_ids, s, speed, d_en): | |
| p = self.pred; d_en = d_en.transpose(-1, -2) | |
| T = input_ids.shape[1]; sx = s[:, :, None].expand(-1, -1, T); x = torch.cat([d_en, sx], 1) | |
| for b in p.text_encoder.lstms: | |
| if isinstance(b, nn.LSTM): x = b(x.transpose(-1, -2))[0].transpose(-1, -2) | |
| else: x = torch.cat([b(x.transpose(-1, -2), s).transpose(-1, -2), sx], 1) | |
| d = x.transpose(-1, -2) | |
| dur = torch.sigmoid(p.duration_proj(p.lstm(d)[0])).sum(-1)[0] | |
| n = torch.clamp(torch.round(dur / speed), min=1).long() | |
| end = torch.cumsum(n, 0); start = end - n; f = torch.arange(end[-1])[None] | |
| aln = ((f >= start[:, None]) & (f < end[:, None])).float()[None] | |
| F0, N = p.F0Ntrain(d.transpose(-1, -2) @ aln, s) | |
| F0 = clamp_onnx(F0, **CLAMP10); F0 = torch.where(F0 < F0_FLOOR, torch.zeros_like(F0), F0) | |
| e = self.te; hh = e.embedding(input_ids).transpose(1, 2) | |
| for c_ in e.cnn: hh = c_(hh) | |
| t_en = e.lstm(hh.transpose(1, 2))[0].transpose(-1, -2) | |
| sty = getattr(self, "sty_in", None) | |
| return self.head(self.dec.trunk(t_en @ aln, F0, N, self.sty if sty is None else sty), F0) | |
| class GS(G): # one file for all voices: the decoder's style row is an input (voice_sound [1,16]) | |
| def forward(self, input_ids, style, speed, ettin_ids, ettin_mask, wavg, widx, voice_sound): | |
| m = torch.zeros_like(input_ids, dtype=torch.bool) | |
| self.sty_in = voice_sound | |
| return self.body(input_ids, style[:, 128:], speed, self.textside(input_ids, m, ettin_ids, ettin_mask, wavg, widx)) | |
| g = GS().eval() | |
| pack = TT.packs[voice] | |
| vol = [json.loads(l) for l in open("/opt/textside/trace_gb.jsonl" if voice in GB else "/opt/textside/trace_us.jsonl") if l.strip()] | |
| tok = wf.tok | |
| od = Path(f"{VD}/export/{name}"); od.mkdir(parents=True, exist_ok=True) | |
| man = [] | |
| for k, r in enumerate(vol): # the trace line's inputs, one set per sentence (in<k>_<name>.bin + raw_inputs.json) | |
| words = (r["text"] or "").split(); pre = (r["prev"] + " ") if r.get("prev") else "" | |
| enc = tok(pre + " ".join(words), return_offsets_mapping=True); off, pos = enc["offset_mapping"], len(pre); S = len(enc["input_ids"]) | |
| A = np.zeros((1, max(1, len(words)), S), np.float32) | |
| for i, w in enumerate(words): | |
| a, b = pos, pos + len(w); pos += len(w) + 1 | |
| idx = [t for t, (s_, e_) in enumerate(off) if e_ > s_ and s_ < b and e_ > a] | |
| A[0, i, idx] = 1.0 / max(1, len(idx)) | |
| arr = dict(input_ids=np.array([[0] + r["ids"] + [0]], np.int64), style=pack[len(r["ids"]) - 1].float().reshape(1, 256).numpy(), | |
| speed=np.ones(1, np.float32)) | |
| arr.update(ettin_ids=np.array([enc["input_ids"]], np.int64), ettin_mask=np.ones((1, S), np.int64), wavg=A, | |
| widx=np.array([[r["widx"][0]] + r["widx"] + [r["widx"][-1]]], np.int64)) | |
| arr["voice_sound"] = sty.float().numpy().reshape(1, 16) | |
| ent = [] | |
| for n_, a_ in arr.items(): | |
| a_.tofile(od / f"in{k}_{n_}.bin"); ent.append(dict(name=n_, dtype=str(a_.dtype), shape=list(a_.shape))) | |
| man.append(ent) | |
| (od / "raw_inputs.json").write_text(json.dumps(man)) | |
| f0 = {e["name"]: torch.from_numpy(np.fromfile(od / f"in0_{e['name']}.bin", dtype=e["dtype"]).reshape(e["shape"])) for e in man[0]} | |
| names = [e["name"] for e in man[0]] | |
| path = str(od / "model.fp32.onnx") | |
| with torch.no_grad(): | |
| yt = g(*[f0[n_] for n_ in names]) | |
| dyn = {"input_ids": {1: "tokens"}, "waveform": {1: "samples"}} | |
| dyn.update({"ettin_ids": {1: "sub"}, "ettin_mask": {1: "sub"}, "wavg": {1: "words", 2: "sub"}, "widx": {1: "tokens"}}) | |
| torch.onnx.export(g, tuple(f0[n_] for n_ in names), path, input_names=names, output_names=["waveform"], dynamic_axes=dyn, opset_version=17, dynamo=False) | |
| import onnxruntime as ort | |
| so = ort.SessionOptions(); so.intra_op_num_threads = 4; res = {} | |
| s_ = ort.InferenceSession(path, so, providers=["CPUExecutionProvider"]) | |
| y = s_.run(None, {n_: f0[n_].numpy() for n_ in names})[0] | |
| res["fp32"] = dict(mb=round(os.path.getsize(path) / 1e6, 2), samples=int(y.size), max_diff_vs_torch=float(np.abs(y.ravel()[:yt.numel()] - yt.numpy().ravel()[:y.size]).max())) | |
| np_ = lambda m_: round(sum(p.numel() for p in m_.parameters()) / 1e6, 3) # noqa: E731 | |
| res.update(name=name, voice=voice, ck=ck, params=dict(student=np_(st), word_encoder=np_(wf.model), predictor=np_(pred), decoder=np_(dec))) | |
| (od / "export.json").write_text(json.dumps(res, indent=1)); data.commit() | |
| return res | |
| def export_entry(ck: str = "", name: str = "", voice: str = "", out: str = str(WORK)): | |
| """defaults from config/textside.json "export" (the release: t10e step 20000, trace voice s057b); copies model.fp32.onnx + | |
| export.json to <out>/<name>/""" | |
| e = CFG["export"]; name = name or e["name"] | |
| print(json.dumps(export10.remote(ck or e["ck"], name, voice or e["voice"]), indent=1)) | |
| sub = VD[len("/data/"):] if VD.startswith("/data/") else VD.lstrip("/") | |
| od = Path(out) / name; od.mkdir(parents=True, exist_ok=True) | |
| for fn in ("model.fp32.onnx", "export.json"): | |
| with open(od / fn, "wb") as f: | |
| for chunk in data.read_file(f"/{sub}/export/{name}/{fn}"): | |
| f.write(chunk) | |
| print("copied to", od) | |