Download recipe/train/model.py from KaleComputer/kale-tts-1-tiny: direct link, hf CLI and curl.
- Browser
- Download file 9.07 kB
-
https://huggingface.co/KaleComputer/kale-tts-1-tiny/resolve/main/recipe/train/model.py
- Command line
-
hf download hf://KaleComputer/kale-tts-1-tiny/recipe/train/model.py
-
curl -L -o model.py https://huggingface.co/KaleComputer/kale-tts-1-tiny/resolve/main/recipe/train/model.py
9.07 kB
| # SPDX-License-Identifier: Apache-2.0 | |
| # Copyright 2026 Kale | |
| # Portions: the decoder losses spec_losses / harm_loss / mod_loss (from paradee train_vhead.py), paradee (Apache-2.0, | |
| # github.com/sahilmahendrakar/paradee @ 2de547e4), see third_party/paradee/LICENSE and NOTICE. | |
| # Dec follows Kokoro's istftnet Decoder layout (kokoro, Apache-2.0, Copyright hexgrad) built from Kokoro's AdainResBlk1d; | |
| # SLMJudge follows StyleTTS2's speech-model discriminator layout (StyleTTS2, MIT, Copyright (c) 2023 Aaron (Yinghao) Li). | |
| """kale-tts-1 model pieces shared by the trainers (train.py, kline.py) and the ONNX export (onnx_export.py). | |
| Dec : Kokoro-style AdaIN trunk + a frame-rate source-filter head (64-harmonic sine source + noise + iSTFT, 24 kHz, no separate | |
| vocoder), conditioned on an external style row. The head (SourceFilterHead2) and the trunk width presets (DEC_PRESETS) | |
| come from paradee (Apache-2.0, vendored unchanged in third_party/paradee/, see its NOTICE). | |
| mas_batch : monotonic alignment search (Glow-TTS rule), used on the teacher aligner's attention for the training durations. | |
| SLMJudge : StyleTTS2's speech-model discriminator layout on frozen HuBERT-base-ls960 hidden states. | |
| spec_losses / harm_loss / mod_loss: the decoder's spectral, harmonic and modulation losses (paradee train_vhead.py maths, unchanged). | |
| """ | |
| import os | |
| import sys | |
| from pathlib import Path | |
| import numpy as np | |
| import torch | |
| import torch.nn as nn | |
| import torch.nn.functional as F | |
| from torch.nn.utils import weight_norm | |
| sys.path.insert(0, os.environ.get("PARADEE_DIR", str(Path(__file__).resolve().parent / "third_party" / "paradee"))) # vendored paradee (Apache-2.0, third_party/paradee/NOTICE) | |
| from kokoro.istftnet import AdainResBlk1d # noqa: E402 | |
| from student_decoder import DEC_PRESETS # noqa: E402 | |
| from vhead import SourceFilterHead2 # noqa: E402 | |
| N_TOK = 178 | |
| HOP = 600 # one sound-feature frame = 600 samples (40 frames/s), pitch/loudness frames = 300 samples | |
| # ---------------------------------------------------------------- decoder: Kokoro-style AdaIN trunk + external source-filter head, external style | |
| class Dec(nn.Module): | |
| def __init__(self, preset, dim_in, style_dim, hdim, hblk): | |
| super().__init__() | |
| W, uic, _ = DEC_PRESETS[preset] | |
| self.encode = AdainResBlk1d(dim_in + 2, W, style_dim) | |
| self.decode = nn.ModuleList([AdainResBlk1d(W + 2 + 64, W, style_dim) for _ in range(3)] | |
| + [AdainResBlk1d(W + 2 + 64, uic, style_dim, upsample=True)]) | |
| self.F0_conv = weight_norm(nn.Conv1d(1, 1, kernel_size=3, stride=2, padding=1)) | |
| self.N_conv = weight_norm(nn.Conv1d(1, 1, kernel_size=3, stride=2, padding=1)) | |
| self.asr_res = nn.Sequential(weight_norm(nn.Conv1d(dim_in, 64, kernel_size=1))) | |
| self.head = SourceFilterHead2(in_ch=uic, dim=hdim, blocks=hblk) | |
| def trunk(self, asr, F0_curve, N, s): | |
| F0 = self.F0_conv(F0_curve.unsqueeze(1)) | |
| Nn = self.N_conv(N.unsqueeze(1)) | |
| x = self.encode(torch.cat([asr, F0, Nn], 1), s) | |
| asr_res = self.asr_res(asr) | |
| res = True | |
| for block in self.decode: | |
| if res: | |
| x = torch.cat([x, asr_res, F0, Nn], 1) | |
| x = block(x, s) | |
| if block.upsample_type != "none": | |
| res = False | |
| return x | |
| def forward(self, asr, F0_curve, N, s, noise=None, c0=None): | |
| return self.head(self.trunk(asr, F0_curve, N, s), F0_curve, noise, c0) | |
| # ---------------------------------------------------------------- monotonic alignment search (ours) | |
| def mas_batch(ll, tl, fl): | |
| """monotonic alignment search (Glow-TTS rule) per item on a [B,T,F] log-likelihood (numpy float32) -> list of durations.""" | |
| from numba import njit, prange | |
| global _mas | |
| if "_mas" not in globals(): | |
| def _m(ll, tl, fl, out): | |
| for b in prange(ll.shape[0]): | |
| tx, ty = tl[b], fl[b] | |
| v = np.full((tx, ty), -1e30, dtype=np.float64) | |
| for y in range(ty): | |
| lo = max(0, tx + y - ty) | |
| hi = min(tx, y + 1) | |
| for x in range(lo, hi): | |
| if y == 0: | |
| v[x, y] = ll[b, x, y] if x == 0 else -1e30 | |
| else: | |
| a = v[x, y - 1] | |
| c = v[x - 1, y - 1] if x > 0 else -1e30 | |
| v[x, y] = ll[b, x, y] + (a if a > c else c) | |
| idx = tx - 1 | |
| for y in range(ty - 1, -1, -1): | |
| out[b, idx] += 1 | |
| if idx != 0 and (idx == y or v[idx, y - 1] < v[idx - 1, y - 1]): | |
| idx -= 1 | |
| _mas = _m | |
| out = np.zeros((ll.shape[0], ll.shape[1]), dtype=np.int32) | |
| _mas(ll, np.asarray(tl, dtype=np.int64), np.asarray(fl, dtype=np.int64), out) | |
| return [out[b, : tl[b]] for b in range(len(tl))] | |
| # ---------------------------------------------------------------- speech-model judge (StyleTTS2 layout, HuBERT features) | |
| class SLMJudge(nn.Module): | |
| def __init__(self, hidden=768, layers=13, ch=64): | |
| super().__init__() | |
| self.pre = weight_norm(nn.Conv1d(hidden * layers, ch, 1)) | |
| self.convs = nn.ModuleList([weight_norm(nn.Conv1d(ch, ch * 2, 5, padding=2)), | |
| weight_norm(nn.Conv1d(ch * 2, ch * 4, 5, padding=2)), | |
| weight_norm(nn.Conv1d(ch * 4, ch * 4, 5, padding=2))]) | |
| self.post = weight_norm(nn.Conv1d(ch * 4, 1, 3, padding=1)) | |
| def forward(self, h): # h [B, layers*hidden, T] | |
| x = self.pre(h) | |
| for c in self.convs: | |
| x = F.leaky_relu(c(x), 0.2) | |
| return self.post(x).flatten(1) | |
| def slm_hidden(hubert, wav24): | |
| import torchaudio.functional as AF | |
| w = AF.resample(wav24, 24000, 16000) | |
| hs = hubert(w, output_hidden_states=True).hidden_states | |
| return torch.stack(hs, 1) # [B, 13, T, 768] | |
| # ---------------------------------------------------------------- losses (Paradee train_vhead.py, unchanged maths) | |
| _mel_cache = {} | |
| def spec_losses(y, t, fine=True): | |
| import torchaudio | |
| dev = y.device | |
| if dev not in _mel_cache: | |
| _mel_cache[dev] = torchaudio.transforms.MelSpectrogram(24000, n_fft=1024, hop_length=256, n_mels=80).to(dev) | |
| mel = _mel_cache[dev] | |
| lm = F.l1_loss(torch.log(mel(y).clamp_min(1e-5)), torch.log(mel(t).clamp_min(1e-5))) | |
| ls = 0 | |
| SZ = (64, 128, 512, 1024, 2048) if fine else (512, 1024, 2048) | |
| for n in SZ: | |
| w = torch.hann_window(n, device=dev) | |
| Y = torch.stft(y, n, n // 4, window=w, return_complex=True).abs() | |
| T = torch.stft(t, n, n // 4, window=w, return_complex=True).abs() | |
| ls = ls + (T - Y).norm() / T.norm().clamp_min(1e-5) + F.l1_loss(torch.log(Y.clamp_min(1e-5)), torch.log(T.clamp_min(1e-5))) | |
| return lm, ls / len(SZ) | |
| def harm_loss(y, t, f0, n=2048, hop=512): | |
| DEV = y.device | |
| w = torch.hann_window(n, device=DEV) | |
| nfr = y.shape[-1] // hop + 1 | |
| idx = (torch.arange(nfr, device=DEV) * hop // 300).clamp(max=f0.shape[1] - 1) | |
| lo = (idx - 2).clamp(min=0) | |
| hi = (idx + 2).clamp(max=f0.shape[1] - 1) | |
| f = f0[:, idx] | |
| v = (torch.minimum(torch.minimum(f0[:, lo], f), f0[:, hi]) > 100).float() | |
| fr = torch.arange(n // 2 + 1, device=DEV) * 24000 / n | |
| ratio = fr[None, :, None] / f.clamp_min(60)[:, None] | |
| frac = (ratio - ratio.round()).abs() | |
| mh, mi = (frac < 0.15).float(), ((frac - 0.5).abs() < 0.15).float() | |
| bands = torch.stack([((fr >= 1000 * b) & (fr < 1000 * (b + 1))).float() for b in range(1, 9)]) | |
| out = [] | |
| for x in (y, t): | |
| P = torch.stft(x, n, hop, window=w, return_complex=True).abs().pow(2) | |
| out.append(torch.log((torch.einsum("bft,kf->bkt", P * mh, bands) + 1e-8) / (torch.einsum("bft,kf->bkt", P * mi, bands) + 1e-8))) | |
| return ((out[0] - out[1]).abs() * v[:, None]).sum() / (v.sum() * 8).clamp_min(1) | |
| MODB = [(4, 8), (8, 12), (12, 16), (16, 21), (21, 29)] | |
| def mod_loss(y, t, cyc, uv, n=64, hop=16, win=75): | |
| DEV = y.device | |
| w = torch.hann_window(n, device=DEV) | |
| k = y.shape[-1] // hop + 1 | |
| ph = 2 * torch.pi * F.pad(cyc, (0, hop))[:, ::hop][:, :k] | |
| v = F.pad(uv, (0, hop))[:, ::hop][:, :k] | |
| carr = torch.stack([fn(m * ph) for m in (1, 2) for fn in (torch.cos, torch.sin)], 1) | |
| D = [] | |
| for x in (y, t): | |
| X = torch.stft(x, n, hop, window=w, return_complex=True).abs() | |
| e = torch.stack([X[:, a:b].sum(1) for a, b in MODB], 1) | |
| num = F.avg_pool1d((e[:, :, None] * carr[:, None]).flatten(1, 2), win, win // 2) | |
| den = F.avg_pool1d(e, win, win // 2).clamp_min(1e-5) | |
| D.append(2 * num.view(e.shape[0], 5, 4, -1) / den[:, :, None]) | |
| vm = (F.avg_pool1d(v[:, None], win, win // 2) > 0.99).float()[:, :, None] | |
| return ((D[0] - D[1]).abs() * vm).sum() / (vm.sum() * 20).clamp_min(1) | |
| def nparams(m): | |
| return sum(p.numel() for p in m.parameters()) / 1e6 | |