Inflect-Mini-v3 / core /baseline /models /mini /code /student_decoder.py
owensong's picture
Package qualified Mini core and optional early-access components
d160964 verified
Raw History Blame Contribute Delete
8.31 kB
"""Acoustic-side student: Kokoro's Decoder with the hard-coded 1024/512 widths made configurable,
generator sized via the same istftnet config keys."""
import torch, torch.nn as nn, torch.nn.functional as F
from torch.nn.utils import weight_norm
from kokoro.istftnet import Decoder, AdainResBlk1d, Generator, SourceModuleHnNSF, SineGen
DEC_PRESETS = { # name: (adain width, upsample_initial_channel, resblock kernels)
"teacher": (1024, 512, [3, 7, 11]),
"A": (256, 128, [3, 7, 11]),
"B": (256, 128, [3, 7]),
"C": (192, 96, [3, 7]),
"D": (128, 64, [3, 7]),
"E": (128, 64, [3, 5]),
"F": (96, 48, [3, 5]),
"W": (192, 256, [3, 7, 11]), # wider vocoder final stage (64 ch), thinner AdaIN blocks
"W2": (128, 256, [3, 7]),
"L": (384, 192, [3, 7, 11]), # 7.86M: capacity probe, 1.5x wider everywhere than A
}
class RichSource(SourceModuleHnNSF):
"""Teacher's source module with more harmonics; harmonics above 11 kHz are muted instead of aliasing."""
def forward(self, x):
with torch.no_grad():
sine, uv, _ = self.l_sin_gen(x); k = torch.arange(1, sine.shape[-1] + 1, device=x.device)
sine = sine * (x * k < 11000)
return self.l_tanh(self.l_linear(sine)), torch.randn_like(uv) * self.sine_amp / 3, uv
class PhaseSineGen(SineGen):
"""SineGen that also keeps each harmonic's instantaneous phase (same RNG use, same output)."""
def _f02sine(self, f0_values):
rad_values = (f0_values / self.sampling_rate) % 1
rand_ini = torch.rand(f0_values.shape[0], f0_values.shape[2], device=f0_values.device); rand_ini[:, 0] = 0
rad_values[:, 0, :] = rad_values[:, 0, :] + rand_ini
rad_values = F.interpolate(rad_values.transpose(1, 2), scale_factor=1 / self.upsample_scale, mode="linear").transpose(1, 2)
phase = torch.cumsum(rad_values, dim=1) * 2 * torch.pi
self.phase = F.interpolate(phase.transpose(1, 2) * self.upsample_scale, scale_factor=self.upsample_scale, mode="linear").transpose(1, 2)
return torch.sin(self.phase)
class BandGenerator(Generator):
"""Kokoro generator plus two optional student-only inputs/outputs, both zero-initialised so a
checkpoint loads to the same function:
bands: every harmonic's phase as its own (cos, sin) pair at the 4800 fps head rate, fed to each
stage next to the mixed 1-channel source STFT, so upper harmonics are no longer summed together.
full_phase: an unbounded additive phase term, since the head's sin(x) alone confines phase to +-1 rad."""
def __init__(self, *a, harmonics=9, bands=False, full_phase=False, **kw):
super().__init__(*a, **kw); self.bands, self.full_phase, self.hop = bands, full_phase, a[-1]
if bands:
self.band_convs = nn.ModuleList([nn.Conv1d(2 * harmonics, c.out_channels, c.kernel_size, c.stride, c.padding) for c in self.noise_convs])
for c in self.band_convs: nn.init.zeros_(c.weight); nn.init.zeros_(c.bias)
if full_phase:
ch = self.conv_post.in_channels; self.conv_phase = nn.Conv1d(ch, self.post_n_fft // 2 + 1, 7, 1, padding=3)
nn.init.zeros_(self.conv_phase.weight); nn.init.zeros_(self.conv_phase.bias)
def forward(self, x, s, f0):
with torch.no_grad():
f0 = self.f0_upsamp(f0[:, None]).transpose(1, 2)
har_source, _, uv = self.m_source(f0)
har_spec, har_phase = self.stft.transform(har_source.transpose(1, 2).squeeze(1))
har = torch.cat([har_spec, har_phase], dim=1)
if self.bands:
ph = self.m_source.l_sin_gen.phase; k = torch.arange(1, ph.shape[-1] + 1, device=ph.device)
g = uv * (f0 * k < 11000)
q = torch.cat([torch.cos(ph) * g, torch.sin(ph) * g], -1).transpose(1, 2)
q = F.pad(q, (0, 1), mode="replicate")[..., ::self.hop] # STFT frame centres
for i in range(self.num_upsamples):
x = F.leaky_relu(x, negative_slope=0.1)
x_source = self.noise_convs[i](har)
if self.bands: x_source = x_source + self.band_convs[i](q)
x_source = self.noise_res[i](x_source, s)
x = self.ups[i](x)
if i == self.num_upsamples - 1: x = self.reflection_pad(x)
x = x + x_source
xs = None
for j in range(self.num_kernels):
xs = self.resblocks[i * self.num_kernels + j](x, s) if xs is None else xs + self.resblocks[i * self.num_kernels + j](x, s)
x = xs / self.num_kernels
h = F.leaky_relu(x); x = self.conv_post(h); nb = self.post_n_fft // 2 + 1
spec = torch.exp(x[:, :nb, :]); phase = torch.sin(x[:, nb:, :])
if self.full_phase: phase = phase + self.conv_phase(h)
return self.stft.inverse(spec, phase)
class StudentDecoder(Decoder):
def __init__(self, preset="C", dim_in=512, style_dim=16, dim_out=80, disable_complex=False, harmonics=9, bands=False, full_phase=False):
W, uic, kernels = DEC_PRESETS[preset]
nn.Module.__init__(self)
self.encode = AdainResBlk1d(dim_in + 2, W, style_dim)
self.decode = nn.ModuleList([AdainResBlk1d(W + 2 + 64, W, style_dim) for _ in range(3)]
+ [AdainResBlk1d(W + 2 + 64, uic, style_dim, upsample=True)])
self.F0_conv = weight_norm(nn.Conv1d(1, 1, kernel_size=3, stride=2, padding=1))
self.N_conv = weight_norm(nn.Conv1d(1, 1, kernel_size=3, stride=2, padding=1))
self.asr_res = nn.Sequential(weight_norm(nn.Conv1d(dim_in, 64, kernel_size=1)))
gargs = (style_dim, kernels, [10, 6], uic, [[1, 3, 5]] * len(kernels), [20, 12], 20, 5)
self.generator = (BandGenerator(*gargs, harmonics=harmonics, bands=bands, full_phase=full_phase, disable_complex=disable_complex)
if bands or full_phase else Generator(*gargs, disable_complex=disable_complex))
self.style = nn.Parameter(torch.randn(1, style_dim) * 0.1)
if harmonics != 9: self.generator.m_source = RichSource(24000, 300, harmonic_num=harmonics - 1, voiced_threshod=10)
if bands: self.generator.m_source.l_sin_gen.__class__ = PhaseSineGen
def forward(self, asr, F0_curve, N, s=None):
return super().forward(asr, F0_curve, N, self.style.expand(asr.shape[0], -1))
if __name__ == "__main__":
from torch.utils.flop_counter import FlopCounterMode
T = 174; asr = torch.randn(1, 512, T); F0 = torch.rand(1, 2 * T) * 200 + 80; N = torch.randn(1, 2 * T)
print(f"{'preset':8s} {'params':>8s} {'gen':>7s} {'adain':>7s} {'GFLOP/s':>8s} {'gen%':>5s}")
for p in DEC_PRESETS:
m = StudentDecoder(p, style_dim=128 if p == "teacher" else 16).eval().requires_grad_(False)
fc = FlopCounterMode(display=False, depth=2)
with fc: out = m(asr, F0, N)
c = {k: sum(v.values()) for k, v in fc.get_flop_counts().items()}; g = c["Global"]; gen = c.get("StudentDecoder.generator", 0)
secs = out.shape[-1] / 24000
np_ = lambda mod: sum(x.numel() for x in mod.parameters()) / 1e6
print(f"{p:8s} {np_(m):7.2f}M {np_(m.generator):6.2f}M {np_(m)-np_(m.generator):6.2f}M {g/secs/1e9:8.2f} {gen/g*100:4.0f}%")
def load_student_decoder(preset, path, device="cpu", **kw):
"""Load a StudentDecoder checkpoint. Matched-excitation checkpoints carry no source-mixing weights
(the module was replaced during training); those are taken from the teacher, as in training."""
import torch
sd = torch.load(path, map_location=device)["model"]
lw = sd.get("generator.m_source.l_linear.weight")
if lw is not None: kw.setdefault("harmonics", lw.shape[1])
if "generator.band_convs.0.weight" in sd: kw.setdefault("bands", True)
if "generator.conv_phase.weight" in sd: kw.setdefault("full_phase", True)
m = StudentDecoder(preset, **kw).to(device).eval()
res = m.load_state_dict(sd, strict=False)
if any("m_source" in k for k in res.missing_keys):
from common import load_teacher
_p, t, _ = load_teacher(device=device, fuse=False)
m.generator.m_source.load_state_dict(t.decoder.generator.m_source.state_dict()); del t
return m