Download models/micro/code/student_decoder.py from owensong/Inflect-Micro-v3: direct link, hf CLI and curl.
- Browser
- Download file 8.31 kB
-
https://huggingface.co/owensong/Inflect-Micro-v3/resolve/main/models/micro/code/student_decoder.py
- Command line
-
hf download hf://owensong/Inflect-Micro-v3/models/micro/code/student_decoder.py
-
curl -L -o student_decoder.py https://huggingface.co/owensong/Inflect-Micro-v3/resolve/main/models/micro/code/student_decoder.py
8.31 kB
| """Acoustic-side student: Kokoro's Decoder with the hard-coded 1024/512 widths made configurable, | |
| generator sized via the same istftnet config keys.""" | |
| import torch, torch.nn as nn, torch.nn.functional as F | |
| from torch.nn.utils import weight_norm | |
| from kokoro.istftnet import Decoder, AdainResBlk1d, Generator, SourceModuleHnNSF, SineGen | |
| DEC_PRESETS = { # name: (adain width, upsample_initial_channel, resblock kernels) | |
| "teacher": (1024, 512, [3, 7, 11]), | |
| "A": (256, 128, [3, 7, 11]), | |
| "B": (256, 128, [3, 7]), | |
| "C": (192, 96, [3, 7]), | |
| "D": (128, 64, [3, 7]), | |
| "E": (128, 64, [3, 5]), | |
| "F": (96, 48, [3, 5]), | |
| "W": (192, 256, [3, 7, 11]), # wider vocoder final stage (64 ch), thinner AdaIN blocks | |
| "W2": (128, 256, [3, 7]), | |
| "L": (384, 192, [3, 7, 11]), # 7.86M: capacity probe, 1.5x wider everywhere than A | |
| } | |
| class RichSource(SourceModuleHnNSF): | |
| """Teacher's source module with more harmonics; harmonics above 11 kHz are muted instead of aliasing.""" | |
| def forward(self, x): | |
| with torch.no_grad(): | |
| sine, uv, _ = self.l_sin_gen(x); k = torch.arange(1, sine.shape[-1] + 1, device=x.device) | |
| sine = sine * (x * k < 11000) | |
| return self.l_tanh(self.l_linear(sine)), torch.randn_like(uv) * self.sine_amp / 3, uv | |
| class PhaseSineGen(SineGen): | |
| """SineGen that also keeps each harmonic's instantaneous phase (same RNG use, same output).""" | |
| def _f02sine(self, f0_values): | |
| rad_values = (f0_values / self.sampling_rate) % 1 | |
| rand_ini = torch.rand(f0_values.shape[0], f0_values.shape[2], device=f0_values.device); rand_ini[:, 0] = 0 | |
| rad_values[:, 0, :] = rad_values[:, 0, :] + rand_ini | |
| rad_values = F.interpolate(rad_values.transpose(1, 2), scale_factor=1 / self.upsample_scale, mode="linear").transpose(1, 2) | |
| phase = torch.cumsum(rad_values, dim=1) * 2 * torch.pi | |
| self.phase = F.interpolate(phase.transpose(1, 2) * self.upsample_scale, scale_factor=self.upsample_scale, mode="linear").transpose(1, 2) | |
| return torch.sin(self.phase) | |
| class BandGenerator(Generator): | |
| """Kokoro generator plus two optional student-only inputs/outputs, both zero-initialised so a | |
| checkpoint loads to the same function: | |
| bands: every harmonic's phase as its own (cos, sin) pair at the 4800 fps head rate, fed to each | |
| stage next to the mixed 1-channel source STFT, so upper harmonics are no longer summed together. | |
| full_phase: an unbounded additive phase term, since the head's sin(x) alone confines phase to +-1 rad.""" | |
| def __init__(self, *a, harmonics=9, bands=False, full_phase=False, **kw): | |
| super().__init__(*a, **kw); self.bands, self.full_phase, self.hop = bands, full_phase, a[-1] | |
| if bands: | |
| self.band_convs = nn.ModuleList([nn.Conv1d(2 * harmonics, c.out_channels, c.kernel_size, c.stride, c.padding) for c in self.noise_convs]) | |
| for c in self.band_convs: nn.init.zeros_(c.weight); nn.init.zeros_(c.bias) | |
| if full_phase: | |
| ch = self.conv_post.in_channels; self.conv_phase = nn.Conv1d(ch, self.post_n_fft // 2 + 1, 7, 1, padding=3) | |
| nn.init.zeros_(self.conv_phase.weight); nn.init.zeros_(self.conv_phase.bias) | |
| def forward(self, x, s, f0): | |
| with torch.no_grad(): | |
| f0 = self.f0_upsamp(f0[:, None]).transpose(1, 2) | |
| har_source, _, uv = self.m_source(f0) | |
| har_spec, har_phase = self.stft.transform(har_source.transpose(1, 2).squeeze(1)) | |
| har = torch.cat([har_spec, har_phase], dim=1) | |
| if self.bands: | |
| ph = self.m_source.l_sin_gen.phase; k = torch.arange(1, ph.shape[-1] + 1, device=ph.device) | |
| g = uv * (f0 * k < 11000) | |
| q = torch.cat([torch.cos(ph) * g, torch.sin(ph) * g], -1).transpose(1, 2) | |
| q = F.pad(q, (0, 1), mode="replicate")[..., ::self.hop] # STFT frame centres | |
| for i in range(self.num_upsamples): | |
| x = F.leaky_relu(x, negative_slope=0.1) | |
| x_source = self.noise_convs[i](har) | |
| if self.bands: x_source = x_source + self.band_convs[i](q) | |
| x_source = self.noise_res[i](x_source, s) | |
| x = self.ups[i](x) | |
| if i == self.num_upsamples - 1: x = self.reflection_pad(x) | |
| x = x + x_source | |
| xs = None | |
| for j in range(self.num_kernels): | |
| xs = self.resblocks[i * self.num_kernels + j](x, s) if xs is None else xs + self.resblocks[i * self.num_kernels + j](x, s) | |
| x = xs / self.num_kernels | |
| h = F.leaky_relu(x); x = self.conv_post(h); nb = self.post_n_fft // 2 + 1 | |
| spec = torch.exp(x[:, :nb, :]); phase = torch.sin(x[:, nb:, :]) | |
| if self.full_phase: phase = phase + self.conv_phase(h) | |
| return self.stft.inverse(spec, phase) | |
| class StudentDecoder(Decoder): | |
| def __init__(self, preset="C", dim_in=512, style_dim=16, dim_out=80, disable_complex=False, harmonics=9, bands=False, full_phase=False): | |
| W, uic, kernels = DEC_PRESETS[preset] | |
| nn.Module.__init__(self) | |
| self.encode = AdainResBlk1d(dim_in + 2, W, style_dim) | |
| self.decode = nn.ModuleList([AdainResBlk1d(W + 2 + 64, W, style_dim) for _ in range(3)] | |
| + [AdainResBlk1d(W + 2 + 64, uic, style_dim, upsample=True)]) | |
| self.F0_conv = weight_norm(nn.Conv1d(1, 1, kernel_size=3, stride=2, padding=1)) | |
| self.N_conv = weight_norm(nn.Conv1d(1, 1, kernel_size=3, stride=2, padding=1)) | |
| self.asr_res = nn.Sequential(weight_norm(nn.Conv1d(dim_in, 64, kernel_size=1))) | |
| gargs = (style_dim, kernels, [10, 6], uic, [[1, 3, 5]] * len(kernels), [20, 12], 20, 5) | |
| self.generator = (BandGenerator(*gargs, harmonics=harmonics, bands=bands, full_phase=full_phase, disable_complex=disable_complex) | |
| if bands or full_phase else Generator(*gargs, disable_complex=disable_complex)) | |
| self.style = nn.Parameter(torch.randn(1, style_dim) * 0.1) | |
| if harmonics != 9: self.generator.m_source = RichSource(24000, 300, harmonic_num=harmonics - 1, voiced_threshod=10) | |
| if bands: self.generator.m_source.l_sin_gen.__class__ = PhaseSineGen | |
| def forward(self, asr, F0_curve, N, s=None): | |
| return super().forward(asr, F0_curve, N, self.style.expand(asr.shape[0], -1)) | |
| if __name__ == "__main__": | |
| from torch.utils.flop_counter import FlopCounterMode | |
| T = 174; asr = torch.randn(1, 512, T); F0 = torch.rand(1, 2 * T) * 200 + 80; N = torch.randn(1, 2 * T) | |
| print(f"{'preset':8s} {'params':>8s} {'gen':>7s} {'adain':>7s} {'GFLOP/s':>8s} {'gen%':>5s}") | |
| for p in DEC_PRESETS: | |
| m = StudentDecoder(p, style_dim=128 if p == "teacher" else 16).eval().requires_grad_(False) | |
| fc = FlopCounterMode(display=False, depth=2) | |
| with fc: out = m(asr, F0, N) | |
| c = {k: sum(v.values()) for k, v in fc.get_flop_counts().items()}; g = c["Global"]; gen = c.get("StudentDecoder.generator", 0) | |
| secs = out.shape[-1] / 24000 | |
| np_ = lambda mod: sum(x.numel() for x in mod.parameters()) / 1e6 | |
| print(f"{p:8s} {np_(m):7.2f}M {np_(m.generator):6.2f}M {np_(m)-np_(m.generator):6.2f}M {g/secs/1e9:8.2f} {gen/g*100:4.0f}%") | |
| def load_student_decoder(preset, path, device="cpu", **kw): | |
| """Load a StudentDecoder checkpoint. Matched-excitation checkpoints carry no source-mixing weights | |
| (the module was replaced during training); those are taken from the teacher, as in training.""" | |
| import torch | |
| sd = torch.load(path, map_location=device)["model"] | |
| lw = sd.get("generator.m_source.l_linear.weight") | |
| if lw is not None: kw.setdefault("harmonics", lw.shape[1]) | |
| if "generator.band_convs.0.weight" in sd: kw.setdefault("bands", True) | |
| if "generator.conv_phase.weight" in sd: kw.setdefault("full_phase", True) | |
| m = StudentDecoder(preset, **kw).to(device).eval() | |
| res = m.load_state_dict(sd, strict=False) | |
| if any("m_source" in k for k in res.missing_keys): | |
| from common import load_teacher | |
| _p, t, _ = load_teacher(device=device, fuse=False) | |
| m.generator.m_source.load_state_dict(t.decoder.generator.m_source.state_dict()); del t | |
| return m | |