File size: 8,309 Bytes
a775b3f
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
"""Acoustic-side student: Kokoro's Decoder with the hard-coded 1024/512 widths made configurable,

generator sized via the same istftnet config keys."""
import torch, torch.nn as nn, torch.nn.functional as F
from torch.nn.utils import weight_norm
from kokoro.istftnet import Decoder, AdainResBlk1d, Generator, SourceModuleHnNSF, SineGen

DEC_PRESETS = {  # name: (adain width, upsample_initial_channel, resblock kernels)
    "teacher": (1024, 512, [3, 7, 11]),
    "A": (256, 128, [3, 7, 11]),
    "B": (256, 128, [3, 7]),
    "C": (192, 96, [3, 7]),
    "D": (128, 64, [3, 7]),
    "E": (128, 64, [3, 5]),
    "F": (96, 48, [3, 5]),
    "W": (192, 256, [3, 7, 11]),   # wider vocoder final stage (64 ch), thinner AdaIN blocks
    "W2": (128, 256, [3, 7]),
    "L": (384, 192, [3, 7, 11]),   # 7.86M: capacity probe, 1.5x wider everywhere than A
}

class RichSource(SourceModuleHnNSF):
    """Teacher's source module with more harmonics; harmonics above 11 kHz are muted instead of aliasing."""
    def forward(self, x):
        with torch.no_grad():
            sine, uv, _ = self.l_sin_gen(x); k = torch.arange(1, sine.shape[-1] + 1, device=x.device)
            sine = sine * (x * k < 11000)
        return self.l_tanh(self.l_linear(sine)), torch.randn_like(uv) * self.sine_amp / 3, uv

class PhaseSineGen(SineGen):
    """SineGen that also keeps each harmonic's instantaneous phase (same RNG use, same output)."""
    def _f02sine(self, f0_values):
        rad_values = (f0_values / self.sampling_rate) % 1
        rand_ini = torch.rand(f0_values.shape[0], f0_values.shape[2], device=f0_values.device); rand_ini[:, 0] = 0
        rad_values[:, 0, :] = rad_values[:, 0, :] + rand_ini
        rad_values = F.interpolate(rad_values.transpose(1, 2), scale_factor=1 / self.upsample_scale, mode="linear").transpose(1, 2)
        phase = torch.cumsum(rad_values, dim=1) * 2 * torch.pi
        self.phase = F.interpolate(phase.transpose(1, 2) * self.upsample_scale, scale_factor=self.upsample_scale, mode="linear").transpose(1, 2)
        return torch.sin(self.phase)

class BandGenerator(Generator):
    """Kokoro generator plus two optional student-only inputs/outputs, both zero-initialised so a

    checkpoint loads to the same function:

    bands: every harmonic's phase as its own (cos, sin) pair at the 4800 fps head rate, fed to each

    stage next to the mixed 1-channel source STFT, so upper harmonics are no longer summed together.

    full_phase: an unbounded additive phase term, since the head's sin(x) alone confines phase to +-1 rad."""
    def __init__(self, *a, harmonics=9, bands=False, full_phase=False, **kw):
        super().__init__(*a, **kw); self.bands, self.full_phase, self.hop = bands, full_phase, a[-1]
        if bands:
            self.band_convs = nn.ModuleList([nn.Conv1d(2 * harmonics, c.out_channels, c.kernel_size, c.stride, c.padding) for c in self.noise_convs])
            for c in self.band_convs: nn.init.zeros_(c.weight); nn.init.zeros_(c.bias)
        if full_phase:
            ch = self.conv_post.in_channels; self.conv_phase = nn.Conv1d(ch, self.post_n_fft // 2 + 1, 7, 1, padding=3)
            nn.init.zeros_(self.conv_phase.weight); nn.init.zeros_(self.conv_phase.bias)

    def forward(self, x, s, f0):
        with torch.no_grad():
            f0 = self.f0_upsamp(f0[:, None]).transpose(1, 2)
            har_source, _, uv = self.m_source(f0)
            har_spec, har_phase = self.stft.transform(har_source.transpose(1, 2).squeeze(1))
            har = torch.cat([har_spec, har_phase], dim=1)
            if self.bands:
                ph = self.m_source.l_sin_gen.phase; k = torch.arange(1, ph.shape[-1] + 1, device=ph.device)
                g = uv * (f0 * k < 11000)
                q = torch.cat([torch.cos(ph) * g, torch.sin(ph) * g], -1).transpose(1, 2)
                q = F.pad(q, (0, 1), mode="replicate")[..., ::self.hop]   # STFT frame centres
        for i in range(self.num_upsamples):
            x = F.leaky_relu(x, negative_slope=0.1)
            x_source = self.noise_convs[i](har)
            if self.bands: x_source = x_source + self.band_convs[i](q)
            x_source = self.noise_res[i](x_source, s)
            x = self.ups[i](x)
            if i == self.num_upsamples - 1: x = self.reflection_pad(x)
            x = x + x_source
            xs = None
            for j in range(self.num_kernels):
                xs = self.resblocks[i * self.num_kernels + j](x, s) if xs is None else xs + self.resblocks[i * self.num_kernels + j](x, s)
            x = xs / self.num_kernels
        h = F.leaky_relu(x); x = self.conv_post(h); nb = self.post_n_fft // 2 + 1
        spec = torch.exp(x[:, :nb, :]); phase = torch.sin(x[:, nb:, :])
        if self.full_phase: phase = phase + self.conv_phase(h)
        return self.stft.inverse(spec, phase)

class StudentDecoder(Decoder):
    def __init__(self, preset="C", dim_in=512, style_dim=16, dim_out=80, disable_complex=False, harmonics=9, bands=False, full_phase=False):
        W, uic, kernels = DEC_PRESETS[preset]
        nn.Module.__init__(self)
        self.encode = AdainResBlk1d(dim_in + 2, W, style_dim)
        self.decode = nn.ModuleList([AdainResBlk1d(W + 2 + 64, W, style_dim) for _ in range(3)]
                                    + [AdainResBlk1d(W + 2 + 64, uic, style_dim, upsample=True)])
        self.F0_conv = weight_norm(nn.Conv1d(1, 1, kernel_size=3, stride=2, padding=1))
        self.N_conv = weight_norm(nn.Conv1d(1, 1, kernel_size=3, stride=2, padding=1))
        self.asr_res = nn.Sequential(weight_norm(nn.Conv1d(dim_in, 64, kernel_size=1)))
        gargs = (style_dim, kernels, [10, 6], uic, [[1, 3, 5]] * len(kernels), [20, 12], 20, 5)
        self.generator = (BandGenerator(*gargs, harmonics=harmonics, bands=bands, full_phase=full_phase, disable_complex=disable_complex)
                          if bands or full_phase else Generator(*gargs, disable_complex=disable_complex))
        self.style = nn.Parameter(torch.randn(1, style_dim) * 0.1)
        if harmonics != 9: self.generator.m_source = RichSource(24000, 300, harmonic_num=harmonics - 1, voiced_threshod=10)
        if bands: self.generator.m_source.l_sin_gen.__class__ = PhaseSineGen

    def forward(self, asr, F0_curve, N, s=None):
        return super().forward(asr, F0_curve, N, self.style.expand(asr.shape[0], -1))

if __name__ == "__main__":
    from torch.utils.flop_counter import FlopCounterMode
    T = 174; asr = torch.randn(1, 512, T); F0 = torch.rand(1, 2 * T) * 200 + 80; N = torch.randn(1, 2 * T)
    print(f"{'preset':8s} {'params':>8s} {'gen':>7s} {'adain':>7s} {'GFLOP/s':>8s} {'gen%':>5s}")
    for p in DEC_PRESETS:
        m = StudentDecoder(p, style_dim=128 if p == "teacher" else 16).eval().requires_grad_(False)
        fc = FlopCounterMode(display=False, depth=2)
        with fc: out = m(asr, F0, N)
        c = {k: sum(v.values()) for k, v in fc.get_flop_counts().items()}; g = c["Global"]; gen = c.get("StudentDecoder.generator", 0)
        secs = out.shape[-1] / 24000
        np_ = lambda mod: sum(x.numel() for x in mod.parameters()) / 1e6
        print(f"{p:8s} {np_(m):7.2f}M {np_(m.generator):6.2f}M {np_(m)-np_(m.generator):6.2f}M {g/secs/1e9:8.2f} {gen/g*100:4.0f}%")

def load_student_decoder(preset, path, device="cpu", **kw):
    """Load a StudentDecoder checkpoint. Matched-excitation checkpoints carry no source-mixing weights

    (the module was replaced during training); those are taken from the teacher, as in training."""
    import torch
    sd = torch.load(path, map_location=device)["model"]
    lw = sd.get("generator.m_source.l_linear.weight")
    if lw is not None: kw.setdefault("harmonics", lw.shape[1])
    if "generator.band_convs.0.weight" in sd: kw.setdefault("bands", True)
    if "generator.conv_phase.weight" in sd: kw.setdefault("full_phase", True)
    m = StudentDecoder(preset, **kw).to(device).eval()
    res = m.load_state_dict(sd, strict=False)
    if any("m_source" in k for k in res.missing_keys):
        from common import load_teacher
        _p, t, _ = load_teacher(device=device, fuse=False)
        m.generator.m_source.load_state_dict(t.decoder.generator.m_source.state_dict()); del t
    return m