File size: 12,936 Bytes
a775b3f
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
"""Frame-rate source-filter head (the Vocos idea with a harmonic source) in place of Kokoro's upsampling
iSTFTNet generator. Decoder A's AdaIN trunk is kept (80 frames a second). The head never upsamples: a few
ConvNeXt blocks at 80 fps predict, for every STFT bin of 2 frames per trunk frame (n_fft 600, hop 150),
a gain and a phase offset applied to the STFT of a clean sine bank built from the F0 curve (harmonic k has
phase k*phi), plus a gain for white noise. Harmonics therefore arrive phase-locked to the pitch by
construction, and the network only learns the spectral envelope and a slow phase correction.
usage: vhead.py   prints params and FLOPs next to decoder A"""
import torch, torch.nn as nn, torch.nn.functional as F
from student_decoder import StudentDecoder
SR, FHOP = 24000, 300   # F0 frames are 300 samples

def sine_bank(f0, K=64, c0=None):
    """f0 [B, frames] (80 fps) -> [B, samples] sum of K harmonics with Kokoro's fundamental phase (frame-rate
    cumulative phase, linearly interpolated between frame centres), harmonics muted above 11 kHz and where
    unvoiced. The running phase is accumulated in float64 and wrapped per frame so harmonic 64 stays exact.
    c0 [B]: phase (cycles) accumulated before this crop, so a segment's bank matches the whole sentence's."""
    B, n = f0.shape; dev = f0.device
    r = (f0.cpu().double() * FHOP / SR)                                  # cycles per frame
    c = ((torch.cumsum(r, 1) + (0 if c0 is None else c0.cpu().double()[:, None])) % 1).float().to(dev); d = F.pad(r[:, 1:], (0, 1)).float().to(dev)   # wrapped start, advance to next centre
    p = (torch.arange(n * FHOP, device=dev) + 0.5) / FHOP - 0.5; i = p.floor().clamp(0, n - 1).long(); fr = (p - i).clamp(0, 1)
    cyc = c[:, i] + fr * d[:, i]                                         # [B, samples] fundamental phase in cycles
    f0s = f0.repeat_interleave(FHOP, 1); k = torch.arange(1, K + 1, device=dev)[None, :, None]
    amp = ((f0s[:, None] * k < 11000) & (f0s[:, None] > 10)).float()
    return (torch.sin(2 * torch.pi * k * cyc[:, None]) * amp).sum(1) * 0.1, cyc, (f0s > 10).float()

class ConvNeXt(nn.Module):
    def __init__(self, dim, mult=3):
        super().__init__(); self.dw = nn.Conv1d(dim, dim, 7, padding=3, groups=dim); self.norm = nn.LayerNorm(dim)
        self.pw1 = nn.Linear(dim, dim * mult); self.pw2 = nn.Linear(dim * mult, dim); self.gamma = nn.Parameter(torch.full((dim,), 1e-2))
    def forward(self, x):
        h = self.norm(self.dw(x).transpose(1, 2)); return x + (self.gamma * self.pw2(F.gelu(self.pw1(h)))).transpose(1, 2)

class SourceFilterHead(nn.Module):
    def __init__(self, in_ch=128, dim=160, blocks=4, n_fft=600, hop=150, K=64, mod=2, mb=33):
        """mod: the noise part can also pulse with the pitch: extra sources noise*cos(m*phi), noise*sin(m*phi) for m = 1..mod,
        each with a signed gain relative to the plain noise gain, predicted at mb points across frequency (zero-init)."""
        super().__init__(); self.n_fft, self.hop, self.K, self.up, self.nb, self.mod, self.mb = n_fft, hop, K, FHOP // hop, n_fft // 2 + 1, mod, mb
        if mod: self.out_mod = nn.Linear(dim, self.up * 2 * mod * mb); nn.init.zeros_(self.out_mod.weight); nn.init.zeros_(self.out_mod.bias)
        self.inp = nn.Conv1d(in_ch, dim, 7, padding=3); self.norm0 = nn.LayerNorm(dim)
        self.blocks = nn.ModuleList([ConvNeXt(dim) for _ in range(blocks)]); self.norm1 = nn.LayerNorm(dim)
        self.out = nn.Linear(dim, self.up * self.nb * 3)   # per bin: harmonic log-gain, harmonic phase offset, noise log-gain
        nn.init.trunc_normal_(self.out.weight, std=0.02)
        with torch.no_grad(): b = self.out.bias.view(self.up, 3, self.nb); b[:, 0] = -3.0; b[:, 1] = 0.0; b[:, 2] = -6.0
        self.register_buffer("window", torch.hann_window(n_fft), persistent=False)
    def stft(self, x): return torch.stft(x, self.n_fft, self.hop, window=self.window, return_complex=True)
    def forward(self, x, f0, noise=None, c0=None):
        B, _, n = x.shape
        h = self.norm0(self.inp(x).transpose(1, 2)).transpose(1, 2)
        for b in self.blocks: h = b(h)
        hn = self.norm1(h.transpose(1, 2)); o = self.out(hn).view(B, n, self.up, 3, self.nb).permute(0, 3, 4, 1, 2).reshape(B, 3, self.nb, n * self.up)
        o = torch.cat([o, o[..., -1:]], -1)                # centred STFT has one more frame
        gh, ph, gn = torch.exp(o[:, 0].clamp(max=6)), o[:, 1], torch.exp(o[:, 2].clamp(max=6))
        with torch.no_grad():
            e, cyc, uv = sine_bank(f0, self.K, c0); E = self.stft(e)
            nz = torch.randn_like(e) if noise is None else noise; Nz = self.stft(nz)
            Nm = [self.stft(nz * uv * fn(2 * torch.pi * m * cyc)) for m in range(1, self.mod + 1) for fn in (torch.cos, torch.sin)]
        self.last = dict(E=E, gh=gh, ph=ph, gn=gn)
        if self.mod:
            a = self.out_mod(hn).view(B, n, self.up, 2 * self.mod, self.mb).permute(0, 3, 1, 2, 4).reshape(B * 2 * self.mod, n * self.up, self.mb)
            a = F.interpolate(a, size=self.nb, mode="linear", align_corners=True).view(B, 2 * self.mod, n * self.up, self.nb).transpose(2, 3)
            a = torch.cat([a, a[..., -1:]], -1); self.last["a"] = a
            Nz = Nz + sum(a[:, j] * Nm[j] for j in range(2 * self.mod))
        Y = E * torch.polar(gh, ph) + Nz * gn
        return torch.istft(Y, self.n_fft, self.hop, window=self.window, length=n * FHOP)

def warp_matrix(nb, G, sr=SR):
    """[nb, G] linear-interpolation weights from G points spaced evenly on a mel-like axis to nb linear STFT bins."""
    f = torch.linspace(0, sr / 2, nb); mel = lambda x: torch.log1p(x / 700); pos = mel(f) / mel(torch.tensor(sr / 2.0)) * (G - 1)
    i = pos.floor().clamp(0, G - 2).long(); w = pos - i; M = torch.zeros(nb, G); M[torch.arange(nb), i] = 1 - w; M[torch.arange(nb), i + 1] += w; return M

class SourceFilterHead2(SourceFilterHead):
    """Second design: every part has one job. The harmonic part is a smooth filter on the sine bank: gain and phase are
    predicted at G points across frequency (mel-spaced) and interpolated, plus one pulse shift per frame (a delay of the
    whole pulse, as a fraction of the pitch period), so it cannot scramble individual bins into fake noise. Anything
    noise-like has to come from the noise part (plain + pulsing with the pitch), whose gains are also on the G points."""
    def __init__(self, in_ch=128, dim=160, blocks=4, n_fft=600, hop=150, K=64, mod=2, G=80, burst=True, Gb=24):
        """burst: a second noise source shaped through a short STFT (150-sample window, 75 hop: 3 ms steps, 4 per trunk
        frame) with gains at Gb points, so consonant bursts and onsets are not smeared over the 25 ms main window."""
        nn.Module.__init__(self); self.burst, self.Gb = burst, Gb
        if burst:
            self.out_b = nn.Linear(dim, 4 * Gb); nn.init.trunc_normal_(self.out_b.weight, std=0.02); nn.init.constant_(self.out_b.bias, -7.0)
            self.register_buffer("wb", torch.hann_window(150), persistent=False); self.register_buffer("Mb", warp_matrix(76, Gb), persistent=False)
        self.n_fft, self.hop, self.K, self.up, self.nb, self.mod, self.G = n_fft, hop, K, FHOP // hop, n_fft // 2 + 1, mod, G
        self.inp = nn.Conv1d(in_ch, dim, 7, padding=3); self.norm0 = nn.LayerNorm(dim)
        self.blocks = nn.ModuleList([ConvNeXt(dim) for _ in range(blocks)]); self.norm1 = nn.LayerNorm(dim)
        self.C = 4 + 2 * mod                               # harmonic log-gain, phase cos, phase sin, noise log-gain, pulsing-noise gains
        self.out = nn.Linear(dim, self.up * (self.C * G + 1)); nn.init.trunc_normal_(self.out.weight, std=0.02)
        with torch.no_grad():
            b = self.out.bias.view(self.up, -1); b.zero_(); g = b[:, :self.C * G].view(self.up, self.C, G); g[:, 0] = -3.0; g[:, 1] = 1.0; g[:, 3] = -6.0
        self.register_buffer("window", torch.hann_window(n_fft), persistent=False); self.register_buffer("M", warp_matrix(self.nb, G), persistent=False)
    def forward(self, x, f0, noise=None, c0=None):
        B, _, n = x.shape; T = n * self.up
        h = self.norm0(self.inp(x).transpose(1, 2)).transpose(1, 2)
        for b in self.blocks: h = b(h)
        hn = self.norm1(h.transpose(1, 2)); o = self.out(hn).view(B, T, self.C * self.G + 1); o = torch.cat([o, o[:, -1:]], 1)   # centred STFT has one more frame
        g = (o[..., :-1].reshape(B, T + 1, self.C, self.G) @ self.M.T).permute(0, 2, 3, 1)                                 # [B, C, nb, T+1]
        gh, gn = torch.exp(g[:, 0].clamp(max=6)), torch.exp(g[:, 3].clamp(max=6)); u = torch.complex(g[:, 1], g[:, 2]); u = u / u.abs().clamp_min(1e-4)
        with torch.no_grad():
            e, cyc, uv = sine_bank(f0, self.K, c0); E = self.stft(e)
            nz = torch.randn_like(e) if noise is None else noise; Nz = self.stft(nz)
            Nm = [self.stft(nz * uv * fn(2 * torch.pi * m * cyc)) for m in range(1, self.mod + 1) for fn in (torch.cos, torch.sin)]
            f0f = torch.cat([f0.repeat_interleave(self.up, 1), f0[:, -1:]], 1).clamp_min(60)                               # per STFT frame
            ratio = torch.linspace(0, SR / 2, self.nb, device=x.device)[None, :, None] / f0f[:, None]                      # harmonic number of each bin
        shift = 0.5 * torch.tanh(o[..., -1])[:, None]                                                                      # pulse shift, cycles
        Hh = gh * u * torch.polar(torch.ones_like(ratio), 2 * torch.pi * ratio * shift)
        self.last = dict(E=E, gh=gh, ph=Hh.angle(), gn=gn, a=g[:, 4:], shift=shift, cyc=cyc, uv=uv)
        Nz = Nz + sum(g[:, 4 + j] * Nm[j] for j in range(2 * self.mod))
        y = torch.istft(E * Hh + Nz * gn, self.n_fft, self.hop, window=self.window, length=n * FHOP)
        if self.burst:
            gb = torch.exp((self.out_b(hn).view(B, n * 4, self.Gb) @ self.Mb.T).clamp(max=6)).transpose(1, 2); gb = torch.cat([gb, gb[..., -1:]], -1)
            with torch.no_grad(): Nb = torch.stft(nz.flip(-1), 150, 75, window=self.wb, return_complex=True)
            self.last["gb"] = gb; y = y + torch.istft(Nb * gb, 150, 75, window=self.wb, length=n * FHOP)
        return y

class VDecoder(nn.Module):
    """Decoder A's AdaIN trunk + the frame-rate head."""
    def __init__(self, preset="A", head="v1", **kw):
        super().__init__(); t = StudentDecoder(preset); uic = t.generator.conv_post.in_channels * 4
        for k in ("encode", "decode", "F0_conv", "N_conv", "asr_res"): setattr(self, k, getattr(t, k))
        self.style = t.style; self.head = (SourceFilterHead2 if head == "v2" else SourceFilterHead)(in_ch=uic, **kw)
    def forward(self, asr, F0_curve, N, noise=None, c0=None):
        s = self.style.expand(asr.shape[0], -1)
        F0 = self.F0_conv(F0_curve.unsqueeze(1)); Nn = self.N_conv(N.unsqueeze(1))
        x = self.encode(torch.cat([asr, F0, Nn], 1), s); asr_res = self.asr_res(asr); res = True
        for block in self.decode:
            if res: x = torch.cat([x, asr_res, F0, Nn], 1)
            x = block(x, s)
            if block.upsample_type != "none": res = False
        return self.head(x, F0_curve, noise, c0)
    def load_trunk(self, path):
        sd = {k: v for k, v in torch.load(path, map_location="cpu")["model"].items() if not k.startswith(("generator.", "head."))}
        res = self.load_state_dict(sd, strict=False); assert all(k.startswith("head.") for k in res.missing_keys) and not res.unexpected_keys, res
    def load_ckpt(self, path, device="cpu"):
        """Load a vhead checkpoint; one from before the pulsing-noise sources gets them zero-initialised (same function)."""
        res = self.load_state_dict(torch.load(path, map_location=device)["model"], strict=False); assert all("out_mod" in k or "out_b" in k for k in res.missing_keys) and not res.unexpected_keys, res
        if any("out_b" in k for k in res.missing_keys): self.head.burst = False   # checkpoint from before the short-window noise path

def HV(path):
    """Which head design a checkpoint holds (v2's output layer is the small one)."""
    return "v2" if torch.load(path, map_location="cpu")["model"]["head.out.weight"].shape[0] < 1500 else "v1"

if __name__ == "__main__":
    from torch.utils.flop_counter import FlopCounterMode
    T = 174; asr = torch.randn(1, 512, T); F0 = torch.rand(1, 2 * T) * 200 + 80; N = torch.randn(1, 2 * T)
    np_ = lambda m: sum(x.numel() for x in m.parameters()) / 1e6
    for name, m in [("decoder A", StudentDecoder("A")), ("A trunk + head", VDecoder()), ("A trunk + head2", VDecoder(head="v2"))]:
        m = m.eval().requires_grad_(False); fc = FlopCounterMode(display=False)
        with fc: out = m(asr, F0, N)
        hp = np_(m.head) if hasattr(m, "head") else np_(m.generator)
        print(f"{name:16s} {np_(m):5.2f}M params (head/generator {hp:.2f}M)  {fc.get_total_flops() / (out.shape[-1] / SR) / 1e9:6.2f} GFLOP/s  out {tuple(out.shape)}")