models/vocoder.py

"""
HiFi-GAN vocoder for mel spectrogram to waveform conversion.

Generator converts mel spectrograms to raw audio waveforms.
Discriminators (Multi-Period + Multi-Scale) provide adversarial training signal.
"""

import torch
import torch.nn as nn
import torch.nn.functional as F


# ---------------------------------------------------------------------------
# Generator
# ---------------------------------------------------------------------------

class ResBlock1(nn.Module):
    """Residual block with dilated convolutions (HiFi-GAN type 1)."""

    def __init__(self, channels: int, kernel_size: int, dilations: tuple[int, ...] = (1, 3, 5)) -> None:
        super().__init__()
        self.convs1 = nn.ModuleList()
        self.convs2 = nn.ModuleList()
        for d in dilations:
            self.convs1.append(
                nn.utils.parametrizations.weight_norm(
                    nn.Conv1d(channels, channels, kernel_size, dilation=d,
                              padding=(kernel_size * d - d) // 2)
                )
            )
            self.convs2.append(
                nn.utils.parametrizations.weight_norm(
                    nn.Conv1d(channels, channels, kernel_size, dilation=1,
                              padding=(kernel_size - 1) // 2)
                )
            )

    def forward(self, x: torch.Tensor) -> torch.Tensor:
        for c1, c2 in zip(self.convs1, self.convs2):
            xt = F.leaky_relu(x, 0.1)
            xt = c1(xt)
            xt = F.leaky_relu(xt, 0.1)
            xt = c2(xt)
            x = xt + x
        return x

    def remove_weight_norm(self) -> None:
        for c in self.convs1:
            nn.utils.parametrize.remove_parametrizations(c, "weight")
        for c in self.convs2:
            nn.utils.parametrize.remove_parametrizations(c, "weight")


class HiFiGANGenerator(nn.Module):
    """HiFi-GAN v1 generator (simplified for 6GB VRAM).

    Upsamples mel spectrogram (128, T) to waveform (1, T * hop_length).
    Uses smaller channel counts than the original paper to fit in memory.
    """

    def __init__(
        self,
        in_channels: int = 128,
        upsample_initial_channel: int = 256,
        upsample_rates: tuple[int, ...] = (8, 8, 2, 2, 2),
        upsample_kernel_sizes: tuple[int, ...] = (16, 16, 4, 4, 4),
        resblock_kernel_sizes: tuple[int, ...] = (3, 7, 11),
        resblock_dilations: tuple[tuple[int, ...], ...] = ((1, 3, 5), (1, 3, 5), (1, 3, 5)),
    ) -> None:
        super().__init__()
        self.num_upsamples = len(upsample_rates)

        # Initial conv
        self.conv_pre = nn.utils.parametrizations.weight_norm(
            nn.Conv1d(in_channels, upsample_initial_channel, 7, padding=3)
        )

        # Upsampling layers
        self.ups = nn.ModuleList()
        ch = upsample_initial_channel
        for i, (u, k) in enumerate(zip(upsample_rates, upsample_kernel_sizes)):
            self.ups.append(
                nn.utils.parametrizations.weight_norm(
                    nn.ConvTranspose1d(ch, ch // 2, k, stride=u,
                                       padding=(k - u) // 2)
                )
            )
            ch = ch // 2

        # Residual blocks after each upsample
        self.resblocks = nn.ModuleList()
        for i in range(len(self.ups)):
            ch_i = upsample_initial_channel // (2 ** (i + 1))
            for k, d in zip(resblock_kernel_sizes, resblock_dilations):
                self.resblocks.append(ResBlock1(ch_i, k, d))

        # Output conv
        self.conv_post = nn.utils.parametrizations.weight_norm(
            nn.Conv1d(ch_i, 1, 7, padding=3)
        )

    def forward(self, x: torch.Tensor) -> torch.Tensor:
        """
        Args:
            x: (batch, n_mels, time) mel spectrogram
        Returns:
            (batch, 1, time * hop_length) waveform
        """
        x = self.conv_pre(x)
        for i, up in enumerate(self.ups):
            x = F.leaky_relu(x, 0.1)
            x = up(x)
            # Apply all resblocks for this upsample level and average
            xs = 0.0
            for j in range(len(self.resblocks) // self.num_upsamples):
                xs = xs + self.resblocks[i * (len(self.resblocks) // self.num_upsamples) + j](x)
            x = xs / (len(self.resblocks) // self.num_upsamples)
        x = F.leaky_relu(x, 0.1)
        x = self.conv_post(x)
        x = torch.tanh(x)
        return x

    def remove_weight_norm(self) -> None:
        nn.utils.parametrize.remove_parametrizations(self.conv_pre, "weight")
        for up in self.ups:
            nn.utils.parametrize.remove_parametrizations(up, "weight")
        for block in self.resblocks:
            block.remove_weight_norm()
        nn.utils.parametrize.remove_parametrizations(self.conv_post, "weight")


# ---------------------------------------------------------------------------
# Discriminators
# ---------------------------------------------------------------------------

class PeriodDiscriminator(nn.Module):
    """Single sub-discriminator for Multi-Period Discriminator."""

    def __init__(self, period: int) -> None:
        super().__init__()
        self.period = period
        self.convs = nn.ModuleList([
            nn.utils.parametrizations.weight_norm(nn.Conv2d(1, 32, (5, 1), (3, 1), (2, 0))),
            nn.utils.parametrizations.weight_norm(nn.Conv2d(32, 64, (5, 1), (3, 1), (2, 0))),
            nn.utils.parametrizations.weight_norm(nn.Conv2d(64, 128, (5, 1), (3, 1), (2, 0))),
            nn.utils.parametrizations.weight_norm(nn.Conv2d(128, 256, (5, 1), (3, 1), (2, 0))),
            nn.utils.parametrizations.weight_norm(nn.Conv2d(256, 256, (5, 1), 1, (2, 0))),
        ])
        self.conv_post = nn.utils.parametrizations.weight_norm(nn.Conv2d(256, 1, (3, 1), 1, (1, 0)))

    def forward(self, x: torch.Tensor) -> tuple[torch.Tensor, list[torch.Tensor]]:
        fmap = []
        # Reshape: (B, 1, T) -> (B, 1, T//p, p)
        b, c, t = x.shape
        if t % self.period != 0:
            x = F.pad(x, (0, self.period - t % self.period), "reflect")
            t = x.shape[-1]
        x = x.view(b, c, t // self.period, self.period)

        for conv in self.convs:
            x = conv(x)
            x = F.leaky_relu(x, 0.1)
            fmap.append(x)
        x = self.conv_post(x)
        fmap.append(x)
        return x.flatten(1, -1), fmap


class MultiPeriodDiscriminator(nn.Module):
    def __init__(self, periods: tuple[int, ...] = (2, 3, 5, 7, 11)) -> None:
        super().__init__()
        self.discriminators = nn.ModuleList([PeriodDiscriminator(p) for p in periods])

    def forward(self, x: torch.Tensor) -> tuple[list[torch.Tensor], list[list[torch.Tensor]]]:
        outs, fmaps = [], []
        for d in self.discriminators:
            o, f = d(x)
            outs.append(o)
            fmaps.append(f)
        return outs, fmaps


class ScaleDiscriminator(nn.Module):
    """Single sub-discriminator for Multi-Scale Discriminator."""

    def __init__(self, use_spectral_norm: bool = False) -> None:
        super().__init__()
        norm_f = nn.utils.parametrizations.spectral_norm if use_spectral_norm else nn.utils.parametrizations.weight_norm
        self.convs = nn.ModuleList([
            norm_f(nn.Conv1d(1, 64, 15, 1, 7)),
            norm_f(nn.Conv1d(64, 128, 41, 2, 20, groups=4)),
            norm_f(nn.Conv1d(128, 256, 41, 2, 20, groups=16)),
            norm_f(nn.Conv1d(256, 512, 41, 4, 20, groups=16)),
            norm_f(nn.Conv1d(512, 512, 41, 4, 20, groups=16)),
            norm_f(nn.Conv1d(512, 512, 5, 1, 2)),
        ])
        self.conv_post = norm_f(nn.Conv1d(512, 1, 3, 1, 1))

    def forward(self, x: torch.Tensor) -> tuple[torch.Tensor, list[torch.Tensor]]:
        fmap = []
        for conv in self.convs:
            x = conv(x)
            x = F.leaky_relu(x, 0.1)
            fmap.append(x)
        x = self.conv_post(x)
        fmap.append(x)
        return x.flatten(1, -1), fmap


class MultiScaleDiscriminator(nn.Module):
    def __init__(self) -> None:
        super().__init__()
        self.discriminators = nn.ModuleList([
            ScaleDiscriminator(use_spectral_norm=True),
            ScaleDiscriminator(),
            ScaleDiscriminator(),
        ])
        self.pools = nn.ModuleList([
            nn.Identity(),
            nn.AvgPool1d(4, 2, 2),
            nn.AvgPool1d(4, 2, 2),
        ])

    def forward(self, x: torch.Tensor) -> tuple[list[torch.Tensor], list[list[torch.Tensor]]]:
        outs, fmaps = [], []
        for pool, disc in zip(self.pools, self.discriminators):
            x_in = pool(x)
            o, f = disc(x_in)
            outs.append(o)
            fmaps.append(f)
        return outs, fmaps