soulxsinger/models/modules/mel_transform.py

import torch
import math
import numpy as np
from librosa.filters import mel as librosa_mel_fn
import torch.nn as nn
from typing import Any, Dict, Optional


def dynamic_range_compression(x, C=1, clip_val=1e-5):
    return np.log(np.clip(x, a_min=clip_val, a_max=None) * C)


def dynamic_range_decompression(x, C=1):
    return np.exp(x) / C


def dynamic_range_compression_torch(x, C=1, clip_val=1e-5):
    return torch.log(torch.clamp(x, min=clip_val) * C)


def dynamic_range_decompression_torch(x, C=1):
    return torch.exp(x) / C


def spectral_normalize_torch(magnitudes):
    output = dynamic_range_compression_torch(magnitudes)
    return output


def spectral_de_normalize_torch(magnitudes):
    output = dynamic_range_decompression_torch(magnitudes)
    return output


class MelSpectrogram(nn.Module):
    def __init__(
        self,
        n_fft,
        num_mels,
        sampling_rate,
        hop_size,
        win_size,
        fmin,
        fmax,
        center=False,
    ):
        super(MelSpectrogram, self).__init__()
        self.n_fft = n_fft
        self.hop_size = hop_size
        self.win_size = win_size
        self.sampling_rate = sampling_rate
        self.num_mels = num_mels
        self.fmin = fmin
        self.fmax = fmax
        self.center = center

        mel_basis = {}
        hann_window = {}

        mel = librosa_mel_fn(
            sr=sampling_rate, n_fft=n_fft, n_mels=num_mels, fmin=fmin, fmax=fmax
        )
        mel_basis = torch.from_numpy(mel).float()
        hann_window = torch.hann_window(win_size)

        self.register_buffer("mel_basis", mel_basis)
        self.register_buffer("hann_window", hann_window)

    def forward(self, y):
        y = torch.nn.functional.pad(
            y.unsqueeze(1),
            (
                int((self.n_fft - self.hop_size) / 2),
                int((self.n_fft - self.hop_size) / 2),
            ),
            mode="reflect",
        )
        y = y.squeeze(1)
        spec = torch.stft(
            y,
            self.n_fft,
            hop_length=self.hop_size,
            win_length=self.win_size,
            window=self.hann_window,
            center=self.center,
            pad_mode="reflect",
            normalized=False,
            onesided=True,
            return_complex=True,
        )
        spec = torch.view_as_real(spec)

        spec = torch.sqrt(spec.pow(2).sum(-1) + (1e-9))

        spec = torch.matmul(self.mel_basis, spec)
        spec = spectral_normalize_torch(spec)

        return spec


def load_mel_spectrogram():
    return load_mel_spectrogram_from_cfg(None)


def _get_from_mapping(cfg: Any, key: str, default: Any = None) -> Any:
    """Safely read a field from a dict/OmegaConf-like object."""
    if cfg is None:
        return default
    if isinstance(cfg, dict):
        return cfg.get(key, default)
    return getattr(cfg, key, default)


def load_mel_spectrogram_from_cfg(audio_cfg: Optional[Any] = None) -> MelSpectrogram:
    """Build MelSpectrogram from `audio_config`-like config.

    Expected keys (either in dict or Hydra/OmegaConf object):
      - hop_size, sample_rate (or sampling_rate), n_fft, num_mels, win_size, fmin, fmax
    """
    # Defaults keep current behavior.
    mel_cfg: Dict[str, Any] = {
        "hop_size": _get_from_mapping(audio_cfg, "hop_size", 480),
        "sampling_rate": _get_from_mapping(
            audio_cfg,
            "sampling_rate",
            _get_from_mapping(audio_cfg, "sample_rate", 24000),
        ),
        "n_fft": _get_from_mapping(audio_cfg, "n_fft", 1920),
        "num_mels": _get_from_mapping(audio_cfg, "num_mels", 128),
        "win_size": _get_from_mapping(audio_cfg, "win_size", 1920),
        "fmin": _get_from_mapping(audio_cfg, "fmin", 0),
        "fmax": _get_from_mapping(audio_cfg, "fmax", 12000),
    }

    mel_model = MelSpectrogram(**mel_cfg)
    mel_model.eval()
    return mel_model


class MelSpectrogramEncoder(nn.Module):
    def __init__(self, audio_config: dict | None = None):
        super(MelSpectrogramEncoder, self).__init__()
        self.model = load_mel_spectrogram_from_cfg(audio_config)
        audio_config = audio_config or {}
        self.mel_mean = audio_config.get("mel_mean", -4.92)
        self.mel_var = audio_config.get("mel_var", 8.14)

    def forward(self, x):
        x = self.model(x).transpose(1, 2)
        x = (x - self.mel_mean) / math.sqrt(self.mel_var)
        return x