Spaces:
Running on Zero
Running on Zero
| """ | |
| Neural Source Filter based modules implementation. | |
| Neural source-filter waveform models for statistical parametric speech synthesis | |
| """ | |
| import numpy as np | |
| import typing as tp | |
| import torch | |
| import torch.nn as nn | |
| import torch.nn.functional as F | |
| from torch.nn.utils import weight_norm, remove_weight_norm | |
| from torch.distributions.uniform import Uniform | |
| from torch.distributions.normal import Normal | |
| class SineGen(torch.nn.Module): | |
| """ Definition of sine generator | |
| SineGen(samp_rate, harmonic_num = 0, | |
| sine_amp = 0.1, noise_std = 0.003, | |
| voiced_threshold = 0, | |
| flag_for_pulse=False) | |
| samp_rate: sampling rate in Hz | |
| harmonic_num: number of harmonic overtones (default 0) | |
| sine_amp: amplitude of sine-wavefrom (default 0.1) | |
| noise_std: std of Gaussian noise (default 0.003) | |
| voiced_thoreshold: F0 threshold for U/V classification (default 0) | |
| flag_for_pulse: this SinGen is used inside PulseGen (default False) | |
| Note: when flag_for_pulse is True, the first time step of a voiced | |
| segment is always sin(np.pi) or cos(0) | |
| """ | |
| def __init__(self, samp_rate, harmonic_num=0, | |
| sine_amp=0.1, noise_std=0.003, | |
| voiced_threshold=0): | |
| super(SineGen, self).__init__() | |
| self.sine_amp = sine_amp | |
| self.noise_std = noise_std | |
| self.harmonic_num = harmonic_num | |
| self.sampling_rate = samp_rate | |
| self.voiced_threshold = voiced_threshold | |
| def _f02uv(self, f0): | |
| # generate uv signal | |
| uv = (f0 > self.voiced_threshold).type(torch.float32) | |
| return uv | |
| def forward(self, f0): | |
| """ | |
| :param f0: [B, 1, sample_len], Hz | |
| :return: [B, 1, sample_len] | |
| """ | |
| F_mat = torch.zeros((f0.size(0), self.harmonic_num + 1, f0.size(-1))).to(f0.device) | |
| for i in range(self.harmonic_num + 1): | |
| F_mat[:, i:i+1, :] = f0 * (i+1) / self.sampling_rate | |
| theta_mat = 2 * np.pi * (torch.cumsum(F_mat, dim=-1) % 1) | |
| u_dist = Uniform(low=-np.pi, high=np.pi) | |
| phase_vec = u_dist.sample(sample_shape=(f0.size(0), self.harmonic_num + 1, 1)).to(F_mat.device) | |
| phase_vec[:, 0, :] = 0 | |
| # generate sine waveforms | |
| sine_waves = self.sine_amp * torch.sin(theta_mat + phase_vec) | |
| # generate uv signal | |
| uv = self._f02uv(f0) | |
| # noise: for unvoiced should be similar to sine_amp | |
| # std = self.sine_amp/3 -> max value ~ self.sine_amp | |
| # . for voiced regions is self.noise_std | |
| noise_amp = uv * self.noise_std + (1 - uv) * self.sine_amp / 3 | |
| noise = noise_amp * torch.randn_like(sine_waves) | |
| # first: set the unvoiced part to 0 by uv | |
| # then: additive noise | |
| sine_waves = sine_waves * uv + noise | |
| return sine_waves, uv, noise | |
| class SourceModuleHnNSF(torch.nn.Module): | |
| """ SourceModule for hn-nsf | |
| SourceModule(sampling_rate, harmonic_num=0, sine_amp=0.1, | |
| add_noise_std=0.003, voiced_threshod=0) | |
| sampling_rate: sampling_rate in Hz | |
| harmonic_num: number of harmonic above F0 (default: 0) | |
| sine_amp: amplitude of sine source signal (default: 0.1) | |
| add_noise_std: std of additive Gaussian noise (default: 0.003) | |
| note that amplitude of noise in unvoiced is decided | |
| by sine_amp | |
| voiced_threshold: threhold to set U/V given F0 (default: 0) | |
| Sine_source, noise_source = SourceModuleHnNSF(F0_sampled) | |
| F0_sampled (batchsize, length, 1) | |
| Sine_source (batchsize, length, 1) | |
| noise_source (batchsize, length 1) | |
| uv (batchsize, length, 1) | |
| """ | |
| def __init__(self, sampling_rate, upsample_scale, harmonic_num=0, sine_amp=0.1, | |
| add_noise_std=0.003, voiced_threshod=0): | |
| super(SourceModuleHnNSF, self).__init__() | |
| self.sine_amp = sine_amp | |
| self.noise_std = add_noise_std | |
| # to produce sine waveforms | |
| self.l_sin_gen = SineGen(sampling_rate, harmonic_num, | |
| sine_amp, add_noise_std, voiced_threshod) | |
| # to merge source harmonics into a single excitation | |
| self.l_linear = torch.nn.Linear(harmonic_num + 1, 1) | |
| self.l_tanh = torch.nn.Tanh() | |
| def forward(self, x): | |
| """ | |
| Sine_source, noise_source = SourceModuleHnNSF(F0_sampled) | |
| F0_sampled (batchsize, length, 1) | |
| Sine_source (batchsize, length, 1) | |
| noise_source (batchsize, length 1) | |
| """ | |
| # source for harmonic branch | |
| with torch.no_grad(): | |
| sine_wavs, uv, _ = self.l_sin_gen(x.transpose(1,2)) | |
| sine_wavs = sine_wavs.transpose(1,2) | |
| uv = uv.transpose(1,2) | |
| sine_merge = self.l_tanh(self.l_linear(sine_wavs)) | |
| # source for noise branch, in the same shape as uv | |
| noise = torch.randn_like(uv) * self.sine_amp / 3 | |
| return sine_merge, noise, uv | |
| class SourceModule(torch.nn.Module): | |
| def __init__(self, | |
| nb_harmonics: int, | |
| upsample_ratio: int, | |
| sampling_rate: int, | |
| alpha: float = 0.1, | |
| sigma: float = 0.003, | |
| voiced_threshold: float = 10 | |
| ): | |
| super(SourceModule, self).__init__() | |
| self.nb_harmonics = nb_harmonics | |
| self.upsample_ratio = upsample_ratio | |
| self.sampling_rate = sampling_rate | |
| self.alpha = alpha | |
| self.sigma = sigma | |
| self.voiced_threshold = voiced_threshold | |
| self.ffn = nn.Sequential( | |
| weight_norm(nn.Conv1d(self.nb_harmonics + 1, 1, kernel_size=1, stride=1)), | |
| nn.Tanh()) | |
| def f02uv(self, f0): | |
| # generate uv signal | |
| uv = (f0 > self.voiced_threshold).type(torch.float32) | |
| return uv | |
| def forward(self, f0): | |
| """ | |
| :param f0: [B, 1, frame_len], Hz | |
| :return: [B, 1, sample_len] | |
| """ | |
| with torch.no_grad(): | |
| uv = self.f02uv(f0) | |
| f0_samples = F.interpolate(f0, scale_factor=(self.upsample_ratio), mode='nearest') | |
| uv_samples = F.interpolate(uv, scale_factor=(self.upsample_ratio), mode='nearest') | |
| F_mat = torch.zeros((f0_samples.size(0), self.nb_harmonics + 1, f0_samples.size(-1))).to(f0_samples.device) | |
| for i in range(self.nb_harmonics + 1): | |
| F_mat[:, i:i+1, :] = f0_samples * (i+1) / self.sampling_rate | |
| theta_mat = 2 * np.pi * (torch.cumsum(F_mat, dim=-1) % 1) | |
| u_dist = Uniform(low=-np.pi, high=np.pi) | |
| phase_vec = u_dist.sample(sample_shape=(f0.size(0), self.nb_harmonics + 1, 1)).to(F_mat.device) | |
| phase_vec[:, 0, :] = 0 | |
| n_dist = Normal(loc=0., scale=self.sigma) | |
| noise = n_dist.sample(sample_shape=(f0_samples.size(0), self.nb_harmonics + 1, f0_samples.size(-1))).to(F_mat.device) | |
| e_voice = self.alpha * torch.sin(theta_mat + phase_vec) + noise | |
| e_unvoice = self.alpha / 3 / self.sigma * noise | |
| e = e_voice * uv_samples + e_unvoice * (1 - uv_samples) | |
| return self.ffn(e) | |
| def remove_weight_norm(self): | |
| remove_weight_norm(self.ffn[0]) | |
| class ConvRNNF0Predictor(nn.Module): | |
| def __init__(self, | |
| num_class: int = 1, | |
| in_channels: int = 80, | |
| cond_channels: int = 512, | |
| use_cond_rnn: bool = True, | |
| bidirectional_rnn: bool = False, | |
| ): | |
| super().__init__() | |
| self.num_class = num_class | |
| self.use_cond_rnn = use_cond_rnn | |
| self.condnet = nn.Sequential( | |
| weight_norm( | |
| nn.Conv1d(in_channels, cond_channels, kernel_size=3, padding=1) | |
| ), | |
| nn.ELU(), | |
| weight_norm( | |
| nn.Conv1d(cond_channels, cond_channels, kernel_size=3, padding=1) | |
| ), | |
| nn.ELU(), | |
| weight_norm( | |
| nn.Conv1d(cond_channels, cond_channels, kernel_size=3, padding=1) | |
| ), | |
| nn.ELU(), | |
| weight_norm( | |
| nn.Conv1d(cond_channels, cond_channels, kernel_size=3, padding=1) | |
| ), | |
| nn.ELU(), | |
| weight_norm( | |
| nn.Conv1d(cond_channels, cond_channels, kernel_size=3, padding=1) | |
| ), | |
| nn.ELU(), | |
| ) | |
| if self.use_cond_rnn: | |
| self.rnn = nn.GRU( | |
| cond_channels, | |
| cond_channels // 2 if bidirectional_rnn else cond_channels, | |
| num_layers=1, | |
| batch_first=True, | |
| bidirectional=bidirectional_rnn, | |
| ) | |
| self.classifier = nn.Linear(in_features=cond_channels, out_features=self.num_class) | |
| def forward(self, x: torch.Tensor) -> torch.Tensor: | |
| x = self.condnet(x) | |
| if self.use_cond_rnn: | |
| x, _ = self.rnn(x.transpose(1, 2)) | |
| else: | |
| x = x.transpose(1, 2) | |
| return torch.abs(self.classifier(x).squeeze(-1)) | |