Download tiny_voca/model.py from RL10x/tiny-voca-hindi: direct link, hf CLI and curl.
- Browser
- Download file 33.1 kB
-
https://huggingface.co/RL10x/tiny-voca-hindi/resolve/main/tiny_voca/model.py
- Command line
-
hf download hf://RL10x/tiny-voca-hindi/tiny_voca/model.py
-
curl -L -o model.py https://huggingface.co/RL10x/tiny-voca-hindi/resolve/main/tiny_voca/model.py
33.1 kB
| """tiny-voca v2: feed-forward transformer with a pitch-driven source-filter head. | |
| All weights start random. Tensor conventions: hidden [B,T,C], frame features | |
| [B,T], waveform [B,S]. Acoustic frame i is centred on sample ``i * hop``. | |
| phoneme + stress + voice -> transformer encoder -> duration predictor | |
| -> repeat tokens by duration (supervised durations while training) | |
| -> frame transformer A -> pitch / voicing / energy | |
| -> frame transformer B -> filter parameters | |
| -> harmonic pulse train (from pitch) and noise, each shaped by its | |
| time-varying filter in the STFT domain -> 24 kHz waveform | |
| The waveform head has no neural upsampling network. The oscillator supplies the | |
| periodicity that a tiny decoder cannot render reliably on its own; the network | |
| only has to predict slowly varying spectral envelopes (complex cepstra), which | |
| is what keeps the voice from sounding buzzy or unstable at this size. | |
| """ | |
| import math | |
| import torch | |
| from torch import nn | |
| from torch.nn import functional as F | |
| from .config import Config | |
| from .text import PHONE_IDS, SYMBOL_TO_ID, VOWELS | |
| class UtteranceTooLong(ValueError): | |
| """The utterance needs more frames than ``max_frames``: split the input.""" | |
| def length_mask(lengths, size=None): | |
| size = int(lengths.max()) if size is None else size | |
| return torch.arange(size, device=lengths.device)[None, :] < lengths[:, None] | |
| def rotary_table(size, width, device): | |
| inverse = 1.0 / (10000 ** (torch.arange(0, width, 2, device=device, dtype=torch.float32) / width)) | |
| angle = torch.arange(size, device=device, dtype=torch.float32)[:, None] * inverse[None] | |
| return angle.cos(), angle.sin() | |
| def rotate(x, table): | |
| cos, sin = table[0].to(x.dtype), table[1].to(x.dtype) | |
| a, b = x[..., 0::2], x[..., 1::2] | |
| return torch.stack((a * cos - b * sin, a * sin + b * cos), dim=-1).flatten(-2) | |
| class Attention(nn.Module): | |
| """Self-attention with rotary (relative) positions: no absolute position | |
| table, so utterances longer than those seen in training stay in range.""" | |
| def __init__(self, channels, heads, dropout): | |
| super().__init__() | |
| self.heads, self.dropout = heads, dropout | |
| self.qkv = nn.Linear(channels, 3 * channels) | |
| self.out = nn.Linear(channels, channels) | |
| def forward(self, x, mask, table): | |
| b, t, c = x.shape | |
| q, k, v = self.qkv(x).view(b, t, 3, self.heads, c // self.heads).permute(2, 0, 3, 1, 4) | |
| y = F.scaled_dot_product_attention(rotate(q, table), rotate(k, table), v, attn_mask=mask[:, None, None, :], | |
| dropout_p=self.dropout if self.training else 0.0) | |
| return self.out(y.transpose(1, 2).reshape(b, t, c)) | |
| class Block(nn.Module): | |
| def __init__(self, c): | |
| super().__init__() | |
| self.norm1 = nn.LayerNorm(c.channels) | |
| self.attention = Attention(c.channels, c.heads, c.dropout) | |
| self.norm2 = nn.LayerNorm(c.channels) | |
| self.depthwise = nn.Conv1d(c.channels, c.channels, c.kernel, padding=c.kernel // 2, groups=c.channels) | |
| self.expand = nn.Linear(c.channels, c.feedforward) | |
| self.project = nn.Linear(c.feedforward, c.channels) | |
| self.dropout = nn.Dropout(c.dropout) | |
| def forward(self, x, mask, table): | |
| keep = mask[..., None] | |
| x = (x + self.dropout(self.attention(self.norm1(x), mask, table))) * keep | |
| # Mask before the convolution so padded positions cannot leak back in. | |
| y = self.depthwise((self.norm2(x) * keep).transpose(1, 2)).transpose(1, 2) | |
| y = self.project(self.dropout(F.gelu(self.expand(y)))) | |
| return (x + self.dropout(y)) * keep | |
| class Stack(nn.Module): | |
| def __init__(self, c, layers): | |
| super().__init__() | |
| self.blocks = nn.ModuleList(Block(c) for _ in range(layers)) | |
| self.norm = nn.LayerNorm(c.channels) | |
| self.width = c.channels // c.heads | |
| def forward(self, x, mask): | |
| table = rotary_table(x.shape[1], self.width, x.device) | |
| for block in self.blocks: | |
| x = block(x, mask, table) | |
| return self.norm(x) * mask[..., None] | |
| class Predictor(nn.Module): | |
| def __init__(self, c, outputs): | |
| super().__init__() | |
| p = c.predictor_channels | |
| self.conv1 = nn.Conv1d(c.channels, p, 3, padding=1) | |
| self.conv2 = nn.Conv1d(p, p, 3, padding=1) | |
| self.norm1, self.norm2 = nn.LayerNorm(p), nn.LayerNorm(p) | |
| self.drop = nn.Dropout(c.dropout) | |
| self.out = nn.Linear(p, outputs) | |
| def forward(self, x, mask): | |
| x = self.conv1((x * mask[..., None]).transpose(1, 2)).transpose(1, 2) | |
| x = self.drop(self.norm1(F.gelu(x))) * mask[..., None] | |
| x = self.conv2(x.transpose(1, 2)).transpose(1, 2) | |
| x = self.drop(self.norm2(F.gelu(x))) | |
| return self.out(x) * mask[..., None] | |
| def pulse_train(f0, sample_rate, hop, limit=0.47, phase=None): | |
| """Band-limited pulse train [B, T*hop] with unit power for f0 [B, T] in Hz. | |
| Frame values sit at samples i*hop and are interpolated in the log domain. | |
| The sum of all harmonics below ``limit * sample_rate`` has the closed form | |
| sin((K+1/2)x) / (2 sin(x/2)) - 1/2; the top harmonic fades in and out so the | |
| signal stays continuous while the pitch glides. Float64 keeps the phase exact. | |
| """ | |
| frames = f0.shape[1] | |
| knots = torch.log(f0.double().clamp_min(1.0)) | |
| knots = torch.cat((knots, knots[:, -1:]), dim=1) | |
| position = torch.arange(frames * hop, device=f0.device, dtype=torch.float64) / hop | |
| low = position.floor().long() | |
| weight = position - low | |
| frequency = torch.exp(knots[:, low] * (1 - weight) + knots[:, low + 1] * weight) | |
| theta = 2 * math.pi * torch.cumsum(frequency / sample_rate, dim=1) | |
| if phase is not None: | |
| theta = theta + phase.double()[:, None] | |
| theta = torch.remainder(theta, 2 * math.pi) | |
| harmonics = (limit * sample_rate / frequency).clamp_min(1.0) | |
| whole = torch.floor(harmonics) | |
| denominator = 2 * torch.sin(theta / 2) | |
| tiny = denominator.abs() < 1e-7 | |
| core = torch.where(tiny, whole, torch.sin((whole + 0.5) * theta) / torch.where(tiny, torch.ones_like(denominator), denominator) - 0.5) | |
| wave = core + (harmonics - whole) * torch.cos((whole + 1) * theta) | |
| return (wave * torch.sqrt(2.0 / harmonics)).float() | |
| def natural_pulse_train(f0, sample_rate, hop, limit=0.47, jitter=0.0, shimmer=0.0, generator=None): | |
| """Hindi port, inference only: pulse_train with the cycle-to-cycle irregularity of a real voice. | |
| jitter: each glottal cycle gets its own frequency factor (1 + jitter * N(0,1)); local jitter (mean |dT|/T) is | |
| ~1.13 x jitter, so 0.005 ~ 0.6% (normal voices: 0.3-1%). | |
| shimmer: each pulse gets its own amplitude factor (1 + shimmer * N(0,1)), interpolated from peak to peak so there | |
| is no step inside a cycle; 0.035 ~ 4% local shimmer (normal voices: 2-6%). | |
| Returns (wave [B, S] float32, unwrapped phase theta [B, S] float64 for pitch-synchronous effects).""" | |
| frames = f0.shape[1] | |
| knots = torch.log(f0.double().clamp_min(1.0)) | |
| knots = torch.cat((knots, knots[:, -1:]), dim=1) | |
| position = torch.arange(frames * hop, device=f0.device, dtype=torch.float64) / hop | |
| low = position.floor().long() | |
| weight = position - low | |
| frequency = torch.exp(knots[:, low] * (1 - weight) + knots[:, low + 1] * weight) | |
| if jitter > 0: | |
| cycle = torch.cumsum(frequency / sample_rate, dim=1).floor().long() | |
| factors = 1 + jitter * torch.randn(f0.shape[0], int(cycle.max()) + 2, generator=generator, dtype=torch.float64) | |
| frequency = frequency * factors.to(f0.device).gather(1, cycle) | |
| unwrapped = 2 * math.pi * torch.cumsum(frequency / sample_rate, dim=1) | |
| theta = torch.remainder(unwrapped, 2 * math.pi) | |
| harmonics = (limit * sample_rate / frequency).clamp_min(1.0) | |
| whole = torch.floor(harmonics) | |
| denominator = 2 * torch.sin(theta / 2) | |
| tiny = denominator.abs() < 1e-7 | |
| core = torch.where(tiny, whole, torch.sin((whole + 0.5) * theta) / torch.where(tiny, torch.ones_like(denominator), denominator) - 0.5) | |
| wave = (core + (harmonics - whole) * torch.cos((whole + 1) * theta)) * torch.sqrt(2.0 / harmonics) | |
| if shimmer > 0: | |
| cycles = unwrapped / (2 * math.pi) | |
| index = cycles.floor().long() | |
| frac = cycles - index | |
| amp = (1 + shimmer * torch.randn(f0.shape[0], int(index.max()) + 2, generator=generator, dtype=torch.float64)).clamp_min(0.2) | |
| amp = amp.to(f0.device) | |
| wave = wave * (amp.gather(1, index) * (1 - frac) + amp.gather(1, index + 1) * frac) | |
| return wave.float(), unwrapped | |
| def filtered(source, response, fft, hop, window): | |
| """Time-varying filtering: response [B, frames, bins] multiplies the STFT.""" | |
| spectrum = torch.stft(source, fft, hop, window=window, center=True, pad_mode="constant", return_complex=True) | |
| if spectrum.shape[-1] != response.shape[1]: | |
| raise ValueError("filter frames do not match the source") | |
| return torch.istft(spectrum * response.transpose(1, 2), fft, hop, window=window, center=True, length=source.shape[-1]) | |
| def cepstrum_response(causal, anticausal, fft): | |
| """Complex frequency response exp(FFT(cepstrum)). Causal coefficients give a | |
| minimum-phase filter; anticausal ones add a maximum-phase part (glottal pulse).""" | |
| buffer = causal.new_zeros(*causal.shape[:-1], fft) | |
| buffer[..., :causal.shape[-1]] = causal | |
| if anticausal is not None: | |
| buffer[..., fft - anticausal.shape[-1]:] = anticausal.flip(-1) | |
| spectrum = torch.fft.rfft(buffer, dim=-1) | |
| return torch.polar(torch.exp(spectrum.real.clamp(-14, 6)), spectrum.imag) | |
| class SourceFilter(nn.Module): | |
| """Harmonic-plus-noise synthesis controlled per frame by two linear heads.""" | |
| def __init__(self, c): | |
| super().__init__() | |
| self.config = c | |
| self.harmonic_steps, self.noise_steps = c.hop // c.synth_hop, c.hop // c.noise_hop | |
| self.harmonic = nn.Linear(c.channels, c.cep_causal + c.cep_anti + self.harmonic_steps) | |
| self.noise = nn.Linear(c.channels, c.noise_cep + self.noise_steps) | |
| for head, level in ((self.harmonic, -1.5), (self.noise, -2.5)): # start quiet and spectrally flat | |
| nn.init.zeros_(head.weight) | |
| nn.init.zeros_(head.bias) | |
| head.bias.data[0] = level | |
| def lifter(size, first): | |
| scale = 1.0 / (1.0 + torch.arange(first, first + size, dtype=torch.float32) / 4) | |
| if first == 0: | |
| scale[0] = 2.0 # coefficient 0 is the log gain and needs range | |
| return scale | |
| self.register_buffer("causal_lifter", lifter(c.cep_causal, 0), persistent=False) | |
| self.register_buffer("anti_lifter", lifter(c.cep_anti, 1), persistent=False) | |
| self.register_buffer("noise_lifter", lifter(c.noise_cep, 0), persistent=False) | |
| self.register_buffer("synth_window", torch.hann_window(c.synth_fft), persistent=False) | |
| self.register_buffer("noise_window", torch.hann_window(c.noise_fft), persistent=False) | |
| def _take(values, index): | |
| return values.gather(1, index[..., None].expand(-1, -1, values.shape[-1])) | |
| def _subframes(self, frames, lengths, steps): | |
| """Sub-frame j uses frame j // steps; indices past an utterance's end | |
| repeat its last sub-frame so padding never changes real audio. Two extra | |
| sub-frames cover the half window of zeros appended to the source.""" | |
| sub = torch.arange(steps * frames + 3, device=lengths.device)[None].expand(len(lengths), -1) | |
| return torch.minimum(sub, (steps * lengths - 1).clamp_min(0)[:, None]) | |
| def _shape(source, response, fft, hop, window): | |
| # Half a window of trailing zeros: the frames around an utterance's end are | |
| # then the same whether it stands alone or is padded inside a batch. | |
| return filtered(F.pad(source, (0, fft // 2)), response, fft, hop, window)[:, :source.shape[-1]] | |
| def _shape_mix(self, a, b, taper, response, fft, hop, window): | |
| """_shape for a source made of `a` below and `b` above a crossover: spectrum(a) * taper + spectrum(b) * | |
| sqrt(1 - taper^2), then the time-varying filter. Equal power density in a and b keeps the envelope.""" | |
| S = a.shape[-1] | |
| sa = torch.stft(F.pad(a, (0, fft // 2)), fft, hop, window=window, center=True, pad_mode="constant", return_complex=True) | |
| sb = torch.stft(F.pad(b, (0, fft // 2)), fft, hop, window=window, center=True, pad_mode="constant", return_complex=True) | |
| t = taper[None, :, None] | |
| mixed = sa * t + sb * torch.sqrt((1 - t * t).clamp_min(0)) | |
| y = torch.istft(mixed * response.transpose(1, 2), fft, hop, window=window, center=True, length=S + fft // 2) | |
| return y[:, :S] | |
| def forward(self, hidden, f0, lengths, noise=None, phase=None, voiced=None): | |
| c = self.config | |
| nat = getattr(self, "naturalness", None) # Hindi port, inference only (see natural_pulse_train) | |
| frames = hidden.shape[1] | |
| samples = frames * c.hop | |
| with torch.autocast(device_type=hidden.device.type, enabled=False): | |
| hidden = hidden.float() | |
| audible = length_mask(lengths * c.hop, samples) | |
| last = (lengths - 1).clamp_min(0)[:, None] | |
| # Harmonic branch: envelope interpolated between frames, one gain per sub-frame. | |
| h = self.harmonic(hidden) | |
| split = c.cep_causal + c.cep_anti | |
| sub = self._subframes(frames, lengths, self.harmonic_steps) | |
| here, following = sub // self.harmonic_steps, torch.minimum((sub + 1) // self.harmonic_steps, last) | |
| causal = 0.5 * (self._take(h[..., :c.cep_causal], here) + self._take(h[..., :c.cep_causal], following)) * self.causal_lifter | |
| anti = 0.5 * (self._take(h[..., c.cep_causal:split], here) + self._take(h[..., c.cep_causal:split], following)) * self.anti_lifter | |
| gains = h[..., split:].flatten(1) | |
| if getattr(self, "smooth_gains", False): | |
| # Hindi port, inference only: the GAN-trained model flickers the per-sub-frame harmonic gain up and down | |
| # (~94 Hz at synth_hop 128, up to ~20 dB) at transitions, heard as a gargle. [1, 2, 1]/4 removes the | |
| # alternation and keeps changes slower than one frame. | |
| gains = F.conv1d(F.pad(gains[:, None], (1, 1), mode="replicate"), gains.new_tensor([[[0.25, 0.5, 0.25]]]))[:, 0] | |
| causal = torch.cat((causal[..., :1] + gains.gather(1, sub)[..., None], causal[..., 1:]), dim=-1) | |
| if getattr(self, "adaptive_lifter", 0.0): | |
| # Hindi port, inference only: above sr/80 = 300 Hz the 80 causal coefficients can describe the harmonic | |
| # comb itself (quefrency 1/F0). Taper the coefficients from adaptive_lifter * sr/F0 to sr/F0 so the | |
| # filter stays a smooth vocal-tract envelope at high pitch (no effect below ~300 Hz). | |
| f0_sub = f0.float().gather(1, torch.minimum(here, last)).clamp_min(1.0) # [B, S] | |
| k = torch.arange(c.cep_causal, device=causal.device, dtype=torch.float32) | |
| period = c.sample_rate / f0_sub[..., None] # in coefficients | |
| start = self.adaptive_lifter * period | |
| taper = 0.5 * (1 + torch.cos(math.pi * ((k - start) / (period - start).clamp_min(1.0)).clamp(0, 1))) | |
| taper[..., 0] = 1.0 | |
| causal = causal * taper | |
| harmonic_response = cepstrum_response(causal, anti, c.synth_fft) | |
| # Noise branch: one envelope per frame, four gains per frame for sharp consonants. | |
| n = self.noise(hidden) | |
| sub = self._subframes(frames, lengths, self.noise_steps) | |
| shape = self._take(n[..., :c.noise_cep], sub // self.noise_steps) * self.noise_lifter | |
| if getattr(self, "smooth_noise_env", False): | |
| # Hindi port, inference only: the noise envelope is held for a whole frame and jumps at every frame | |
| # boundary, an amplitude flutter at the frame rate (24000/256 = 93.75 Hz) heard as a buzz on earbuds. | |
| # Interpolate it linearly between frame centres instead. | |
| pos = sub.float() / self.noise_steps | |
| lo = pos.floor().long() | |
| hi = torch.minimum(lo + 1, last) | |
| w = (pos - lo.float())[..., None] | |
| shape = ((1 - w) * self._take(n[..., :c.noise_cep], lo) + w * self._take(n[..., :c.noise_cep], hi)) * self.noise_lifter | |
| noise_gains = n[..., c.noise_cep:].flatten(1) | |
| if getattr(self, "smooth_noise_gains", False): | |
| # same flicker in the noise gains (a per-frame sawtooth); [1, 2, 2, 2, 1]/8 removes period-2 and period-4 parts | |
| noise_gains = F.conv1d(F.pad(noise_gains[:, None], (2, 2), mode="replicate"), | |
| noise_gains.new_tensor([[[1, 2, 2, 2, 1]]]) / 8)[:, 0] | |
| shape = torch.cat((shape[..., :1] + noise_gains.gather(1, sub)[..., None], shape[..., 1:]), dim=-1) | |
| noise_response = cepstrum_response(shape, None, c.noise_fft) | |
| # Hold the last real pitch value through padding: the pitch glide inside an | |
| # utterance's final frame must not depend on what the batch was padded with. | |
| f0 = f0.float().gather(1, torch.minimum(torch.arange(frames, device=f0.device)[None], last)) | |
| if nat: | |
| gen = torch.Generator().manual_seed(int(nat.get("seed", 0)) + 104729) | |
| pulses, theta = natural_pulse_train(f0, c.sample_rate, c.hop, c.harmonic_limit, nat.get("jitter", 0.0), | |
| nat.get("shimmer", 0.0), gen) | |
| pulses = pulses * audible | |
| else: | |
| pulses = pulse_train(f0, c.sample_rate, c.hop, c.harmonic_limit, phase) * audible | |
| if getattr(self, "mixed_excitation", False) and voiced is not None: | |
| # Hindi port, inference only. The oscillator runs through unvoiced sounds too, and the model learned to | |
| # build much of /s/, /k/, /t/ and breaths from PULSES (measured: the pulse branch is ~8 dB louder than | |
| # the noise branch in unvoiced frames) -> an electronic buzz under the consonants (the "buzzing | |
| # artifact" of DDSP vocoders, Wu et al. 2022). Where a frame is unvoiced, excite the same filter with | |
| # white noise of the same power instead (mixed excitation, as MELP does); crossfade over ~1 frame. | |
| vm = F.avg_pool1d(F.pad(voiced.float()[:, None], (1, 1), mode="replicate"), 3, 1)[:, 0] | |
| vm = torch.cat((vm, vm[:, -1:]), dim=1) | |
| position = torch.arange(samples, device=vm.device, dtype=torch.float32) / c.hop | |
| low = position.floor().long() | |
| weight = position - low | |
| vs = (vm[:, low] * (1 - weight) + vm[:, low + 1] * weight).to(pulses.device) | |
| alt = torch.randn(hidden.shape[0], samples, generator=torch.Generator().manual_seed(977)).to(pulses.device) | |
| pulses = (pulses * vs + alt * torch.sqrt((1 - vs * vs).clamp_min(0))) * audible | |
| if noise is None: | |
| noise = torch.randn(hidden.shape[0], samples, device=hidden.device) | |
| if nat and nat.get("breath_pulse", 0.0) > 0 and voiced is not None: | |
| # breath noise puffs with each glottal cycle (strongest in the open phase, a quarter cycle before the | |
| # closure pulse at theta = 0), only where the frame is voiced; mean power is unchanged. | |
| d, kappa = float(nat["breath_pulse"]), 2.0 | |
| v = F.avg_pool1d(F.pad(voiced.float()[:, None], (2, 2), mode="replicate"), 5, 1)[:, 0] | |
| v = v.repeat_interleave(c.hop, dim=1)[:, :samples].to(noise.device) | |
| g = torch.exp(kappa * (torch.cos(theta - (-math.pi / 2)) - 1)).float().to(noise.device) | |
| mean_g = math.exp(-kappa) * float(torch.special.i0(torch.tensor(kappa))) | |
| noise = noise * torch.sqrt((1 - d * v) + d * v * g / mean_g) | |
| if nat and nat.get("mvf_hz", 0.0) > 0: | |
| # voice turns into breath at high frequencies: above the maximum voiced frequency the harmonic branch is | |
| # excited by noise instead of pulses (same filter, same power density), cosine crossover | |
| lo, width = float(nat["mvf_hz"]), float(nat.get("mvf_width_hz", 2500.0)) | |
| freqs = torch.arange(c.synth_fft // 2 + 1, dtype=torch.float32) * c.sample_rate / c.synth_fft | |
| taper = 0.5 * (1 + torch.cos(math.pi * ((freqs - lo) / width).clamp(0, 1))) | |
| breath = torch.randn(hidden.shape[0], samples, generator=torch.Generator().manual_seed(int(nat.get("seed", 0)) + 7)) | |
| harmonic = self._shape_mix(pulses, breath.to(pulses.device) * audible, taper.to(pulses.device), harmonic_response, | |
| c.synth_fft, c.synth_hop, self.synth_window) | |
| else: | |
| harmonic = self._shape(pulses, harmonic_response, c.synth_fft, c.synth_hop, self.synth_window) | |
| wave = harmonic + self._shape(noise * audible, noise_response, c.noise_fft, c.noise_hop, self.noise_window) | |
| return wave * audible | |
| class TinyVoca(nn.Module): | |
| def __init__(self, config=Config()): | |
| super().__init__() | |
| self.config = c = config | |
| self.tokens = nn.Embedding(c.vocab_size, c.channels, padding_idx=0) | |
| self.stress = nn.Embedding(3, c.channels) | |
| self.voices = nn.Embedding(len(c.speakers), c.channels) | |
| self.encoder = Stack(c, c.encoder_layers) | |
| # Outputs per token: log(1 + frames) and, for pause-capable tokens, a | |
| # "has frames at all" logit. A pause is absent more often than present, so | |
| # one regression would average both cases into a pause that is too short. | |
| self.duration = Predictor(c, 2) | |
| self.frame_voices = nn.Embedding(len(c.speakers), c.channels) | |
| self.frame_position = nn.Linear(3, c.channels) | |
| self.prosody_stack = Stack(c, c.prosody_layers) | |
| # Outputs: continuous log(F0/reference), voiced logit, normalized log RMS energy. | |
| self.prosody_head = nn.Linear(c.channels, 3) | |
| self.prosody_projection = nn.Linear(3, c.channels) | |
| self.acoustic_stack = Stack(c, c.acoustic_layers) | |
| self.vocoder = SourceFilter(c) | |
| phones = torch.zeros(c.vocab_size, dtype=torch.bool) | |
| phones[sorted(PHONE_IDS)] = True | |
| self.register_buffer("phone_tokens", phones, persistent=False) | |
| vowels = torch.zeros(c.vocab_size, dtype=torch.bool) | |
| vowels[[SYMBOL_TO_ID[v] for v in VOWELS]] = True | |
| self.register_buffer("vowel_tokens", vowels, persistent=False) | |
| def encode(self, tokens, stress, lengths, speakers): | |
| if tokens.ndim != 2 or stress.shape != tokens.shape or lengths.shape != (tokens.shape[0],) or speakers.shape != lengths.shape: | |
| raise ValueError("expected tokens[B,N], stress[B,N], lengths[B], speakers[B]") | |
| if tokens.shape[1] > self.config.max_tokens or bool((lengths < 1).any()) or bool((lengths > tokens.shape[1]).any()): | |
| raise ValueError("invalid token length") | |
| mask = length_mask(lengths, tokens.shape[1]) | |
| x = (self.tokens(tokens) + self.stress(stress) + self.voices(speakers)[:, None]) * mask[..., None] | |
| return self.encoder(x, mask), mask | |
| def expand(self, hidden, durations, token_mask, speakers): | |
| """Repeat token states by integer durations (zero is allowed for pause | |
| tokens) and add each frame's position inside its token.""" | |
| if durations.shape != token_mask.shape or bool((durations < 0).any()) or bool((durations[~token_mask] != 0).any()): | |
| raise ValueError("invalid durations") | |
| lengths = durations.sum(1) | |
| if bool((lengths < 1).any()): | |
| raise ValueError("utterance has no frames") | |
| if bool((lengths > self.config.max_frames).any()): | |
| raise UtteranceTooLong("utterance exceeds max_frames; split the input") | |
| size = int(lengths.max()) | |
| ends = durations.cumsum(1) | |
| frame = torch.arange(size, device=hidden.device)[None].expand(hidden.shape[0], size).contiguous() | |
| index = torch.searchsorted(ends, frame, right=True).clamp(max=durations.shape[1] - 1) | |
| owned = durations.gather(1, index).clamp_min(1) | |
| offset = frame - (ends.gather(1, index) - durations.gather(1, index)) | |
| relative = (offset.to(hidden.dtype) + 0.5) / owned | |
| features = torch.stack((relative, 1 - relative, torch.log1p(owned.to(hidden.dtype)) / 4), dim=-1) | |
| mask = frame < lengths[:, None] | |
| expanded = hidden.gather(1, index[..., None].expand(-1, -1, hidden.shape[-1])) | |
| expanded = expanded + self.frame_position(features) + self.frame_voices(speakers)[:, None] | |
| return expanded * mask[..., None], mask, lengths | |
| def predict_prosody(self, frames, frame_mask): | |
| hidden = self.prosody_stack(frames, frame_mask) | |
| return hidden, self.prosody_head(hidden) * frame_mask[..., None] | |
| def hard_prosody(prediction): | |
| """What the decoder is conditioned on when it is not teacher-forced.""" | |
| return torch.stack((prediction[..., 0].clamp(-2.0, 1.5), (prediction[..., 1] > 0).to(prediction.dtype), | |
| prediction[..., 2].clamp(-4.5, 3.0)), dim=-1) | |
| def decode(self, prosody_hidden, condition, frame_mask, noise=None): | |
| h = prosody_hidden + self.prosody_projection(condition) | |
| if getattr(self, "frame_noise", None) is not None: | |
| # Hindi port (anti-blur fine-tune): a random vector per frame lets the acoustic stack pick ONE natural | |
| # version of the spectrum instead of the average of all of them (an average is what sounds blurred). | |
| if noise is None: | |
| t = getattr(self, "noise_temperature", 1.0) | |
| noise = t * torch.randn(*h.shape[:2], self.frame_noise.in_features, device=h.device) | |
| h = h + self.frame_noise(noise.to(h.dtype)) | |
| return self.acoustic_stack(h * frame_mask[..., None], frame_mask) | |
| def add_frame_noise(self, dim=16): | |
| """Add the per-frame noise input (zero-initialised: the model's output is unchanged until it is trained).""" | |
| self.frame_noise = nn.Linear(dim, self.config.channels) | |
| nn.init.zeros_(self.frame_noise.weight) | |
| nn.init.zeros_(self.frame_noise.bias) | |
| return self | |
| def frequency(self, log_f0): | |
| c = self.config | |
| return (c.pitch_reference_hz * torch.exp(log_f0.float())).clamp(c.f0_min, c.f0_max) | |
| def predict_durations(self, hidden, mask, tokens, speed=1.0): | |
| output = self.duration(hidden, mask).float() | |
| frames = torch.round(torch.expm1(output[..., 0].clamp(0, math.log1p(400))) / speed).long().clamp_min(1) | |
| # Every phoneme is audible for at least one frame. A pause-capable token | |
| # (edge, punctuation, word boundary) lasts its predicted length when the | |
| # presence logit says there is a pause, and vanishes otherwise. | |
| present = self.phone_tokens[tokens] | (output[..., 1] > 0) | |
| return torch.where(present, frames, torch.zeros_like(frames)) * mask | |
| def widen_pitch(prosody, frame_mask, pitch_range): | |
| """Scale the predicted pitch movement around each utterance's voiced mean. | |
| Regression predicts a flatter contour than the recordings; values above 1 | |
| restore range (1.0 leaves the prediction untouched).""" | |
| if pitch_range == 1.0: | |
| return prosody | |
| voiced = ((prosody[..., 1] > 0) & frame_mask).to(prosody.dtype) | |
| mean = (prosody[..., 0] * voiced).sum(1, keepdim=True) / voiced.sum(1, keepdim=True).clamp_min(1) | |
| pitch = mean + (prosody[..., 0] - mean) * pitch_range | |
| return torch.cat((pitch[..., None], prosody[..., 1:]), dim=-1) * frame_mask[..., None] | |
| def pitch_wobble(shape, semitones, seed, device): | |
| """Small random pitch movement for the oscillator only (log-F0 units). | |
| Recorded pitch has cycle-to-cycle irregularity that a predicted contour | |
| lacks; two thirds of the wobble is slow drift, one third frame-level.""" | |
| generator = torch.Generator().manual_seed(seed + 7919) | |
| fast = torch.randn(shape, generator=generator) | |
| slow = F.avg_pool1d(torch.randn(shape[0], 1, shape[1] + 16, generator=generator), 17, stride=1)[:, 0] | |
| slow = slow / slow.std().clamp_min(1e-6) | |
| return ((2 * slow + fast) / 5 ** 0.5 * semitones * math.log(2) / 12).to(device) | |
| def synthesize(self, tokens, stress, lengths, speakers, speed=1.0, pitch_shift=0.0, seed=0, pitch_range=1.0, pitch_jitter=0.0, | |
| voiced_vowels=True, pitch_smooth=0, octave_fix=0.0): | |
| """Free-running synthesis: predicted durations, pitch, voicing and energy. | |
| Hindi port: ``pitch_smooth`` = width (frames, odd) of a running median over the predicted log F0. The YIN pitch | |
| targets carry ~10 jumps of > 4 semitones per 1,000 voiced frames (tracker errors), the model learned them, and | |
| they are heard as hiccups; a 5-frame (53 ms) median removes 10 ms blips and keeps the intonation.""" | |
| if not math.isfinite(speed) or not 0.25 <= speed <= 4 or not math.isfinite(pitch_shift) or not 0 <= pitch_range <= 3: | |
| raise ValueError("speed must be between 0.25 and 4, pitch_shift finite and pitch_range between 0 and 3") | |
| hidden, mask = self.encode(tokens, stress, lengths, speakers) | |
| durations = self.predict_durations(hidden, mask, tokens, speed) | |
| frames, frame_mask, frame_lengths = self.expand(hidden, durations, mask, speakers) | |
| prosody_hidden, prosody = self.predict_prosody(frames, frame_mask) | |
| if voiced_vowels: | |
| # The pitch tracker missed creaky vowels in the recordings, and the | |
| # voicing predictor learned to drop whole vowels (heard as a whispered | |
| # word). A vowel is voiced by definition, so its frames are kept voiced. | |
| ends = durations.cumsum(1) | |
| frame = torch.arange(frames.shape[1], device=tokens.device)[None].expand(len(lengths), -1).contiguous() | |
| owner = torch.searchsorted(ends, frame, right=True).clamp(max=tokens.shape[1] - 1) | |
| vowel = self.vowel_tokens[tokens].gather(1, owner) & frame_mask | |
| prosody = torch.cat((prosody[..., :1], torch.where(vowel, prosody[..., 1].clamp_min(2.0), prosody[..., 1])[..., None], | |
| prosody[..., 2:]), dim=-1) | |
| if octave_fix: | |
| # YIN octave errors in the training targets taught the model short excursions to ~2x the pitch (heard as | |
| # hiccups). Pitch above the utterance's voiced median is soft-limited: median + T*tanh(excess/T), with | |
| # T = log(octave_fix), so a rise can approach but never pass octave_fix x the median, with no new edges. | |
| voiced = (prosody[..., 1] > 0) & frame_mask | |
| logf0 = prosody[..., 0] | |
| masked = torch.where(voiced, logf0, torch.full_like(logf0, float("nan"))) | |
| median = masked.nanmedian(1, keepdim=True).values.nan_to_num(0.0) | |
| limit = math.log(octave_fix) | |
| excess = logf0 - median | |
| limited = torch.where(excess > 0, median + limit * torch.tanh(excess / limit), logf0) | |
| prosody = torch.cat((limited[..., None], prosody[..., 1:]), dim=-1) | |
| if pitch_smooth and pitch_smooth > 1: | |
| w = int(pitch_smooth) | 1 | |
| padded = F.pad(prosody[..., 0][:, None], (w // 2, w // 2), mode="replicate")[:, 0] | |
| smoothed = padded.unfold(1, w, 1).median(-1).values | |
| prosody = torch.cat((smoothed[..., None], prosody[..., 1:]), dim=-1) * frame_mask[..., None] | |
| prosody = self.widen_pitch(prosody, frame_mask, pitch_range) | |
| acoustic = self.decode(prosody_hidden, self.hard_prosody(prosody), frame_mask) | |
| # One noise stream per row, drawn at the row's own length, so batching or | |
| # padding never changes a row's audio. | |
| noise = nn.utils.rnn.pad_sequence([torch.randn(int(n) * self.config.hop, generator=torch.Generator().manual_seed(seed + row)) | |
| for row, n in enumerate(frame_lengths)], batch_first=True) | |
| pitch = prosody[..., 0] + pitch_shift | |
| if pitch_jitter > 0: | |
| pitch = pitch + self.pitch_wobble(pitch.shape, pitch_jitter, seed, pitch.device) | |
| wave = self.vocoder(acoustic, self.frequency(pitch), frame_lengths, noise=noise.to(hidden.device), | |
| voiced=(prosody[..., 1] > 0) & frame_mask) | |
| return wave, frame_lengths * self.config.hop, {"durations": durations, "prosody": prosody, "frame_mask": frame_mask} | |