RL10x's picture
Add inference code (Python + C++ engine + web page)
ab1b068 verified
Raw History Blame Contribute Delete
33.1 kB
"""tiny-voca v2: feed-forward transformer with a pitch-driven source-filter head.
All weights start random. Tensor conventions: hidden [B,T,C], frame features
[B,T], waveform [B,S]. Acoustic frame i is centred on sample ``i * hop``.
phoneme + stress + voice -> transformer encoder -> duration predictor
-> repeat tokens by duration (supervised durations while training)
-> frame transformer A -> pitch / voicing / energy
-> frame transformer B -> filter parameters
-> harmonic pulse train (from pitch) and noise, each shaped by its
time-varying filter in the STFT domain -> 24 kHz waveform
The waveform head has no neural upsampling network. The oscillator supplies the
periodicity that a tiny decoder cannot render reliably on its own; the network
only has to predict slowly varying spectral envelopes (complex cepstra), which
is what keeps the voice from sounding buzzy or unstable at this size.
"""
import math
import torch
from torch import nn
from torch.nn import functional as F
from .config import Config
from .text import PHONE_IDS, SYMBOL_TO_ID, VOWELS
class UtteranceTooLong(ValueError):
"""The utterance needs more frames than ``max_frames``: split the input."""
def length_mask(lengths, size=None):
size = int(lengths.max()) if size is None else size
return torch.arange(size, device=lengths.device)[None, :] < lengths[:, None]
def rotary_table(size, width, device):
inverse = 1.0 / (10000 ** (torch.arange(0, width, 2, device=device, dtype=torch.float32) / width))
angle = torch.arange(size, device=device, dtype=torch.float32)[:, None] * inverse[None]
return angle.cos(), angle.sin()
def rotate(x, table):
cos, sin = table[0].to(x.dtype), table[1].to(x.dtype)
a, b = x[..., 0::2], x[..., 1::2]
return torch.stack((a * cos - b * sin, a * sin + b * cos), dim=-1).flatten(-2)
class Attention(nn.Module):
"""Self-attention with rotary (relative) positions: no absolute position
table, so utterances longer than those seen in training stay in range."""
def __init__(self, channels, heads, dropout):
super().__init__()
self.heads, self.dropout = heads, dropout
self.qkv = nn.Linear(channels, 3 * channels)
self.out = nn.Linear(channels, channels)
def forward(self, x, mask, table):
b, t, c = x.shape
q, k, v = self.qkv(x).view(b, t, 3, self.heads, c // self.heads).permute(2, 0, 3, 1, 4)
y = F.scaled_dot_product_attention(rotate(q, table), rotate(k, table), v, attn_mask=mask[:, None, None, :],
dropout_p=self.dropout if self.training else 0.0)
return self.out(y.transpose(1, 2).reshape(b, t, c))
class Block(nn.Module):
def __init__(self, c):
super().__init__()
self.norm1 = nn.LayerNorm(c.channels)
self.attention = Attention(c.channels, c.heads, c.dropout)
self.norm2 = nn.LayerNorm(c.channels)
self.depthwise = nn.Conv1d(c.channels, c.channels, c.kernel, padding=c.kernel // 2, groups=c.channels)
self.expand = nn.Linear(c.channels, c.feedforward)
self.project = nn.Linear(c.feedforward, c.channels)
self.dropout = nn.Dropout(c.dropout)
def forward(self, x, mask, table):
keep = mask[..., None]
x = (x + self.dropout(self.attention(self.norm1(x), mask, table))) * keep
# Mask before the convolution so padded positions cannot leak back in.
y = self.depthwise((self.norm2(x) * keep).transpose(1, 2)).transpose(1, 2)
y = self.project(self.dropout(F.gelu(self.expand(y))))
return (x + self.dropout(y)) * keep
class Stack(nn.Module):
def __init__(self, c, layers):
super().__init__()
self.blocks = nn.ModuleList(Block(c) for _ in range(layers))
self.norm = nn.LayerNorm(c.channels)
self.width = c.channels // c.heads
def forward(self, x, mask):
table = rotary_table(x.shape[1], self.width, x.device)
for block in self.blocks:
x = block(x, mask, table)
return self.norm(x) * mask[..., None]
class Predictor(nn.Module):
def __init__(self, c, outputs):
super().__init__()
p = c.predictor_channels
self.conv1 = nn.Conv1d(c.channels, p, 3, padding=1)
self.conv2 = nn.Conv1d(p, p, 3, padding=1)
self.norm1, self.norm2 = nn.LayerNorm(p), nn.LayerNorm(p)
self.drop = nn.Dropout(c.dropout)
self.out = nn.Linear(p, outputs)
def forward(self, x, mask):
x = self.conv1((x * mask[..., None]).transpose(1, 2)).transpose(1, 2)
x = self.drop(self.norm1(F.gelu(x))) * mask[..., None]
x = self.conv2(x.transpose(1, 2)).transpose(1, 2)
x = self.drop(self.norm2(F.gelu(x)))
return self.out(x) * mask[..., None]
def pulse_train(f0, sample_rate, hop, limit=0.47, phase=None):
"""Band-limited pulse train [B, T*hop] with unit power for f0 [B, T] in Hz.
Frame values sit at samples i*hop and are interpolated in the log domain.
The sum of all harmonics below ``limit * sample_rate`` has the closed form
sin((K+1/2)x) / (2 sin(x/2)) - 1/2; the top harmonic fades in and out so the
signal stays continuous while the pitch glides. Float64 keeps the phase exact.
"""
frames = f0.shape[1]
knots = torch.log(f0.double().clamp_min(1.0))
knots = torch.cat((knots, knots[:, -1:]), dim=1)
position = torch.arange(frames * hop, device=f0.device, dtype=torch.float64) / hop
low = position.floor().long()
weight = position - low
frequency = torch.exp(knots[:, low] * (1 - weight) + knots[:, low + 1] * weight)
theta = 2 * math.pi * torch.cumsum(frequency / sample_rate, dim=1)
if phase is not None:
theta = theta + phase.double()[:, None]
theta = torch.remainder(theta, 2 * math.pi)
harmonics = (limit * sample_rate / frequency).clamp_min(1.0)
whole = torch.floor(harmonics)
denominator = 2 * torch.sin(theta / 2)
tiny = denominator.abs() < 1e-7
core = torch.where(tiny, whole, torch.sin((whole + 0.5) * theta) / torch.where(tiny, torch.ones_like(denominator), denominator) - 0.5)
wave = core + (harmonics - whole) * torch.cos((whole + 1) * theta)
return (wave * torch.sqrt(2.0 / harmonics)).float()
def natural_pulse_train(f0, sample_rate, hop, limit=0.47, jitter=0.0, shimmer=0.0, generator=None):
"""Hindi port, inference only: pulse_train with the cycle-to-cycle irregularity of a real voice.
jitter: each glottal cycle gets its own frequency factor (1 + jitter * N(0,1)); local jitter (mean |dT|/T) is
~1.13 x jitter, so 0.005 ~ 0.6% (normal voices: 0.3-1%).
shimmer: each pulse gets its own amplitude factor (1 + shimmer * N(0,1)), interpolated from peak to peak so there
is no step inside a cycle; 0.035 ~ 4% local shimmer (normal voices: 2-6%).
Returns (wave [B, S] float32, unwrapped phase theta [B, S] float64 for pitch-synchronous effects)."""
frames = f0.shape[1]
knots = torch.log(f0.double().clamp_min(1.0))
knots = torch.cat((knots, knots[:, -1:]), dim=1)
position = torch.arange(frames * hop, device=f0.device, dtype=torch.float64) / hop
low = position.floor().long()
weight = position - low
frequency = torch.exp(knots[:, low] * (1 - weight) + knots[:, low + 1] * weight)
if jitter > 0:
cycle = torch.cumsum(frequency / sample_rate, dim=1).floor().long()
factors = 1 + jitter * torch.randn(f0.shape[0], int(cycle.max()) + 2, generator=generator, dtype=torch.float64)
frequency = frequency * factors.to(f0.device).gather(1, cycle)
unwrapped = 2 * math.pi * torch.cumsum(frequency / sample_rate, dim=1)
theta = torch.remainder(unwrapped, 2 * math.pi)
harmonics = (limit * sample_rate / frequency).clamp_min(1.0)
whole = torch.floor(harmonics)
denominator = 2 * torch.sin(theta / 2)
tiny = denominator.abs() < 1e-7
core = torch.where(tiny, whole, torch.sin((whole + 0.5) * theta) / torch.where(tiny, torch.ones_like(denominator), denominator) - 0.5)
wave = (core + (harmonics - whole) * torch.cos((whole + 1) * theta)) * torch.sqrt(2.0 / harmonics)
if shimmer > 0:
cycles = unwrapped / (2 * math.pi)
index = cycles.floor().long()
frac = cycles - index
amp = (1 + shimmer * torch.randn(f0.shape[0], int(index.max()) + 2, generator=generator, dtype=torch.float64)).clamp_min(0.2)
amp = amp.to(f0.device)
wave = wave * (amp.gather(1, index) * (1 - frac) + amp.gather(1, index + 1) * frac)
return wave.float(), unwrapped
def filtered(source, response, fft, hop, window):
"""Time-varying filtering: response [B, frames, bins] multiplies the STFT."""
spectrum = torch.stft(source, fft, hop, window=window, center=True, pad_mode="constant", return_complex=True)
if spectrum.shape[-1] != response.shape[1]:
raise ValueError("filter frames do not match the source")
return torch.istft(spectrum * response.transpose(1, 2), fft, hop, window=window, center=True, length=source.shape[-1])
def cepstrum_response(causal, anticausal, fft):
"""Complex frequency response exp(FFT(cepstrum)). Causal coefficients give a
minimum-phase filter; anticausal ones add a maximum-phase part (glottal pulse)."""
buffer = causal.new_zeros(*causal.shape[:-1], fft)
buffer[..., :causal.shape[-1]] = causal
if anticausal is not None:
buffer[..., fft - anticausal.shape[-1]:] = anticausal.flip(-1)
spectrum = torch.fft.rfft(buffer, dim=-1)
return torch.polar(torch.exp(spectrum.real.clamp(-14, 6)), spectrum.imag)
class SourceFilter(nn.Module):
"""Harmonic-plus-noise synthesis controlled per frame by two linear heads."""
def __init__(self, c):
super().__init__()
self.config = c
self.harmonic_steps, self.noise_steps = c.hop // c.synth_hop, c.hop // c.noise_hop
self.harmonic = nn.Linear(c.channels, c.cep_causal + c.cep_anti + self.harmonic_steps)
self.noise = nn.Linear(c.channels, c.noise_cep + self.noise_steps)
for head, level in ((self.harmonic, -1.5), (self.noise, -2.5)): # start quiet and spectrally flat
nn.init.zeros_(head.weight)
nn.init.zeros_(head.bias)
head.bias.data[0] = level
def lifter(size, first):
scale = 1.0 / (1.0 + torch.arange(first, first + size, dtype=torch.float32) / 4)
if first == 0:
scale[0] = 2.0 # coefficient 0 is the log gain and needs range
return scale
self.register_buffer("causal_lifter", lifter(c.cep_causal, 0), persistent=False)
self.register_buffer("anti_lifter", lifter(c.cep_anti, 1), persistent=False)
self.register_buffer("noise_lifter", lifter(c.noise_cep, 0), persistent=False)
self.register_buffer("synth_window", torch.hann_window(c.synth_fft), persistent=False)
self.register_buffer("noise_window", torch.hann_window(c.noise_fft), persistent=False)
@staticmethod
def _take(values, index):
return values.gather(1, index[..., None].expand(-1, -1, values.shape[-1]))
def _subframes(self, frames, lengths, steps):
"""Sub-frame j uses frame j // steps; indices past an utterance's end
repeat its last sub-frame so padding never changes real audio. Two extra
sub-frames cover the half window of zeros appended to the source."""
sub = torch.arange(steps * frames + 3, device=lengths.device)[None].expand(len(lengths), -1)
return torch.minimum(sub, (steps * lengths - 1).clamp_min(0)[:, None])
@staticmethod
def _shape(source, response, fft, hop, window):
# Half a window of trailing zeros: the frames around an utterance's end are
# then the same whether it stands alone or is padded inside a batch.
return filtered(F.pad(source, (0, fft // 2)), response, fft, hop, window)[:, :source.shape[-1]]
def _shape_mix(self, a, b, taper, response, fft, hop, window):
"""_shape for a source made of `a` below and `b` above a crossover: spectrum(a) * taper + spectrum(b) *
sqrt(1 - taper^2), then the time-varying filter. Equal power density in a and b keeps the envelope."""
S = a.shape[-1]
sa = torch.stft(F.pad(a, (0, fft // 2)), fft, hop, window=window, center=True, pad_mode="constant", return_complex=True)
sb = torch.stft(F.pad(b, (0, fft // 2)), fft, hop, window=window, center=True, pad_mode="constant", return_complex=True)
t = taper[None, :, None]
mixed = sa * t + sb * torch.sqrt((1 - t * t).clamp_min(0))
y = torch.istft(mixed * response.transpose(1, 2), fft, hop, window=window, center=True, length=S + fft // 2)
return y[:, :S]
def forward(self, hidden, f0, lengths, noise=None, phase=None, voiced=None):
c = self.config
nat = getattr(self, "naturalness", None) # Hindi port, inference only (see natural_pulse_train)
frames = hidden.shape[1]
samples = frames * c.hop
with torch.autocast(device_type=hidden.device.type, enabled=False):
hidden = hidden.float()
audible = length_mask(lengths * c.hop, samples)
last = (lengths - 1).clamp_min(0)[:, None]
# Harmonic branch: envelope interpolated between frames, one gain per sub-frame.
h = self.harmonic(hidden)
split = c.cep_causal + c.cep_anti
sub = self._subframes(frames, lengths, self.harmonic_steps)
here, following = sub // self.harmonic_steps, torch.minimum((sub + 1) // self.harmonic_steps, last)
causal = 0.5 * (self._take(h[..., :c.cep_causal], here) + self._take(h[..., :c.cep_causal], following)) * self.causal_lifter
anti = 0.5 * (self._take(h[..., c.cep_causal:split], here) + self._take(h[..., c.cep_causal:split], following)) * self.anti_lifter
gains = h[..., split:].flatten(1)
if getattr(self, "smooth_gains", False):
# Hindi port, inference only: the GAN-trained model flickers the per-sub-frame harmonic gain up and down
# (~94 Hz at synth_hop 128, up to ~20 dB) at transitions, heard as a gargle. [1, 2, 1]/4 removes the
# alternation and keeps changes slower than one frame.
gains = F.conv1d(F.pad(gains[:, None], (1, 1), mode="replicate"), gains.new_tensor([[[0.25, 0.5, 0.25]]]))[:, 0]
causal = torch.cat((causal[..., :1] + gains.gather(1, sub)[..., None], causal[..., 1:]), dim=-1)
if getattr(self, "adaptive_lifter", 0.0):
# Hindi port, inference only: above sr/80 = 300 Hz the 80 causal coefficients can describe the harmonic
# comb itself (quefrency 1/F0). Taper the coefficients from adaptive_lifter * sr/F0 to sr/F0 so the
# filter stays a smooth vocal-tract envelope at high pitch (no effect below ~300 Hz).
f0_sub = f0.float().gather(1, torch.minimum(here, last)).clamp_min(1.0) # [B, S]
k = torch.arange(c.cep_causal, device=causal.device, dtype=torch.float32)
period = c.sample_rate / f0_sub[..., None] # in coefficients
start = self.adaptive_lifter * period
taper = 0.5 * (1 + torch.cos(math.pi * ((k - start) / (period - start).clamp_min(1.0)).clamp(0, 1)))
taper[..., 0] = 1.0
causal = causal * taper
harmonic_response = cepstrum_response(causal, anti, c.synth_fft)
# Noise branch: one envelope per frame, four gains per frame for sharp consonants.
n = self.noise(hidden)
sub = self._subframes(frames, lengths, self.noise_steps)
shape = self._take(n[..., :c.noise_cep], sub // self.noise_steps) * self.noise_lifter
if getattr(self, "smooth_noise_env", False):
# Hindi port, inference only: the noise envelope is held for a whole frame and jumps at every frame
# boundary, an amplitude flutter at the frame rate (24000/256 = 93.75 Hz) heard as a buzz on earbuds.
# Interpolate it linearly between frame centres instead.
pos = sub.float() / self.noise_steps
lo = pos.floor().long()
hi = torch.minimum(lo + 1, last)
w = (pos - lo.float())[..., None]
shape = ((1 - w) * self._take(n[..., :c.noise_cep], lo) + w * self._take(n[..., :c.noise_cep], hi)) * self.noise_lifter
noise_gains = n[..., c.noise_cep:].flatten(1)
if getattr(self, "smooth_noise_gains", False):
# same flicker in the noise gains (a per-frame sawtooth); [1, 2, 2, 2, 1]/8 removes period-2 and period-4 parts
noise_gains = F.conv1d(F.pad(noise_gains[:, None], (2, 2), mode="replicate"),
noise_gains.new_tensor([[[1, 2, 2, 2, 1]]]) / 8)[:, 0]
shape = torch.cat((shape[..., :1] + noise_gains.gather(1, sub)[..., None], shape[..., 1:]), dim=-1)
noise_response = cepstrum_response(shape, None, c.noise_fft)
# Hold the last real pitch value through padding: the pitch glide inside an
# utterance's final frame must not depend on what the batch was padded with.
f0 = f0.float().gather(1, torch.minimum(torch.arange(frames, device=f0.device)[None], last))
if nat:
gen = torch.Generator().manual_seed(int(nat.get("seed", 0)) + 104729)
pulses, theta = natural_pulse_train(f0, c.sample_rate, c.hop, c.harmonic_limit, nat.get("jitter", 0.0),
nat.get("shimmer", 0.0), gen)
pulses = pulses * audible
else:
pulses = pulse_train(f0, c.sample_rate, c.hop, c.harmonic_limit, phase) * audible
if getattr(self, "mixed_excitation", False) and voiced is not None:
# Hindi port, inference only. The oscillator runs through unvoiced sounds too, and the model learned to
# build much of /s/, /k/, /t/ and breaths from PULSES (measured: the pulse branch is ~8 dB louder than
# the noise branch in unvoiced frames) -> an electronic buzz under the consonants (the "buzzing
# artifact" of DDSP vocoders, Wu et al. 2022). Where a frame is unvoiced, excite the same filter with
# white noise of the same power instead (mixed excitation, as MELP does); crossfade over ~1 frame.
vm = F.avg_pool1d(F.pad(voiced.float()[:, None], (1, 1), mode="replicate"), 3, 1)[:, 0]
vm = torch.cat((vm, vm[:, -1:]), dim=1)
position = torch.arange(samples, device=vm.device, dtype=torch.float32) / c.hop
low = position.floor().long()
weight = position - low
vs = (vm[:, low] * (1 - weight) + vm[:, low + 1] * weight).to(pulses.device)
alt = torch.randn(hidden.shape[0], samples, generator=torch.Generator().manual_seed(977)).to(pulses.device)
pulses = (pulses * vs + alt * torch.sqrt((1 - vs * vs).clamp_min(0))) * audible
if noise is None:
noise = torch.randn(hidden.shape[0], samples, device=hidden.device)
if nat and nat.get("breath_pulse", 0.0) > 0 and voiced is not None:
# breath noise puffs with each glottal cycle (strongest in the open phase, a quarter cycle before the
# closure pulse at theta = 0), only where the frame is voiced; mean power is unchanged.
d, kappa = float(nat["breath_pulse"]), 2.0
v = F.avg_pool1d(F.pad(voiced.float()[:, None], (2, 2), mode="replicate"), 5, 1)[:, 0]
v = v.repeat_interleave(c.hop, dim=1)[:, :samples].to(noise.device)
g = torch.exp(kappa * (torch.cos(theta - (-math.pi / 2)) - 1)).float().to(noise.device)
mean_g = math.exp(-kappa) * float(torch.special.i0(torch.tensor(kappa)))
noise = noise * torch.sqrt((1 - d * v) + d * v * g / mean_g)
if nat and nat.get("mvf_hz", 0.0) > 0:
# voice turns into breath at high frequencies: above the maximum voiced frequency the harmonic branch is
# excited by noise instead of pulses (same filter, same power density), cosine crossover
lo, width = float(nat["mvf_hz"]), float(nat.get("mvf_width_hz", 2500.0))
freqs = torch.arange(c.synth_fft // 2 + 1, dtype=torch.float32) * c.sample_rate / c.synth_fft
taper = 0.5 * (1 + torch.cos(math.pi * ((freqs - lo) / width).clamp(0, 1)))
breath = torch.randn(hidden.shape[0], samples, generator=torch.Generator().manual_seed(int(nat.get("seed", 0)) + 7))
harmonic = self._shape_mix(pulses, breath.to(pulses.device) * audible, taper.to(pulses.device), harmonic_response,
c.synth_fft, c.synth_hop, self.synth_window)
else:
harmonic = self._shape(pulses, harmonic_response, c.synth_fft, c.synth_hop, self.synth_window)
wave = harmonic + self._shape(noise * audible, noise_response, c.noise_fft, c.noise_hop, self.noise_window)
return wave * audible
class TinyVoca(nn.Module):
def __init__(self, config=Config()):
super().__init__()
self.config = c = config
self.tokens = nn.Embedding(c.vocab_size, c.channels, padding_idx=0)
self.stress = nn.Embedding(3, c.channels)
self.voices = nn.Embedding(len(c.speakers), c.channels)
self.encoder = Stack(c, c.encoder_layers)
# Outputs per token: log(1 + frames) and, for pause-capable tokens, a
# "has frames at all" logit. A pause is absent more often than present, so
# one regression would average both cases into a pause that is too short.
self.duration = Predictor(c, 2)
self.frame_voices = nn.Embedding(len(c.speakers), c.channels)
self.frame_position = nn.Linear(3, c.channels)
self.prosody_stack = Stack(c, c.prosody_layers)
# Outputs: continuous log(F0/reference), voiced logit, normalized log RMS energy.
self.prosody_head = nn.Linear(c.channels, 3)
self.prosody_projection = nn.Linear(3, c.channels)
self.acoustic_stack = Stack(c, c.acoustic_layers)
self.vocoder = SourceFilter(c)
phones = torch.zeros(c.vocab_size, dtype=torch.bool)
phones[sorted(PHONE_IDS)] = True
self.register_buffer("phone_tokens", phones, persistent=False)
vowels = torch.zeros(c.vocab_size, dtype=torch.bool)
vowels[[SYMBOL_TO_ID[v] for v in VOWELS]] = True
self.register_buffer("vowel_tokens", vowels, persistent=False)
def encode(self, tokens, stress, lengths, speakers):
if tokens.ndim != 2 or stress.shape != tokens.shape or lengths.shape != (tokens.shape[0],) or speakers.shape != lengths.shape:
raise ValueError("expected tokens[B,N], stress[B,N], lengths[B], speakers[B]")
if tokens.shape[1] > self.config.max_tokens or bool((lengths < 1).any()) or bool((lengths > tokens.shape[1]).any()):
raise ValueError("invalid token length")
mask = length_mask(lengths, tokens.shape[1])
x = (self.tokens(tokens) + self.stress(stress) + self.voices(speakers)[:, None]) * mask[..., None]
return self.encoder(x, mask), mask
def expand(self, hidden, durations, token_mask, speakers):
"""Repeat token states by integer durations (zero is allowed for pause
tokens) and add each frame's position inside its token."""
if durations.shape != token_mask.shape or bool((durations < 0).any()) or bool((durations[~token_mask] != 0).any()):
raise ValueError("invalid durations")
lengths = durations.sum(1)
if bool((lengths < 1).any()):
raise ValueError("utterance has no frames")
if bool((lengths > self.config.max_frames).any()):
raise UtteranceTooLong("utterance exceeds max_frames; split the input")
size = int(lengths.max())
ends = durations.cumsum(1)
frame = torch.arange(size, device=hidden.device)[None].expand(hidden.shape[0], size).contiguous()
index = torch.searchsorted(ends, frame, right=True).clamp(max=durations.shape[1] - 1)
owned = durations.gather(1, index).clamp_min(1)
offset = frame - (ends.gather(1, index) - durations.gather(1, index))
relative = (offset.to(hidden.dtype) + 0.5) / owned
features = torch.stack((relative, 1 - relative, torch.log1p(owned.to(hidden.dtype)) / 4), dim=-1)
mask = frame < lengths[:, None]
expanded = hidden.gather(1, index[..., None].expand(-1, -1, hidden.shape[-1]))
expanded = expanded + self.frame_position(features) + self.frame_voices(speakers)[:, None]
return expanded * mask[..., None], mask, lengths
def predict_prosody(self, frames, frame_mask):
hidden = self.prosody_stack(frames, frame_mask)
return hidden, self.prosody_head(hidden) * frame_mask[..., None]
@staticmethod
def hard_prosody(prediction):
"""What the decoder is conditioned on when it is not teacher-forced."""
return torch.stack((prediction[..., 0].clamp(-2.0, 1.5), (prediction[..., 1] > 0).to(prediction.dtype),
prediction[..., 2].clamp(-4.5, 3.0)), dim=-1)
def decode(self, prosody_hidden, condition, frame_mask, noise=None):
h = prosody_hidden + self.prosody_projection(condition)
if getattr(self, "frame_noise", None) is not None:
# Hindi port (anti-blur fine-tune): a random vector per frame lets the acoustic stack pick ONE natural
# version of the spectrum instead of the average of all of them (an average is what sounds blurred).
if noise is None:
t = getattr(self, "noise_temperature", 1.0)
noise = t * torch.randn(*h.shape[:2], self.frame_noise.in_features, device=h.device)
h = h + self.frame_noise(noise.to(h.dtype))
return self.acoustic_stack(h * frame_mask[..., None], frame_mask)
def add_frame_noise(self, dim=16):
"""Add the per-frame noise input (zero-initialised: the model's output is unchanged until it is trained)."""
self.frame_noise = nn.Linear(dim, self.config.channels)
nn.init.zeros_(self.frame_noise.weight)
nn.init.zeros_(self.frame_noise.bias)
return self
def frequency(self, log_f0):
c = self.config
return (c.pitch_reference_hz * torch.exp(log_f0.float())).clamp(c.f0_min, c.f0_max)
def predict_durations(self, hidden, mask, tokens, speed=1.0):
output = self.duration(hidden, mask).float()
frames = torch.round(torch.expm1(output[..., 0].clamp(0, math.log1p(400))) / speed).long().clamp_min(1)
# Every phoneme is audible for at least one frame. A pause-capable token
# (edge, punctuation, word boundary) lasts its predicted length when the
# presence logit says there is a pause, and vanishes otherwise.
present = self.phone_tokens[tokens] | (output[..., 1] > 0)
return torch.where(present, frames, torch.zeros_like(frames)) * mask
@staticmethod
def widen_pitch(prosody, frame_mask, pitch_range):
"""Scale the predicted pitch movement around each utterance's voiced mean.
Regression predicts a flatter contour than the recordings; values above 1
restore range (1.0 leaves the prediction untouched)."""
if pitch_range == 1.0:
return prosody
voiced = ((prosody[..., 1] > 0) & frame_mask).to(prosody.dtype)
mean = (prosody[..., 0] * voiced).sum(1, keepdim=True) / voiced.sum(1, keepdim=True).clamp_min(1)
pitch = mean + (prosody[..., 0] - mean) * pitch_range
return torch.cat((pitch[..., None], prosody[..., 1:]), dim=-1) * frame_mask[..., None]
@staticmethod
def pitch_wobble(shape, semitones, seed, device):
"""Small random pitch movement for the oscillator only (log-F0 units).
Recorded pitch has cycle-to-cycle irregularity that a predicted contour
lacks; two thirds of the wobble is slow drift, one third frame-level."""
generator = torch.Generator().manual_seed(seed + 7919)
fast = torch.randn(shape, generator=generator)
slow = F.avg_pool1d(torch.randn(shape[0], 1, shape[1] + 16, generator=generator), 17, stride=1)[:, 0]
slow = slow / slow.std().clamp_min(1e-6)
return ((2 * slow + fast) / 5 ** 0.5 * semitones * math.log(2) / 12).to(device)
@torch.inference_mode()
def synthesize(self, tokens, stress, lengths, speakers, speed=1.0, pitch_shift=0.0, seed=0, pitch_range=1.0, pitch_jitter=0.0,
voiced_vowels=True, pitch_smooth=0, octave_fix=0.0):
"""Free-running synthesis: predicted durations, pitch, voicing and energy.
Hindi port: ``pitch_smooth`` = width (frames, odd) of a running median over the predicted log F0. The YIN pitch
targets carry ~10 jumps of > 4 semitones per 1,000 voiced frames (tracker errors), the model learned them, and
they are heard as hiccups; a 5-frame (53 ms) median removes 10 ms blips and keeps the intonation."""
if not math.isfinite(speed) or not 0.25 <= speed <= 4 or not math.isfinite(pitch_shift) or not 0 <= pitch_range <= 3:
raise ValueError("speed must be between 0.25 and 4, pitch_shift finite and pitch_range between 0 and 3")
hidden, mask = self.encode(tokens, stress, lengths, speakers)
durations = self.predict_durations(hidden, mask, tokens, speed)
frames, frame_mask, frame_lengths = self.expand(hidden, durations, mask, speakers)
prosody_hidden, prosody = self.predict_prosody(frames, frame_mask)
if voiced_vowels:
# The pitch tracker missed creaky vowels in the recordings, and the
# voicing predictor learned to drop whole vowels (heard as a whispered
# word). A vowel is voiced by definition, so its frames are kept voiced.
ends = durations.cumsum(1)
frame = torch.arange(frames.shape[1], device=tokens.device)[None].expand(len(lengths), -1).contiguous()
owner = torch.searchsorted(ends, frame, right=True).clamp(max=tokens.shape[1] - 1)
vowel = self.vowel_tokens[tokens].gather(1, owner) & frame_mask
prosody = torch.cat((prosody[..., :1], torch.where(vowel, prosody[..., 1].clamp_min(2.0), prosody[..., 1])[..., None],
prosody[..., 2:]), dim=-1)
if octave_fix:
# YIN octave errors in the training targets taught the model short excursions to ~2x the pitch (heard as
# hiccups). Pitch above the utterance's voiced median is soft-limited: median + T*tanh(excess/T), with
# T = log(octave_fix), so a rise can approach but never pass octave_fix x the median, with no new edges.
voiced = (prosody[..., 1] > 0) & frame_mask
logf0 = prosody[..., 0]
masked = torch.where(voiced, logf0, torch.full_like(logf0, float("nan")))
median = masked.nanmedian(1, keepdim=True).values.nan_to_num(0.0)
limit = math.log(octave_fix)
excess = logf0 - median
limited = torch.where(excess > 0, median + limit * torch.tanh(excess / limit), logf0)
prosody = torch.cat((limited[..., None], prosody[..., 1:]), dim=-1)
if pitch_smooth and pitch_smooth > 1:
w = int(pitch_smooth) | 1
padded = F.pad(prosody[..., 0][:, None], (w // 2, w // 2), mode="replicate")[:, 0]
smoothed = padded.unfold(1, w, 1).median(-1).values
prosody = torch.cat((smoothed[..., None], prosody[..., 1:]), dim=-1) * frame_mask[..., None]
prosody = self.widen_pitch(prosody, frame_mask, pitch_range)
acoustic = self.decode(prosody_hidden, self.hard_prosody(prosody), frame_mask)
# One noise stream per row, drawn at the row's own length, so batching or
# padding never changes a row's audio.
noise = nn.utils.rnn.pad_sequence([torch.randn(int(n) * self.config.hop, generator=torch.Generator().manual_seed(seed + row))
for row, n in enumerate(frame_lengths)], batch_first=True)
pitch = prosody[..., 0] + pitch_shift
if pitch_jitter > 0:
pitch = pitch + self.pitch_wobble(pitch.shape, pitch_jitter, seed, pitch.device)
wave = self.vocoder(acoustic, self.frequency(pitch), frame_lengths, noise=noise.to(hidden.device),
voiced=(prosody[..., 1] > 0) & frame_mask)
return wave, frame_lengths * self.config.hop, {"durations": durations, "prosody": prosody, "frame_mask": frame_mask}