File size: 33,055 Bytes
ab1b068 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 | """tiny-voca v2: feed-forward transformer with a pitch-driven source-filter head.
All weights start random. Tensor conventions: hidden [B,T,C], frame features
[B,T], waveform [B,S]. Acoustic frame i is centred on sample ``i * hop``.
phoneme + stress + voice -> transformer encoder -> duration predictor
-> repeat tokens by duration (supervised durations while training)
-> frame transformer A -> pitch / voicing / energy
-> frame transformer B -> filter parameters
-> harmonic pulse train (from pitch) and noise, each shaped by its
time-varying filter in the STFT domain -> 24 kHz waveform
The waveform head has no neural upsampling network. The oscillator supplies the
periodicity that a tiny decoder cannot render reliably on its own; the network
only has to predict slowly varying spectral envelopes (complex cepstra), which
is what keeps the voice from sounding buzzy or unstable at this size.
"""
import math
import torch
from torch import nn
from torch.nn import functional as F
from .config import Config
from .text import PHONE_IDS, SYMBOL_TO_ID, VOWELS
class UtteranceTooLong(ValueError):
"""The utterance needs more frames than ``max_frames``: split the input."""
def length_mask(lengths, size=None):
size = int(lengths.max()) if size is None else size
return torch.arange(size, device=lengths.device)[None, :] < lengths[:, None]
def rotary_table(size, width, device):
inverse = 1.0 / (10000 ** (torch.arange(0, width, 2, device=device, dtype=torch.float32) / width))
angle = torch.arange(size, device=device, dtype=torch.float32)[:, None] * inverse[None]
return angle.cos(), angle.sin()
def rotate(x, table):
cos, sin = table[0].to(x.dtype), table[1].to(x.dtype)
a, b = x[..., 0::2], x[..., 1::2]
return torch.stack((a * cos - b * sin, a * sin + b * cos), dim=-1).flatten(-2)
class Attention(nn.Module):
"""Self-attention with rotary (relative) positions: no absolute position
table, so utterances longer than those seen in training stay in range."""
def __init__(self, channels, heads, dropout):
super().__init__()
self.heads, self.dropout = heads, dropout
self.qkv = nn.Linear(channels, 3 * channels)
self.out = nn.Linear(channels, channels)
def forward(self, x, mask, table):
b, t, c = x.shape
q, k, v = self.qkv(x).view(b, t, 3, self.heads, c // self.heads).permute(2, 0, 3, 1, 4)
y = F.scaled_dot_product_attention(rotate(q, table), rotate(k, table), v, attn_mask=mask[:, None, None, :],
dropout_p=self.dropout if self.training else 0.0)
return self.out(y.transpose(1, 2).reshape(b, t, c))
class Block(nn.Module):
def __init__(self, c):
super().__init__()
self.norm1 = nn.LayerNorm(c.channels)
self.attention = Attention(c.channels, c.heads, c.dropout)
self.norm2 = nn.LayerNorm(c.channels)
self.depthwise = nn.Conv1d(c.channels, c.channels, c.kernel, padding=c.kernel // 2, groups=c.channels)
self.expand = nn.Linear(c.channels, c.feedforward)
self.project = nn.Linear(c.feedforward, c.channels)
self.dropout = nn.Dropout(c.dropout)
def forward(self, x, mask, table):
keep = mask[..., None]
x = (x + self.dropout(self.attention(self.norm1(x), mask, table))) * keep
# Mask before the convolution so padded positions cannot leak back in.
y = self.depthwise((self.norm2(x) * keep).transpose(1, 2)).transpose(1, 2)
y = self.project(self.dropout(F.gelu(self.expand(y))))
return (x + self.dropout(y)) * keep
class Stack(nn.Module):
def __init__(self, c, layers):
super().__init__()
self.blocks = nn.ModuleList(Block(c) for _ in range(layers))
self.norm = nn.LayerNorm(c.channels)
self.width = c.channels // c.heads
def forward(self, x, mask):
table = rotary_table(x.shape[1], self.width, x.device)
for block in self.blocks:
x = block(x, mask, table)
return self.norm(x) * mask[..., None]
class Predictor(nn.Module):
def __init__(self, c, outputs):
super().__init__()
p = c.predictor_channels
self.conv1 = nn.Conv1d(c.channels, p, 3, padding=1)
self.conv2 = nn.Conv1d(p, p, 3, padding=1)
self.norm1, self.norm2 = nn.LayerNorm(p), nn.LayerNorm(p)
self.drop = nn.Dropout(c.dropout)
self.out = nn.Linear(p, outputs)
def forward(self, x, mask):
x = self.conv1((x * mask[..., None]).transpose(1, 2)).transpose(1, 2)
x = self.drop(self.norm1(F.gelu(x))) * mask[..., None]
x = self.conv2(x.transpose(1, 2)).transpose(1, 2)
x = self.drop(self.norm2(F.gelu(x)))
return self.out(x) * mask[..., None]
def pulse_train(f0, sample_rate, hop, limit=0.47, phase=None):
"""Band-limited pulse train [B, T*hop] with unit power for f0 [B, T] in Hz.
Frame values sit at samples i*hop and are interpolated in the log domain.
The sum of all harmonics below ``limit * sample_rate`` has the closed form
sin((K+1/2)x) / (2 sin(x/2)) - 1/2; the top harmonic fades in and out so the
signal stays continuous while the pitch glides. Float64 keeps the phase exact.
"""
frames = f0.shape[1]
knots = torch.log(f0.double().clamp_min(1.0))
knots = torch.cat((knots, knots[:, -1:]), dim=1)
position = torch.arange(frames * hop, device=f0.device, dtype=torch.float64) / hop
low = position.floor().long()
weight = position - low
frequency = torch.exp(knots[:, low] * (1 - weight) + knots[:, low + 1] * weight)
theta = 2 * math.pi * torch.cumsum(frequency / sample_rate, dim=1)
if phase is not None:
theta = theta + phase.double()[:, None]
theta = torch.remainder(theta, 2 * math.pi)
harmonics = (limit * sample_rate / frequency).clamp_min(1.0)
whole = torch.floor(harmonics)
denominator = 2 * torch.sin(theta / 2)
tiny = denominator.abs() < 1e-7
core = torch.where(tiny, whole, torch.sin((whole + 0.5) * theta) / torch.where(tiny, torch.ones_like(denominator), denominator) - 0.5)
wave = core + (harmonics - whole) * torch.cos((whole + 1) * theta)
return (wave * torch.sqrt(2.0 / harmonics)).float()
def natural_pulse_train(f0, sample_rate, hop, limit=0.47, jitter=0.0, shimmer=0.0, generator=None):
"""Hindi port, inference only: pulse_train with the cycle-to-cycle irregularity of a real voice.
jitter: each glottal cycle gets its own frequency factor (1 + jitter * N(0,1)); local jitter (mean |dT|/T) is
~1.13 x jitter, so 0.005 ~ 0.6% (normal voices: 0.3-1%).
shimmer: each pulse gets its own amplitude factor (1 + shimmer * N(0,1)), interpolated from peak to peak so there
is no step inside a cycle; 0.035 ~ 4% local shimmer (normal voices: 2-6%).
Returns (wave [B, S] float32, unwrapped phase theta [B, S] float64 for pitch-synchronous effects)."""
frames = f0.shape[1]
knots = torch.log(f0.double().clamp_min(1.0))
knots = torch.cat((knots, knots[:, -1:]), dim=1)
position = torch.arange(frames * hop, device=f0.device, dtype=torch.float64) / hop
low = position.floor().long()
weight = position - low
frequency = torch.exp(knots[:, low] * (1 - weight) + knots[:, low + 1] * weight)
if jitter > 0:
cycle = torch.cumsum(frequency / sample_rate, dim=1).floor().long()
factors = 1 + jitter * torch.randn(f0.shape[0], int(cycle.max()) + 2, generator=generator, dtype=torch.float64)
frequency = frequency * factors.to(f0.device).gather(1, cycle)
unwrapped = 2 * math.pi * torch.cumsum(frequency / sample_rate, dim=1)
theta = torch.remainder(unwrapped, 2 * math.pi)
harmonics = (limit * sample_rate / frequency).clamp_min(1.0)
whole = torch.floor(harmonics)
denominator = 2 * torch.sin(theta / 2)
tiny = denominator.abs() < 1e-7
core = torch.where(tiny, whole, torch.sin((whole + 0.5) * theta) / torch.where(tiny, torch.ones_like(denominator), denominator) - 0.5)
wave = (core + (harmonics - whole) * torch.cos((whole + 1) * theta)) * torch.sqrt(2.0 / harmonics)
if shimmer > 0:
cycles = unwrapped / (2 * math.pi)
index = cycles.floor().long()
frac = cycles - index
amp = (1 + shimmer * torch.randn(f0.shape[0], int(index.max()) + 2, generator=generator, dtype=torch.float64)).clamp_min(0.2)
amp = amp.to(f0.device)
wave = wave * (amp.gather(1, index) * (1 - frac) + amp.gather(1, index + 1) * frac)
return wave.float(), unwrapped
def filtered(source, response, fft, hop, window):
"""Time-varying filtering: response [B, frames, bins] multiplies the STFT."""
spectrum = torch.stft(source, fft, hop, window=window, center=True, pad_mode="constant", return_complex=True)
if spectrum.shape[-1] != response.shape[1]:
raise ValueError("filter frames do not match the source")
return torch.istft(spectrum * response.transpose(1, 2), fft, hop, window=window, center=True, length=source.shape[-1])
def cepstrum_response(causal, anticausal, fft):
"""Complex frequency response exp(FFT(cepstrum)). Causal coefficients give a
minimum-phase filter; anticausal ones add a maximum-phase part (glottal pulse)."""
buffer = causal.new_zeros(*causal.shape[:-1], fft)
buffer[..., :causal.shape[-1]] = causal
if anticausal is not None:
buffer[..., fft - anticausal.shape[-1]:] = anticausal.flip(-1)
spectrum = torch.fft.rfft(buffer, dim=-1)
return torch.polar(torch.exp(spectrum.real.clamp(-14, 6)), spectrum.imag)
class SourceFilter(nn.Module):
"""Harmonic-plus-noise synthesis controlled per frame by two linear heads."""
def __init__(self, c):
super().__init__()
self.config = c
self.harmonic_steps, self.noise_steps = c.hop // c.synth_hop, c.hop // c.noise_hop
self.harmonic = nn.Linear(c.channels, c.cep_causal + c.cep_anti + self.harmonic_steps)
self.noise = nn.Linear(c.channels, c.noise_cep + self.noise_steps)
for head, level in ((self.harmonic, -1.5), (self.noise, -2.5)): # start quiet and spectrally flat
nn.init.zeros_(head.weight)
nn.init.zeros_(head.bias)
head.bias.data[0] = level
def lifter(size, first):
scale = 1.0 / (1.0 + torch.arange(first, first + size, dtype=torch.float32) / 4)
if first == 0:
scale[0] = 2.0 # coefficient 0 is the log gain and needs range
return scale
self.register_buffer("causal_lifter", lifter(c.cep_causal, 0), persistent=False)
self.register_buffer("anti_lifter", lifter(c.cep_anti, 1), persistent=False)
self.register_buffer("noise_lifter", lifter(c.noise_cep, 0), persistent=False)
self.register_buffer("synth_window", torch.hann_window(c.synth_fft), persistent=False)
self.register_buffer("noise_window", torch.hann_window(c.noise_fft), persistent=False)
@staticmethod
def _take(values, index):
return values.gather(1, index[..., None].expand(-1, -1, values.shape[-1]))
def _subframes(self, frames, lengths, steps):
"""Sub-frame j uses frame j // steps; indices past an utterance's end
repeat its last sub-frame so padding never changes real audio. Two extra
sub-frames cover the half window of zeros appended to the source."""
sub = torch.arange(steps * frames + 3, device=lengths.device)[None].expand(len(lengths), -1)
return torch.minimum(sub, (steps * lengths - 1).clamp_min(0)[:, None])
@staticmethod
def _shape(source, response, fft, hop, window):
# Half a window of trailing zeros: the frames around an utterance's end are
# then the same whether it stands alone or is padded inside a batch.
return filtered(F.pad(source, (0, fft // 2)), response, fft, hop, window)[:, :source.shape[-1]]
def _shape_mix(self, a, b, taper, response, fft, hop, window):
"""_shape for a source made of `a` below and `b` above a crossover: spectrum(a) * taper + spectrum(b) *
sqrt(1 - taper^2), then the time-varying filter. Equal power density in a and b keeps the envelope."""
S = a.shape[-1]
sa = torch.stft(F.pad(a, (0, fft // 2)), fft, hop, window=window, center=True, pad_mode="constant", return_complex=True)
sb = torch.stft(F.pad(b, (0, fft // 2)), fft, hop, window=window, center=True, pad_mode="constant", return_complex=True)
t = taper[None, :, None]
mixed = sa * t + sb * torch.sqrt((1 - t * t).clamp_min(0))
y = torch.istft(mixed * response.transpose(1, 2), fft, hop, window=window, center=True, length=S + fft // 2)
return y[:, :S]
def forward(self, hidden, f0, lengths, noise=None, phase=None, voiced=None):
c = self.config
nat = getattr(self, "naturalness", None) # Hindi port, inference only (see natural_pulse_train)
frames = hidden.shape[1]
samples = frames * c.hop
with torch.autocast(device_type=hidden.device.type, enabled=False):
hidden = hidden.float()
audible = length_mask(lengths * c.hop, samples)
last = (lengths - 1).clamp_min(0)[:, None]
# Harmonic branch: envelope interpolated between frames, one gain per sub-frame.
h = self.harmonic(hidden)
split = c.cep_causal + c.cep_anti
sub = self._subframes(frames, lengths, self.harmonic_steps)
here, following = sub // self.harmonic_steps, torch.minimum((sub + 1) // self.harmonic_steps, last)
causal = 0.5 * (self._take(h[..., :c.cep_causal], here) + self._take(h[..., :c.cep_causal], following)) * self.causal_lifter
anti = 0.5 * (self._take(h[..., c.cep_causal:split], here) + self._take(h[..., c.cep_causal:split], following)) * self.anti_lifter
gains = h[..., split:].flatten(1)
if getattr(self, "smooth_gains", False):
# Hindi port, inference only: the GAN-trained model flickers the per-sub-frame harmonic gain up and down
# (~94 Hz at synth_hop 128, up to ~20 dB) at transitions, heard as a gargle. [1, 2, 1]/4 removes the
# alternation and keeps changes slower than one frame.
gains = F.conv1d(F.pad(gains[:, None], (1, 1), mode="replicate"), gains.new_tensor([[[0.25, 0.5, 0.25]]]))[:, 0]
causal = torch.cat((causal[..., :1] + gains.gather(1, sub)[..., None], causal[..., 1:]), dim=-1)
if getattr(self, "adaptive_lifter", 0.0):
# Hindi port, inference only: above sr/80 = 300 Hz the 80 causal coefficients can describe the harmonic
# comb itself (quefrency 1/F0). Taper the coefficients from adaptive_lifter * sr/F0 to sr/F0 so the
# filter stays a smooth vocal-tract envelope at high pitch (no effect below ~300 Hz).
f0_sub = f0.float().gather(1, torch.minimum(here, last)).clamp_min(1.0) # [B, S]
k = torch.arange(c.cep_causal, device=causal.device, dtype=torch.float32)
period = c.sample_rate / f0_sub[..., None] # in coefficients
start = self.adaptive_lifter * period
taper = 0.5 * (1 + torch.cos(math.pi * ((k - start) / (period - start).clamp_min(1.0)).clamp(0, 1)))
taper[..., 0] = 1.0
causal = causal * taper
harmonic_response = cepstrum_response(causal, anti, c.synth_fft)
# Noise branch: one envelope per frame, four gains per frame for sharp consonants.
n = self.noise(hidden)
sub = self._subframes(frames, lengths, self.noise_steps)
shape = self._take(n[..., :c.noise_cep], sub // self.noise_steps) * self.noise_lifter
if getattr(self, "smooth_noise_env", False):
# Hindi port, inference only: the noise envelope is held for a whole frame and jumps at every frame
# boundary, an amplitude flutter at the frame rate (24000/256 = 93.75 Hz) heard as a buzz on earbuds.
# Interpolate it linearly between frame centres instead.
pos = sub.float() / self.noise_steps
lo = pos.floor().long()
hi = torch.minimum(lo + 1, last)
w = (pos - lo.float())[..., None]
shape = ((1 - w) * self._take(n[..., :c.noise_cep], lo) + w * self._take(n[..., :c.noise_cep], hi)) * self.noise_lifter
noise_gains = n[..., c.noise_cep:].flatten(1)
if getattr(self, "smooth_noise_gains", False):
# same flicker in the noise gains (a per-frame sawtooth); [1, 2, 2, 2, 1]/8 removes period-2 and period-4 parts
noise_gains = F.conv1d(F.pad(noise_gains[:, None], (2, 2), mode="replicate"),
noise_gains.new_tensor([[[1, 2, 2, 2, 1]]]) / 8)[:, 0]
shape = torch.cat((shape[..., :1] + noise_gains.gather(1, sub)[..., None], shape[..., 1:]), dim=-1)
noise_response = cepstrum_response(shape, None, c.noise_fft)
# Hold the last real pitch value through padding: the pitch glide inside an
# utterance's final frame must not depend on what the batch was padded with.
f0 = f0.float().gather(1, torch.minimum(torch.arange(frames, device=f0.device)[None], last))
if nat:
gen = torch.Generator().manual_seed(int(nat.get("seed", 0)) + 104729)
pulses, theta = natural_pulse_train(f0, c.sample_rate, c.hop, c.harmonic_limit, nat.get("jitter", 0.0),
nat.get("shimmer", 0.0), gen)
pulses = pulses * audible
else:
pulses = pulse_train(f0, c.sample_rate, c.hop, c.harmonic_limit, phase) * audible
if getattr(self, "mixed_excitation", False) and voiced is not None:
# Hindi port, inference only. The oscillator runs through unvoiced sounds too, and the model learned to
# build much of /s/, /k/, /t/ and breaths from PULSES (measured: the pulse branch is ~8 dB louder than
# the noise branch in unvoiced frames) -> an electronic buzz under the consonants (the "buzzing
# artifact" of DDSP vocoders, Wu et al. 2022). Where a frame is unvoiced, excite the same filter with
# white noise of the same power instead (mixed excitation, as MELP does); crossfade over ~1 frame.
vm = F.avg_pool1d(F.pad(voiced.float()[:, None], (1, 1), mode="replicate"), 3, 1)[:, 0]
vm = torch.cat((vm, vm[:, -1:]), dim=1)
position = torch.arange(samples, device=vm.device, dtype=torch.float32) / c.hop
low = position.floor().long()
weight = position - low
vs = (vm[:, low] * (1 - weight) + vm[:, low + 1] * weight).to(pulses.device)
alt = torch.randn(hidden.shape[0], samples, generator=torch.Generator().manual_seed(977)).to(pulses.device)
pulses = (pulses * vs + alt * torch.sqrt((1 - vs * vs).clamp_min(0))) * audible
if noise is None:
noise = torch.randn(hidden.shape[0], samples, device=hidden.device)
if nat and nat.get("breath_pulse", 0.0) > 0 and voiced is not None:
# breath noise puffs with each glottal cycle (strongest in the open phase, a quarter cycle before the
# closure pulse at theta = 0), only where the frame is voiced; mean power is unchanged.
d, kappa = float(nat["breath_pulse"]), 2.0
v = F.avg_pool1d(F.pad(voiced.float()[:, None], (2, 2), mode="replicate"), 5, 1)[:, 0]
v = v.repeat_interleave(c.hop, dim=1)[:, :samples].to(noise.device)
g = torch.exp(kappa * (torch.cos(theta - (-math.pi / 2)) - 1)).float().to(noise.device)
mean_g = math.exp(-kappa) * float(torch.special.i0(torch.tensor(kappa)))
noise = noise * torch.sqrt((1 - d * v) + d * v * g / mean_g)
if nat and nat.get("mvf_hz", 0.0) > 0:
# voice turns into breath at high frequencies: above the maximum voiced frequency the harmonic branch is
# excited by noise instead of pulses (same filter, same power density), cosine crossover
lo, width = float(nat["mvf_hz"]), float(nat.get("mvf_width_hz", 2500.0))
freqs = torch.arange(c.synth_fft // 2 + 1, dtype=torch.float32) * c.sample_rate / c.synth_fft
taper = 0.5 * (1 + torch.cos(math.pi * ((freqs - lo) / width).clamp(0, 1)))
breath = torch.randn(hidden.shape[0], samples, generator=torch.Generator().manual_seed(int(nat.get("seed", 0)) + 7))
harmonic = self._shape_mix(pulses, breath.to(pulses.device) * audible, taper.to(pulses.device), harmonic_response,
c.synth_fft, c.synth_hop, self.synth_window)
else:
harmonic = self._shape(pulses, harmonic_response, c.synth_fft, c.synth_hop, self.synth_window)
wave = harmonic + self._shape(noise * audible, noise_response, c.noise_fft, c.noise_hop, self.noise_window)
return wave * audible
class TinyVoca(nn.Module):
def __init__(self, config=Config()):
super().__init__()
self.config = c = config
self.tokens = nn.Embedding(c.vocab_size, c.channels, padding_idx=0)
self.stress = nn.Embedding(3, c.channels)
self.voices = nn.Embedding(len(c.speakers), c.channels)
self.encoder = Stack(c, c.encoder_layers)
# Outputs per token: log(1 + frames) and, for pause-capable tokens, a
# "has frames at all" logit. A pause is absent more often than present, so
# one regression would average both cases into a pause that is too short.
self.duration = Predictor(c, 2)
self.frame_voices = nn.Embedding(len(c.speakers), c.channels)
self.frame_position = nn.Linear(3, c.channels)
self.prosody_stack = Stack(c, c.prosody_layers)
# Outputs: continuous log(F0/reference), voiced logit, normalized log RMS energy.
self.prosody_head = nn.Linear(c.channels, 3)
self.prosody_projection = nn.Linear(3, c.channels)
self.acoustic_stack = Stack(c, c.acoustic_layers)
self.vocoder = SourceFilter(c)
phones = torch.zeros(c.vocab_size, dtype=torch.bool)
phones[sorted(PHONE_IDS)] = True
self.register_buffer("phone_tokens", phones, persistent=False)
vowels = torch.zeros(c.vocab_size, dtype=torch.bool)
vowels[[SYMBOL_TO_ID[v] for v in VOWELS]] = True
self.register_buffer("vowel_tokens", vowels, persistent=False)
def encode(self, tokens, stress, lengths, speakers):
if tokens.ndim != 2 or stress.shape != tokens.shape or lengths.shape != (tokens.shape[0],) or speakers.shape != lengths.shape:
raise ValueError("expected tokens[B,N], stress[B,N], lengths[B], speakers[B]")
if tokens.shape[1] > self.config.max_tokens or bool((lengths < 1).any()) or bool((lengths > tokens.shape[1]).any()):
raise ValueError("invalid token length")
mask = length_mask(lengths, tokens.shape[1])
x = (self.tokens(tokens) + self.stress(stress) + self.voices(speakers)[:, None]) * mask[..., None]
return self.encoder(x, mask), mask
def expand(self, hidden, durations, token_mask, speakers):
"""Repeat token states by integer durations (zero is allowed for pause
tokens) and add each frame's position inside its token."""
if durations.shape != token_mask.shape or bool((durations < 0).any()) or bool((durations[~token_mask] != 0).any()):
raise ValueError("invalid durations")
lengths = durations.sum(1)
if bool((lengths < 1).any()):
raise ValueError("utterance has no frames")
if bool((lengths > self.config.max_frames).any()):
raise UtteranceTooLong("utterance exceeds max_frames; split the input")
size = int(lengths.max())
ends = durations.cumsum(1)
frame = torch.arange(size, device=hidden.device)[None].expand(hidden.shape[0], size).contiguous()
index = torch.searchsorted(ends, frame, right=True).clamp(max=durations.shape[1] - 1)
owned = durations.gather(1, index).clamp_min(1)
offset = frame - (ends.gather(1, index) - durations.gather(1, index))
relative = (offset.to(hidden.dtype) + 0.5) / owned
features = torch.stack((relative, 1 - relative, torch.log1p(owned.to(hidden.dtype)) / 4), dim=-1)
mask = frame < lengths[:, None]
expanded = hidden.gather(1, index[..., None].expand(-1, -1, hidden.shape[-1]))
expanded = expanded + self.frame_position(features) + self.frame_voices(speakers)[:, None]
return expanded * mask[..., None], mask, lengths
def predict_prosody(self, frames, frame_mask):
hidden = self.prosody_stack(frames, frame_mask)
return hidden, self.prosody_head(hidden) * frame_mask[..., None]
@staticmethod
def hard_prosody(prediction):
"""What the decoder is conditioned on when it is not teacher-forced."""
return torch.stack((prediction[..., 0].clamp(-2.0, 1.5), (prediction[..., 1] > 0).to(prediction.dtype),
prediction[..., 2].clamp(-4.5, 3.0)), dim=-1)
def decode(self, prosody_hidden, condition, frame_mask, noise=None):
h = prosody_hidden + self.prosody_projection(condition)
if getattr(self, "frame_noise", None) is not None:
# Hindi port (anti-blur fine-tune): a random vector per frame lets the acoustic stack pick ONE natural
# version of the spectrum instead of the average of all of them (an average is what sounds blurred).
if noise is None:
t = getattr(self, "noise_temperature", 1.0)
noise = t * torch.randn(*h.shape[:2], self.frame_noise.in_features, device=h.device)
h = h + self.frame_noise(noise.to(h.dtype))
return self.acoustic_stack(h * frame_mask[..., None], frame_mask)
def add_frame_noise(self, dim=16):
"""Add the per-frame noise input (zero-initialised: the model's output is unchanged until it is trained)."""
self.frame_noise = nn.Linear(dim, self.config.channels)
nn.init.zeros_(self.frame_noise.weight)
nn.init.zeros_(self.frame_noise.bias)
return self
def frequency(self, log_f0):
c = self.config
return (c.pitch_reference_hz * torch.exp(log_f0.float())).clamp(c.f0_min, c.f0_max)
def predict_durations(self, hidden, mask, tokens, speed=1.0):
output = self.duration(hidden, mask).float()
frames = torch.round(torch.expm1(output[..., 0].clamp(0, math.log1p(400))) / speed).long().clamp_min(1)
# Every phoneme is audible for at least one frame. A pause-capable token
# (edge, punctuation, word boundary) lasts its predicted length when the
# presence logit says there is a pause, and vanishes otherwise.
present = self.phone_tokens[tokens] | (output[..., 1] > 0)
return torch.where(present, frames, torch.zeros_like(frames)) * mask
@staticmethod
def widen_pitch(prosody, frame_mask, pitch_range):
"""Scale the predicted pitch movement around each utterance's voiced mean.
Regression predicts a flatter contour than the recordings; values above 1
restore range (1.0 leaves the prediction untouched)."""
if pitch_range == 1.0:
return prosody
voiced = ((prosody[..., 1] > 0) & frame_mask).to(prosody.dtype)
mean = (prosody[..., 0] * voiced).sum(1, keepdim=True) / voiced.sum(1, keepdim=True).clamp_min(1)
pitch = mean + (prosody[..., 0] - mean) * pitch_range
return torch.cat((pitch[..., None], prosody[..., 1:]), dim=-1) * frame_mask[..., None]
@staticmethod
def pitch_wobble(shape, semitones, seed, device):
"""Small random pitch movement for the oscillator only (log-F0 units).
Recorded pitch has cycle-to-cycle irregularity that a predicted contour
lacks; two thirds of the wobble is slow drift, one third frame-level."""
generator = torch.Generator().manual_seed(seed + 7919)
fast = torch.randn(shape, generator=generator)
slow = F.avg_pool1d(torch.randn(shape[0], 1, shape[1] + 16, generator=generator), 17, stride=1)[:, 0]
slow = slow / slow.std().clamp_min(1e-6)
return ((2 * slow + fast) / 5 ** 0.5 * semitones * math.log(2) / 12).to(device)
@torch.inference_mode()
def synthesize(self, tokens, stress, lengths, speakers, speed=1.0, pitch_shift=0.0, seed=0, pitch_range=1.0, pitch_jitter=0.0,
voiced_vowels=True, pitch_smooth=0, octave_fix=0.0):
"""Free-running synthesis: predicted durations, pitch, voicing and energy.
Hindi port: ``pitch_smooth`` = width (frames, odd) of a running median over the predicted log F0. The YIN pitch
targets carry ~10 jumps of > 4 semitones per 1,000 voiced frames (tracker errors), the model learned them, and
they are heard as hiccups; a 5-frame (53 ms) median removes 10 ms blips and keeps the intonation."""
if not math.isfinite(speed) or not 0.25 <= speed <= 4 or not math.isfinite(pitch_shift) or not 0 <= pitch_range <= 3:
raise ValueError("speed must be between 0.25 and 4, pitch_shift finite and pitch_range between 0 and 3")
hidden, mask = self.encode(tokens, stress, lengths, speakers)
durations = self.predict_durations(hidden, mask, tokens, speed)
frames, frame_mask, frame_lengths = self.expand(hidden, durations, mask, speakers)
prosody_hidden, prosody = self.predict_prosody(frames, frame_mask)
if voiced_vowels:
# The pitch tracker missed creaky vowels in the recordings, and the
# voicing predictor learned to drop whole vowels (heard as a whispered
# word). A vowel is voiced by definition, so its frames are kept voiced.
ends = durations.cumsum(1)
frame = torch.arange(frames.shape[1], device=tokens.device)[None].expand(len(lengths), -1).contiguous()
owner = torch.searchsorted(ends, frame, right=True).clamp(max=tokens.shape[1] - 1)
vowel = self.vowel_tokens[tokens].gather(1, owner) & frame_mask
prosody = torch.cat((prosody[..., :1], torch.where(vowel, prosody[..., 1].clamp_min(2.0), prosody[..., 1])[..., None],
prosody[..., 2:]), dim=-1)
if octave_fix:
# YIN octave errors in the training targets taught the model short excursions to ~2x the pitch (heard as
# hiccups). Pitch above the utterance's voiced median is soft-limited: median + T*tanh(excess/T), with
# T = log(octave_fix), so a rise can approach but never pass octave_fix x the median, with no new edges.
voiced = (prosody[..., 1] > 0) & frame_mask
logf0 = prosody[..., 0]
masked = torch.where(voiced, logf0, torch.full_like(logf0, float("nan")))
median = masked.nanmedian(1, keepdim=True).values.nan_to_num(0.0)
limit = math.log(octave_fix)
excess = logf0 - median
limited = torch.where(excess > 0, median + limit * torch.tanh(excess / limit), logf0)
prosody = torch.cat((limited[..., None], prosody[..., 1:]), dim=-1)
if pitch_smooth and pitch_smooth > 1:
w = int(pitch_smooth) | 1
padded = F.pad(prosody[..., 0][:, None], (w // 2, w // 2), mode="replicate")[:, 0]
smoothed = padded.unfold(1, w, 1).median(-1).values
prosody = torch.cat((smoothed[..., None], prosody[..., 1:]), dim=-1) * frame_mask[..., None]
prosody = self.widen_pitch(prosody, frame_mask, pitch_range)
acoustic = self.decode(prosody_hidden, self.hard_prosody(prosody), frame_mask)
# One noise stream per row, drawn at the row's own length, so batching or
# padding never changes a row's audio.
noise = nn.utils.rnn.pad_sequence([torch.randn(int(n) * self.config.hop, generator=torch.Generator().manual_seed(seed + row))
for row, n in enumerate(frame_lengths)], batch_first=True)
pitch = prosody[..., 0] + pitch_shift
if pitch_jitter > 0:
pitch = pitch + self.pitch_wobble(pitch.shape, pitch_jitter, seed, pitch.device)
wave = self.vocoder(acoustic, self.frequency(pitch), frame_lengths, noise=noise.to(hidden.device),
voiced=(prosody[..., 1] > 0) & frame_mask)
return wave, frame_lengths * self.config.hop, {"durations": durations, "prosody": prosody, "frame_mask": frame_mask}
|