File size: 33,055 Bytes
ab1b068
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
"""tiny-voca v2: feed-forward transformer with a pitch-driven source-filter head.

All weights start random. Tensor conventions: hidden [B,T,C], frame features
[B,T], waveform [B,S]. Acoustic frame i is centred on sample ``i * hop``.

    phoneme + stress + voice -> transformer encoder -> duration predictor
        -> repeat tokens by duration (supervised durations while training)
        -> frame transformer A -> pitch / voicing / energy
        -> frame transformer B -> filter parameters
        -> harmonic pulse train (from pitch) and noise, each shaped by its
           time-varying filter in the STFT domain -> 24 kHz waveform

The waveform head has no neural upsampling network. The oscillator supplies the
periodicity that a tiny decoder cannot render reliably on its own; the network
only has to predict slowly varying spectral envelopes (complex cepstra), which
is what keeps the voice from sounding buzzy or unstable at this size.
"""

import math
import torch
from torch import nn
from torch.nn import functional as F
from .config import Config
from .text import PHONE_IDS, SYMBOL_TO_ID, VOWELS


class UtteranceTooLong(ValueError):
    """The utterance needs more frames than ``max_frames``: split the input."""


def length_mask(lengths, size=None):
    size = int(lengths.max()) if size is None else size
    return torch.arange(size, device=lengths.device)[None, :] < lengths[:, None]


def rotary_table(size, width, device):
    inverse = 1.0 / (10000 ** (torch.arange(0, width, 2, device=device, dtype=torch.float32) / width))
    angle = torch.arange(size, device=device, dtype=torch.float32)[:, None] * inverse[None]
    return angle.cos(), angle.sin()


def rotate(x, table):
    cos, sin = table[0].to(x.dtype), table[1].to(x.dtype)
    a, b = x[..., 0::2], x[..., 1::2]
    return torch.stack((a * cos - b * sin, a * sin + b * cos), dim=-1).flatten(-2)


class Attention(nn.Module):
    """Self-attention with rotary (relative) positions: no absolute position
    table, so utterances longer than those seen in training stay in range."""

    def __init__(self, channels, heads, dropout):
        super().__init__()
        self.heads, self.dropout = heads, dropout
        self.qkv = nn.Linear(channels, 3 * channels)
        self.out = nn.Linear(channels, channels)

    def forward(self, x, mask, table):
        b, t, c = x.shape
        q, k, v = self.qkv(x).view(b, t, 3, self.heads, c // self.heads).permute(2, 0, 3, 1, 4)
        y = F.scaled_dot_product_attention(rotate(q, table), rotate(k, table), v, attn_mask=mask[:, None, None, :],
                                           dropout_p=self.dropout if self.training else 0.0)
        return self.out(y.transpose(1, 2).reshape(b, t, c))


class Block(nn.Module):
    def __init__(self, c):
        super().__init__()
        self.norm1 = nn.LayerNorm(c.channels)
        self.attention = Attention(c.channels, c.heads, c.dropout)
        self.norm2 = nn.LayerNorm(c.channels)
        self.depthwise = nn.Conv1d(c.channels, c.channels, c.kernel, padding=c.kernel // 2, groups=c.channels)
        self.expand = nn.Linear(c.channels, c.feedforward)
        self.project = nn.Linear(c.feedforward, c.channels)
        self.dropout = nn.Dropout(c.dropout)

    def forward(self, x, mask, table):
        keep = mask[..., None]
        x = (x + self.dropout(self.attention(self.norm1(x), mask, table))) * keep
        # Mask before the convolution so padded positions cannot leak back in.
        y = self.depthwise((self.norm2(x) * keep).transpose(1, 2)).transpose(1, 2)
        y = self.project(self.dropout(F.gelu(self.expand(y))))
        return (x + self.dropout(y)) * keep


class Stack(nn.Module):
    def __init__(self, c, layers):
        super().__init__()
        self.blocks = nn.ModuleList(Block(c) for _ in range(layers))
        self.norm = nn.LayerNorm(c.channels)
        self.width = c.channels // c.heads

    def forward(self, x, mask):
        table = rotary_table(x.shape[1], self.width, x.device)
        for block in self.blocks:
            x = block(x, mask, table)
        return self.norm(x) * mask[..., None]


class Predictor(nn.Module):
    def __init__(self, c, outputs):
        super().__init__()
        p = c.predictor_channels
        self.conv1 = nn.Conv1d(c.channels, p, 3, padding=1)
        self.conv2 = nn.Conv1d(p, p, 3, padding=1)
        self.norm1, self.norm2 = nn.LayerNorm(p), nn.LayerNorm(p)
        self.drop = nn.Dropout(c.dropout)
        self.out = nn.Linear(p, outputs)

    def forward(self, x, mask):
        x = self.conv1((x * mask[..., None]).transpose(1, 2)).transpose(1, 2)
        x = self.drop(self.norm1(F.gelu(x))) * mask[..., None]
        x = self.conv2(x.transpose(1, 2)).transpose(1, 2)
        x = self.drop(self.norm2(F.gelu(x)))
        return self.out(x) * mask[..., None]


def pulse_train(f0, sample_rate, hop, limit=0.47, phase=None):
    """Band-limited pulse train [B, T*hop] with unit power for f0 [B, T] in Hz.

    Frame values sit at samples i*hop and are interpolated in the log domain.
    The sum of all harmonics below ``limit * sample_rate`` has the closed form
    sin((K+1/2)x) / (2 sin(x/2)) - 1/2; the top harmonic fades in and out so the
    signal stays continuous while the pitch glides. Float64 keeps the phase exact.
    """
    frames = f0.shape[1]
    knots = torch.log(f0.double().clamp_min(1.0))
    knots = torch.cat((knots, knots[:, -1:]), dim=1)
    position = torch.arange(frames * hop, device=f0.device, dtype=torch.float64) / hop
    low = position.floor().long()
    weight = position - low
    frequency = torch.exp(knots[:, low] * (1 - weight) + knots[:, low + 1] * weight)
    theta = 2 * math.pi * torch.cumsum(frequency / sample_rate, dim=1)
    if phase is not None:
        theta = theta + phase.double()[:, None]
    theta = torch.remainder(theta, 2 * math.pi)
    harmonics = (limit * sample_rate / frequency).clamp_min(1.0)
    whole = torch.floor(harmonics)
    denominator = 2 * torch.sin(theta / 2)
    tiny = denominator.abs() < 1e-7
    core = torch.where(tiny, whole, torch.sin((whole + 0.5) * theta) / torch.where(tiny, torch.ones_like(denominator), denominator) - 0.5)
    wave = core + (harmonics - whole) * torch.cos((whole + 1) * theta)
    return (wave * torch.sqrt(2.0 / harmonics)).float()


def natural_pulse_train(f0, sample_rate, hop, limit=0.47, jitter=0.0, shimmer=0.0, generator=None):
    """Hindi port, inference only: pulse_train with the cycle-to-cycle irregularity of a real voice.

    jitter:  each glottal cycle gets its own frequency factor (1 + jitter * N(0,1)); local jitter (mean |dT|/T) is
             ~1.13 x jitter, so 0.005 ~ 0.6% (normal voices: 0.3-1%).
    shimmer: each pulse gets its own amplitude factor (1 + shimmer * N(0,1)), interpolated from peak to peak so there
             is no step inside a cycle; 0.035 ~ 4% local shimmer (normal voices: 2-6%).
    Returns (wave [B, S] float32, unwrapped phase theta [B, S] float64 for pitch-synchronous effects)."""
    frames = f0.shape[1]
    knots = torch.log(f0.double().clamp_min(1.0))
    knots = torch.cat((knots, knots[:, -1:]), dim=1)
    position = torch.arange(frames * hop, device=f0.device, dtype=torch.float64) / hop
    low = position.floor().long()
    weight = position - low
    frequency = torch.exp(knots[:, low] * (1 - weight) + knots[:, low + 1] * weight)
    if jitter > 0:
        cycle = torch.cumsum(frequency / sample_rate, dim=1).floor().long()
        factors = 1 + jitter * torch.randn(f0.shape[0], int(cycle.max()) + 2, generator=generator, dtype=torch.float64)
        frequency = frequency * factors.to(f0.device).gather(1, cycle)
    unwrapped = 2 * math.pi * torch.cumsum(frequency / sample_rate, dim=1)
    theta = torch.remainder(unwrapped, 2 * math.pi)
    harmonics = (limit * sample_rate / frequency).clamp_min(1.0)
    whole = torch.floor(harmonics)
    denominator = 2 * torch.sin(theta / 2)
    tiny = denominator.abs() < 1e-7
    core = torch.where(tiny, whole, torch.sin((whole + 0.5) * theta) / torch.where(tiny, torch.ones_like(denominator), denominator) - 0.5)
    wave = (core + (harmonics - whole) * torch.cos((whole + 1) * theta)) * torch.sqrt(2.0 / harmonics)
    if shimmer > 0:
        cycles = unwrapped / (2 * math.pi)
        index = cycles.floor().long()
        frac = cycles - index
        amp = (1 + shimmer * torch.randn(f0.shape[0], int(index.max()) + 2, generator=generator, dtype=torch.float64)).clamp_min(0.2)
        amp = amp.to(f0.device)
        wave = wave * (amp.gather(1, index) * (1 - frac) + amp.gather(1, index + 1) * frac)
    return wave.float(), unwrapped


def filtered(source, response, fft, hop, window):
    """Time-varying filtering: response [B, frames, bins] multiplies the STFT."""
    spectrum = torch.stft(source, fft, hop, window=window, center=True, pad_mode="constant", return_complex=True)
    if spectrum.shape[-1] != response.shape[1]:
        raise ValueError("filter frames do not match the source")
    return torch.istft(spectrum * response.transpose(1, 2), fft, hop, window=window, center=True, length=source.shape[-1])


def cepstrum_response(causal, anticausal, fft):
    """Complex frequency response exp(FFT(cepstrum)). Causal coefficients give a
    minimum-phase filter; anticausal ones add a maximum-phase part (glottal pulse)."""
    buffer = causal.new_zeros(*causal.shape[:-1], fft)
    buffer[..., :causal.shape[-1]] = causal
    if anticausal is not None:
        buffer[..., fft - anticausal.shape[-1]:] = anticausal.flip(-1)
    spectrum = torch.fft.rfft(buffer, dim=-1)
    return torch.polar(torch.exp(spectrum.real.clamp(-14, 6)), spectrum.imag)


class SourceFilter(nn.Module):
    """Harmonic-plus-noise synthesis controlled per frame by two linear heads."""

    def __init__(self, c):
        super().__init__()
        self.config = c
        self.harmonic_steps, self.noise_steps = c.hop // c.synth_hop, c.hop // c.noise_hop
        self.harmonic = nn.Linear(c.channels, c.cep_causal + c.cep_anti + self.harmonic_steps)
        self.noise = nn.Linear(c.channels, c.noise_cep + self.noise_steps)
        for head, level in ((self.harmonic, -1.5), (self.noise, -2.5)):   # start quiet and spectrally flat
            nn.init.zeros_(head.weight)
            nn.init.zeros_(head.bias)
            head.bias.data[0] = level

        def lifter(size, first):
            scale = 1.0 / (1.0 + torch.arange(first, first + size, dtype=torch.float32) / 4)
            if first == 0:
                scale[0] = 2.0      # coefficient 0 is the log gain and needs range
            return scale
        self.register_buffer("causal_lifter", lifter(c.cep_causal, 0), persistent=False)
        self.register_buffer("anti_lifter", lifter(c.cep_anti, 1), persistent=False)
        self.register_buffer("noise_lifter", lifter(c.noise_cep, 0), persistent=False)
        self.register_buffer("synth_window", torch.hann_window(c.synth_fft), persistent=False)
        self.register_buffer("noise_window", torch.hann_window(c.noise_fft), persistent=False)

    @staticmethod
    def _take(values, index):
        return values.gather(1, index[..., None].expand(-1, -1, values.shape[-1]))

    def _subframes(self, frames, lengths, steps):
        """Sub-frame j uses frame j // steps; indices past an utterance's end
        repeat its last sub-frame so padding never changes real audio. Two extra
        sub-frames cover the half window of zeros appended to the source."""
        sub = torch.arange(steps * frames + 3, device=lengths.device)[None].expand(len(lengths), -1)
        return torch.minimum(sub, (steps * lengths - 1).clamp_min(0)[:, None])

    @staticmethod
    def _shape(source, response, fft, hop, window):
        # Half a window of trailing zeros: the frames around an utterance's end are
        # then the same whether it stands alone or is padded inside a batch.
        return filtered(F.pad(source, (0, fft // 2)), response, fft, hop, window)[:, :source.shape[-1]]

    def _shape_mix(self, a, b, taper, response, fft, hop, window):
        """_shape for a source made of `a` below and `b` above a crossover: spectrum(a) * taper + spectrum(b) *
        sqrt(1 - taper^2), then the time-varying filter. Equal power density in a and b keeps the envelope."""
        S = a.shape[-1]
        sa = torch.stft(F.pad(a, (0, fft // 2)), fft, hop, window=window, center=True, pad_mode="constant", return_complex=True)
        sb = torch.stft(F.pad(b, (0, fft // 2)), fft, hop, window=window, center=True, pad_mode="constant", return_complex=True)
        t = taper[None, :, None]
        mixed = sa * t + sb * torch.sqrt((1 - t * t).clamp_min(0))
        y = torch.istft(mixed * response.transpose(1, 2), fft, hop, window=window, center=True, length=S + fft // 2)
        return y[:, :S]

    def forward(self, hidden, f0, lengths, noise=None, phase=None, voiced=None):
        c = self.config
        nat = getattr(self, "naturalness", None)       # Hindi port, inference only (see natural_pulse_train)
        frames = hidden.shape[1]
        samples = frames * c.hop
        with torch.autocast(device_type=hidden.device.type, enabled=False):
            hidden = hidden.float()
            audible = length_mask(lengths * c.hop, samples)
            last = (lengths - 1).clamp_min(0)[:, None]
            # Harmonic branch: envelope interpolated between frames, one gain per sub-frame.
            h = self.harmonic(hidden)
            split = c.cep_causal + c.cep_anti
            sub = self._subframes(frames, lengths, self.harmonic_steps)
            here, following = sub // self.harmonic_steps, torch.minimum((sub + 1) // self.harmonic_steps, last)
            causal = 0.5 * (self._take(h[..., :c.cep_causal], here) + self._take(h[..., :c.cep_causal], following)) * self.causal_lifter
            anti = 0.5 * (self._take(h[..., c.cep_causal:split], here) + self._take(h[..., c.cep_causal:split], following)) * self.anti_lifter
            gains = h[..., split:].flatten(1)
            if getattr(self, "smooth_gains", False):
                # Hindi port, inference only: the GAN-trained model flickers the per-sub-frame harmonic gain up and down
                # (~94 Hz at synth_hop 128, up to ~20 dB) at transitions, heard as a gargle. [1, 2, 1]/4 removes the
                # alternation and keeps changes slower than one frame.
                gains = F.conv1d(F.pad(gains[:, None], (1, 1), mode="replicate"), gains.new_tensor([[[0.25, 0.5, 0.25]]]))[:, 0]
            causal = torch.cat((causal[..., :1] + gains.gather(1, sub)[..., None], causal[..., 1:]), dim=-1)
            if getattr(self, "adaptive_lifter", 0.0):
                # Hindi port, inference only: above sr/80 = 300 Hz the 80 causal coefficients can describe the harmonic
                # comb itself (quefrency 1/F0). Taper the coefficients from adaptive_lifter * sr/F0 to sr/F0 so the
                # filter stays a smooth vocal-tract envelope at high pitch (no effect below ~300 Hz).
                f0_sub = f0.float().gather(1, torch.minimum(here, last)).clamp_min(1.0)                   # [B, S]
                k = torch.arange(c.cep_causal, device=causal.device, dtype=torch.float32)
                period = c.sample_rate / f0_sub[..., None]                                             # in coefficients
                start = self.adaptive_lifter * period
                taper = 0.5 * (1 + torch.cos(math.pi * ((k - start) / (period - start).clamp_min(1.0)).clamp(0, 1)))
                taper[..., 0] = 1.0
                causal = causal * taper
            harmonic_response = cepstrum_response(causal, anti, c.synth_fft)
            # Noise branch: one envelope per frame, four gains per frame for sharp consonants.
            n = self.noise(hidden)
            sub = self._subframes(frames, lengths, self.noise_steps)
            shape = self._take(n[..., :c.noise_cep], sub // self.noise_steps) * self.noise_lifter
            if getattr(self, "smooth_noise_env", False):
                # Hindi port, inference only: the noise envelope is held for a whole frame and jumps at every frame
                # boundary, an amplitude flutter at the frame rate (24000/256 = 93.75 Hz) heard as a buzz on earbuds.
                # Interpolate it linearly between frame centres instead.
                pos = sub.float() / self.noise_steps
                lo = pos.floor().long()
                hi = torch.minimum(lo + 1, last)
                w = (pos - lo.float())[..., None]
                shape = ((1 - w) * self._take(n[..., :c.noise_cep], lo) + w * self._take(n[..., :c.noise_cep], hi)) * self.noise_lifter
            noise_gains = n[..., c.noise_cep:].flatten(1)
            if getattr(self, "smooth_noise_gains", False):
                # same flicker in the noise gains (a per-frame sawtooth); [1, 2, 2, 2, 1]/8 removes period-2 and period-4 parts
                noise_gains = F.conv1d(F.pad(noise_gains[:, None], (2, 2), mode="replicate"),
                                       noise_gains.new_tensor([[[1, 2, 2, 2, 1]]]) / 8)[:, 0]
            shape = torch.cat((shape[..., :1] + noise_gains.gather(1, sub)[..., None], shape[..., 1:]), dim=-1)
            noise_response = cepstrum_response(shape, None, c.noise_fft)
            # Hold the last real pitch value through padding: the pitch glide inside an
            # utterance's final frame must not depend on what the batch was padded with.
            f0 = f0.float().gather(1, torch.minimum(torch.arange(frames, device=f0.device)[None], last))
            if nat:
                gen = torch.Generator().manual_seed(int(nat.get("seed", 0)) + 104729)
                pulses, theta = natural_pulse_train(f0, c.sample_rate, c.hop, c.harmonic_limit, nat.get("jitter", 0.0),
                                                    nat.get("shimmer", 0.0), gen)
                pulses = pulses * audible
            else:
                pulses = pulse_train(f0, c.sample_rate, c.hop, c.harmonic_limit, phase) * audible
            if getattr(self, "mixed_excitation", False) and voiced is not None:
                # Hindi port, inference only. The oscillator runs through unvoiced sounds too, and the model learned to
                # build much of /s/, /k/, /t/ and breaths from PULSES (measured: the pulse branch is ~8 dB louder than
                # the noise branch in unvoiced frames) -> an electronic buzz under the consonants (the "buzzing
                # artifact" of DDSP vocoders, Wu et al. 2022). Where a frame is unvoiced, excite the same filter with
                # white noise of the same power instead (mixed excitation, as MELP does); crossfade over ~1 frame.
                vm = F.avg_pool1d(F.pad(voiced.float()[:, None], (1, 1), mode="replicate"), 3, 1)[:, 0]
                vm = torch.cat((vm, vm[:, -1:]), dim=1)
                position = torch.arange(samples, device=vm.device, dtype=torch.float32) / c.hop
                low = position.floor().long()
                weight = position - low
                vs = (vm[:, low] * (1 - weight) + vm[:, low + 1] * weight).to(pulses.device)
                alt = torch.randn(hidden.shape[0], samples, generator=torch.Generator().manual_seed(977)).to(pulses.device)
                pulses = (pulses * vs + alt * torch.sqrt((1 - vs * vs).clamp_min(0))) * audible
            if noise is None:
                noise = torch.randn(hidden.shape[0], samples, device=hidden.device)
            if nat and nat.get("breath_pulse", 0.0) > 0 and voiced is not None:
                # breath noise puffs with each glottal cycle (strongest in the open phase, a quarter cycle before the
                # closure pulse at theta = 0), only where the frame is voiced; mean power is unchanged.
                d, kappa = float(nat["breath_pulse"]), 2.0
                v = F.avg_pool1d(F.pad(voiced.float()[:, None], (2, 2), mode="replicate"), 5, 1)[:, 0]
                v = v.repeat_interleave(c.hop, dim=1)[:, :samples].to(noise.device)
                g = torch.exp(kappa * (torch.cos(theta - (-math.pi / 2)) - 1)).float().to(noise.device)
                mean_g = math.exp(-kappa) * float(torch.special.i0(torch.tensor(kappa)))
                noise = noise * torch.sqrt((1 - d * v) + d * v * g / mean_g)
            if nat and nat.get("mvf_hz", 0.0) > 0:
                # voice turns into breath at high frequencies: above the maximum voiced frequency the harmonic branch is
                # excited by noise instead of pulses (same filter, same power density), cosine crossover
                lo, width = float(nat["mvf_hz"]), float(nat.get("mvf_width_hz", 2500.0))
                freqs = torch.arange(c.synth_fft // 2 + 1, dtype=torch.float32) * c.sample_rate / c.synth_fft
                taper = 0.5 * (1 + torch.cos(math.pi * ((freqs - lo) / width).clamp(0, 1)))
                breath = torch.randn(hidden.shape[0], samples, generator=torch.Generator().manual_seed(int(nat.get("seed", 0)) + 7))
                harmonic = self._shape_mix(pulses, breath.to(pulses.device) * audible, taper.to(pulses.device), harmonic_response,
                                           c.synth_fft, c.synth_hop, self.synth_window)
            else:
                harmonic = self._shape(pulses, harmonic_response, c.synth_fft, c.synth_hop, self.synth_window)
            wave = harmonic + self._shape(noise * audible, noise_response, c.noise_fft, c.noise_hop, self.noise_window)
            return wave * audible


class TinyVoca(nn.Module):
    def __init__(self, config=Config()):
        super().__init__()
        self.config = c = config
        self.tokens = nn.Embedding(c.vocab_size, c.channels, padding_idx=0)
        self.stress = nn.Embedding(3, c.channels)
        self.voices = nn.Embedding(len(c.speakers), c.channels)
        self.encoder = Stack(c, c.encoder_layers)
        # Outputs per token: log(1 + frames) and, for pause-capable tokens, a
        # "has frames at all" logit. A pause is absent more often than present, so
        # one regression would average both cases into a pause that is too short.
        self.duration = Predictor(c, 2)
        self.frame_voices = nn.Embedding(len(c.speakers), c.channels)
        self.frame_position = nn.Linear(3, c.channels)
        self.prosody_stack = Stack(c, c.prosody_layers)
        # Outputs: continuous log(F0/reference), voiced logit, normalized log RMS energy.
        self.prosody_head = nn.Linear(c.channels, 3)
        self.prosody_projection = nn.Linear(3, c.channels)
        self.acoustic_stack = Stack(c, c.acoustic_layers)
        self.vocoder = SourceFilter(c)
        phones = torch.zeros(c.vocab_size, dtype=torch.bool)
        phones[sorted(PHONE_IDS)] = True
        self.register_buffer("phone_tokens", phones, persistent=False)
        vowels = torch.zeros(c.vocab_size, dtype=torch.bool)
        vowels[[SYMBOL_TO_ID[v] for v in VOWELS]] = True
        self.register_buffer("vowel_tokens", vowels, persistent=False)

    def encode(self, tokens, stress, lengths, speakers):
        if tokens.ndim != 2 or stress.shape != tokens.shape or lengths.shape != (tokens.shape[0],) or speakers.shape != lengths.shape:
            raise ValueError("expected tokens[B,N], stress[B,N], lengths[B], speakers[B]")
        if tokens.shape[1] > self.config.max_tokens or bool((lengths < 1).any()) or bool((lengths > tokens.shape[1]).any()):
            raise ValueError("invalid token length")
        mask = length_mask(lengths, tokens.shape[1])
        x = (self.tokens(tokens) + self.stress(stress) + self.voices(speakers)[:, None]) * mask[..., None]
        return self.encoder(x, mask), mask

    def expand(self, hidden, durations, token_mask, speakers):
        """Repeat token states by integer durations (zero is allowed for pause
        tokens) and add each frame's position inside its token."""
        if durations.shape != token_mask.shape or bool((durations < 0).any()) or bool((durations[~token_mask] != 0).any()):
            raise ValueError("invalid durations")
        lengths = durations.sum(1)
        if bool((lengths < 1).any()):
            raise ValueError("utterance has no frames")
        if bool((lengths > self.config.max_frames).any()):
            raise UtteranceTooLong("utterance exceeds max_frames; split the input")
        size = int(lengths.max())
        ends = durations.cumsum(1)
        frame = torch.arange(size, device=hidden.device)[None].expand(hidden.shape[0], size).contiguous()
        index = torch.searchsorted(ends, frame, right=True).clamp(max=durations.shape[1] - 1)
        owned = durations.gather(1, index).clamp_min(1)
        offset = frame - (ends.gather(1, index) - durations.gather(1, index))
        relative = (offset.to(hidden.dtype) + 0.5) / owned
        features = torch.stack((relative, 1 - relative, torch.log1p(owned.to(hidden.dtype)) / 4), dim=-1)
        mask = frame < lengths[:, None]
        expanded = hidden.gather(1, index[..., None].expand(-1, -1, hidden.shape[-1]))
        expanded = expanded + self.frame_position(features) + self.frame_voices(speakers)[:, None]
        return expanded * mask[..., None], mask, lengths

    def predict_prosody(self, frames, frame_mask):
        hidden = self.prosody_stack(frames, frame_mask)
        return hidden, self.prosody_head(hidden) * frame_mask[..., None]

    @staticmethod
    def hard_prosody(prediction):
        """What the decoder is conditioned on when it is not teacher-forced."""
        return torch.stack((prediction[..., 0].clamp(-2.0, 1.5), (prediction[..., 1] > 0).to(prediction.dtype),
                            prediction[..., 2].clamp(-4.5, 3.0)), dim=-1)

    def decode(self, prosody_hidden, condition, frame_mask, noise=None):
        h = prosody_hidden + self.prosody_projection(condition)
        if getattr(self, "frame_noise", None) is not None:
            # Hindi port (anti-blur fine-tune): a random vector per frame lets the acoustic stack pick ONE natural
            # version of the spectrum instead of the average of all of them (an average is what sounds blurred).
            if noise is None:
                t = getattr(self, "noise_temperature", 1.0)
                noise = t * torch.randn(*h.shape[:2], self.frame_noise.in_features, device=h.device)
            h = h + self.frame_noise(noise.to(h.dtype))
        return self.acoustic_stack(h * frame_mask[..., None], frame_mask)

    def add_frame_noise(self, dim=16):
        """Add the per-frame noise input (zero-initialised: the model's output is unchanged until it is trained)."""
        self.frame_noise = nn.Linear(dim, self.config.channels)
        nn.init.zeros_(self.frame_noise.weight)
        nn.init.zeros_(self.frame_noise.bias)
        return self

    def frequency(self, log_f0):
        c = self.config
        return (c.pitch_reference_hz * torch.exp(log_f0.float())).clamp(c.f0_min, c.f0_max)

    def predict_durations(self, hidden, mask, tokens, speed=1.0):
        output = self.duration(hidden, mask).float()
        frames = torch.round(torch.expm1(output[..., 0].clamp(0, math.log1p(400))) / speed).long().clamp_min(1)
        # Every phoneme is audible for at least one frame. A pause-capable token
        # (edge, punctuation, word boundary) lasts its predicted length when the
        # presence logit says there is a pause, and vanishes otherwise.
        present = self.phone_tokens[tokens] | (output[..., 1] > 0)
        return torch.where(present, frames, torch.zeros_like(frames)) * mask

    @staticmethod
    def widen_pitch(prosody, frame_mask, pitch_range):
        """Scale the predicted pitch movement around each utterance's voiced mean.
        Regression predicts a flatter contour than the recordings; values above 1
        restore range (1.0 leaves the prediction untouched)."""
        if pitch_range == 1.0:
            return prosody
        voiced = ((prosody[..., 1] > 0) & frame_mask).to(prosody.dtype)
        mean = (prosody[..., 0] * voiced).sum(1, keepdim=True) / voiced.sum(1, keepdim=True).clamp_min(1)
        pitch = mean + (prosody[..., 0] - mean) * pitch_range
        return torch.cat((pitch[..., None], prosody[..., 1:]), dim=-1) * frame_mask[..., None]

    @staticmethod
    def pitch_wobble(shape, semitones, seed, device):
        """Small random pitch movement for the oscillator only (log-F0 units).
        Recorded pitch has cycle-to-cycle irregularity that a predicted contour
        lacks; two thirds of the wobble is slow drift, one third frame-level."""
        generator = torch.Generator().manual_seed(seed + 7919)
        fast = torch.randn(shape, generator=generator)
        slow = F.avg_pool1d(torch.randn(shape[0], 1, shape[1] + 16, generator=generator), 17, stride=1)[:, 0]
        slow = slow / slow.std().clamp_min(1e-6)
        return ((2 * slow + fast) / 5 ** 0.5 * semitones * math.log(2) / 12).to(device)

    @torch.inference_mode()
    def synthesize(self, tokens, stress, lengths, speakers, speed=1.0, pitch_shift=0.0, seed=0, pitch_range=1.0, pitch_jitter=0.0,
                   voiced_vowels=True, pitch_smooth=0, octave_fix=0.0):
        """Free-running synthesis: predicted durations, pitch, voicing and energy.

        Hindi port: ``pitch_smooth`` = width (frames, odd) of a running median over the predicted log F0. The YIN pitch
        targets carry ~10 jumps of > 4 semitones per 1,000 voiced frames (tracker errors), the model learned them, and
        they are heard as hiccups; a 5-frame (53 ms) median removes 10 ms blips and keeps the intonation."""
        if not math.isfinite(speed) or not 0.25 <= speed <= 4 or not math.isfinite(pitch_shift) or not 0 <= pitch_range <= 3:
            raise ValueError("speed must be between 0.25 and 4, pitch_shift finite and pitch_range between 0 and 3")
        hidden, mask = self.encode(tokens, stress, lengths, speakers)
        durations = self.predict_durations(hidden, mask, tokens, speed)
        frames, frame_mask, frame_lengths = self.expand(hidden, durations, mask, speakers)
        prosody_hidden, prosody = self.predict_prosody(frames, frame_mask)
        if voiced_vowels:
            # The pitch tracker missed creaky vowels in the recordings, and the
            # voicing predictor learned to drop whole vowels (heard as a whispered
            # word). A vowel is voiced by definition, so its frames are kept voiced.
            ends = durations.cumsum(1)
            frame = torch.arange(frames.shape[1], device=tokens.device)[None].expand(len(lengths), -1).contiguous()
            owner = torch.searchsorted(ends, frame, right=True).clamp(max=tokens.shape[1] - 1)
            vowel = self.vowel_tokens[tokens].gather(1, owner) & frame_mask
            prosody = torch.cat((prosody[..., :1], torch.where(vowel, prosody[..., 1].clamp_min(2.0), prosody[..., 1])[..., None],
                                 prosody[..., 2:]), dim=-1)
        if octave_fix:
            # YIN octave errors in the training targets taught the model short excursions to ~2x the pitch (heard as
            # hiccups). Pitch above the utterance's voiced median is soft-limited: median + T*tanh(excess/T), with
            # T = log(octave_fix), so a rise can approach but never pass octave_fix x the median, with no new edges.
            voiced = (prosody[..., 1] > 0) & frame_mask
            logf0 = prosody[..., 0]
            masked = torch.where(voiced, logf0, torch.full_like(logf0, float("nan")))
            median = masked.nanmedian(1, keepdim=True).values.nan_to_num(0.0)
            limit = math.log(octave_fix)
            excess = logf0 - median
            limited = torch.where(excess > 0, median + limit * torch.tanh(excess / limit), logf0)
            prosody = torch.cat((limited[..., None], prosody[..., 1:]), dim=-1)
        if pitch_smooth and pitch_smooth > 1:
            w = int(pitch_smooth) | 1
            padded = F.pad(prosody[..., 0][:, None], (w // 2, w // 2), mode="replicate")[:, 0]
            smoothed = padded.unfold(1, w, 1).median(-1).values
            prosody = torch.cat((smoothed[..., None], prosody[..., 1:]), dim=-1) * frame_mask[..., None]
        prosody = self.widen_pitch(prosody, frame_mask, pitch_range)
        acoustic = self.decode(prosody_hidden, self.hard_prosody(prosody), frame_mask)
        # One noise stream per row, drawn at the row's own length, so batching or
        # padding never changes a row's audio.
        noise = nn.utils.rnn.pad_sequence([torch.randn(int(n) * self.config.hop, generator=torch.Generator().manual_seed(seed + row))
                                           for row, n in enumerate(frame_lengths)], batch_first=True)
        pitch = prosody[..., 0] + pitch_shift
        if pitch_jitter > 0:
            pitch = pitch + self.pitch_wobble(pitch.shape, pitch_jitter, seed, pitch.device)
        wave = self.vocoder(acoustic, self.frequency(pitch), frame_lengths, noise=noise.to(hidden.device),
                            voiced=(prosody[..., 1] > 0) & frame_mask)
        return wave, frame_lengths * self.config.hop, {"durations": durations, "prosody": prosody, "frame_mask": frame_mask}