File size: 4,929 Bytes
ab1b068
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
"""Reference intermediates from the PyTorch Hindi tiny-voca model, for checking the C++ engine (tts_cpp/) stage by stage.

  python scripts/reference.py --out cpp/ref

Writes <name>.bin (raw float32 or int32, row-major) and meta.json (shapes, dtypes, inputs) per test case. The
step-by-step pipeline below must reproduce model.synthesize exactly (asserted), so the dumps are the real model's.
"""
import argparse
import json
import math
import sys
from pathlib import Path

import numpy as np
import torch
from torch import nn

ROOT = Path(__file__).resolve().parent.parent
sys.path.insert(0, str(ROOT))
from tiny_voca.infer import OCTAVE_FIX, load_model                 # noqa: E402
from tiny_voca.text import phonemize                               # noqa: E402

CASES = [("neutral", "भारत एक विशाल देश है, जहाँ अनेक भाषाएँ बोली जाती हैं."),
         ("surprise", "क्या? तुमने यह सब अकेले बनाया?"),
         ("conv", "अरे यार, तुम कल आए क्यों नहीं?")]


@torch.inference_mode()
def run(m, text, style, seed, out):
    c = m.config
    ph = phonemize(text)
    tokens = torch.tensor([ph.ids]); stress = torch.tensor([ph.stress]); lengths = torch.tensor([len(ph)])
    speakers = torch.tensor([c.speakers.index(style)])
    hidden, mask = m.encode(tokens, stress, lengths, speakers)
    dur_raw = m.duration(hidden, mask).float()
    durations = m.predict_durations(hidden, mask, tokens)
    frames, frame_mask, frame_lengths = m.expand(hidden, durations, mask, speakers)
    prosody_hidden, prosody_raw = m.predict_prosody(frames, frame_mask)
    # same post-processing as synthesize(voiced_vowels=True, octave_fix=OCTAVE_FIX)
    ends = durations.cumsum(1)
    frame = torch.arange(frames.shape[1])[None].contiguous()
    owner = torch.searchsorted(ends, frame, right=True).clamp(max=tokens.shape[1] - 1)
    vowel = m.vowel_tokens[tokens].gather(1, owner) & frame_mask
    prosody = torch.cat((prosody_raw[..., :1], torch.where(vowel, prosody_raw[..., 1].clamp_min(2.0), prosody_raw[..., 1])[..., None],
                         prosody_raw[..., 2:]), dim=-1)
    voiced = (prosody[..., 1] > 0) & frame_mask
    logf0 = prosody[..., 0]
    median = torch.where(voiced, logf0, torch.full_like(logf0, float("nan"))).nanmedian(1, keepdim=True).values.nan_to_num(0.0)
    limit = math.log(OCTAVE_FIX)
    excess = logf0 - median
    prosody = torch.cat((torch.where(excess > 0, median + limit * torch.tanh(excess / limit), logf0)[..., None], prosody[..., 1:]), dim=-1)
    acoustic = m.decode(prosody_hidden, m.hard_prosody(prosody), frame_mask)
    noise = nn.utils.rnn.pad_sequence([torch.randn(int(frame_lengths[0]) * c.hop, generator=torch.Generator().manual_seed(seed))], batch_first=True)
    f0 = m.frequency(prosody[..., 0])
    wave = m.vocoder(acoustic, f0, frame_lengths, noise=noise)
    ref, _, _ = m.synthesize(tokens, stress, lengths, speakers, seed=seed, octave_fix=OCTAVE_FIX)
    assert torch.equal(wave, ref), float((wave - ref).abs().max())
    harm = m.vocoder.harmonic(acoustic.float()); nz = m.vocoder.noise(acoustic.float())
    arrays = {"ids": np.array(ph.ids, np.int32), "stress": np.array(ph.stress, np.int32),
              "encoder": hidden[0].numpy(), "duration_raw": dur_raw[0].numpy(), "durations": durations[0].numpy().astype(np.int32),
              "frames": frames[0].numpy(), "prosody_hidden": prosody_hidden[0].numpy(), "prosody_raw": prosody_raw[0].numpy(),
              "prosody": prosody[0].numpy(), "acoustic": acoustic[0].numpy(), "f0": f0[0].numpy(),
              "harmonic_head": harm[0].numpy(), "noise_head": nz[0].numpy(), "noise": noise[0].numpy(), "wave": wave[0].numpy()}
    out.mkdir(parents=True, exist_ok=True)
    meta = {"text": text, "style": style, "style_id": int(speakers[0]), "seed": seed, "symbols": ph.symbols,
            "octave_fix": OCTAVE_FIX, "arrays": {}}
    for k, v in arrays.items():
        v = np.ascontiguousarray(v)
        v.astype(v.dtype).tofile(out / f"{k}.bin")
        meta["arrays"][k] = {"shape": list(v.shape), "dtype": str(v.dtype)}
    (out / "meta.json").write_text(json.dumps(meta, ensure_ascii=False, indent=1), encoding="utf-8")
    (out / "inputs.txt").write_text(f"{int(speakers[0])} {seed} {OCTAVE_FIX}\n")       # for the C++ checker
    return len(ph), int(frame_lengths[0])


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--checkpoint", default="models/tiny_voca_hindi.pt")
    ap.add_argument("--out", default="cpp/ref")
    a = ap.parse_args()
    m = load_model(a.checkpoint, "cpu").eval()
    for i, (style, text) in enumerate(CASES):
        n, t = run(m, text, style, seed=i, out=Path(a.out) / f"case{i}")
        print(json.dumps({"case": i, "style": style, "tokens": n, "frames": t}))


if __name__ == "__main__":
    main()