File size: 4,929 Bytes
ab1b068 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 | """Reference intermediates from the PyTorch Hindi tiny-voca model, for checking the C++ engine (tts_cpp/) stage by stage.
python scripts/reference.py --out cpp/ref
Writes <name>.bin (raw float32 or int32, row-major) and meta.json (shapes, dtypes, inputs) per test case. The
step-by-step pipeline below must reproduce model.synthesize exactly (asserted), so the dumps are the real model's.
"""
import argparse
import json
import math
import sys
from pathlib import Path
import numpy as np
import torch
from torch import nn
ROOT = Path(__file__).resolve().parent.parent
sys.path.insert(0, str(ROOT))
from tiny_voca.infer import OCTAVE_FIX, load_model # noqa: E402
from tiny_voca.text import phonemize # noqa: E402
CASES = [("neutral", "भारत एक विशाल देश है, जहाँ अनेक भाषाएँ बोली जाती हैं."),
("surprise", "क्या? तुमने यह सब अकेले बनाया?"),
("conv", "अरे यार, तुम कल आए क्यों नहीं?")]
@torch.inference_mode()
def run(m, text, style, seed, out):
c = m.config
ph = phonemize(text)
tokens = torch.tensor([ph.ids]); stress = torch.tensor([ph.stress]); lengths = torch.tensor([len(ph)])
speakers = torch.tensor([c.speakers.index(style)])
hidden, mask = m.encode(tokens, stress, lengths, speakers)
dur_raw = m.duration(hidden, mask).float()
durations = m.predict_durations(hidden, mask, tokens)
frames, frame_mask, frame_lengths = m.expand(hidden, durations, mask, speakers)
prosody_hidden, prosody_raw = m.predict_prosody(frames, frame_mask)
# same post-processing as synthesize(voiced_vowels=True, octave_fix=OCTAVE_FIX)
ends = durations.cumsum(1)
frame = torch.arange(frames.shape[1])[None].contiguous()
owner = torch.searchsorted(ends, frame, right=True).clamp(max=tokens.shape[1] - 1)
vowel = m.vowel_tokens[tokens].gather(1, owner) & frame_mask
prosody = torch.cat((prosody_raw[..., :1], torch.where(vowel, prosody_raw[..., 1].clamp_min(2.0), prosody_raw[..., 1])[..., None],
prosody_raw[..., 2:]), dim=-1)
voiced = (prosody[..., 1] > 0) & frame_mask
logf0 = prosody[..., 0]
median = torch.where(voiced, logf0, torch.full_like(logf0, float("nan"))).nanmedian(1, keepdim=True).values.nan_to_num(0.0)
limit = math.log(OCTAVE_FIX)
excess = logf0 - median
prosody = torch.cat((torch.where(excess > 0, median + limit * torch.tanh(excess / limit), logf0)[..., None], prosody[..., 1:]), dim=-1)
acoustic = m.decode(prosody_hidden, m.hard_prosody(prosody), frame_mask)
noise = nn.utils.rnn.pad_sequence([torch.randn(int(frame_lengths[0]) * c.hop, generator=torch.Generator().manual_seed(seed))], batch_first=True)
f0 = m.frequency(prosody[..., 0])
wave = m.vocoder(acoustic, f0, frame_lengths, noise=noise)
ref, _, _ = m.synthesize(tokens, stress, lengths, speakers, seed=seed, octave_fix=OCTAVE_FIX)
assert torch.equal(wave, ref), float((wave - ref).abs().max())
harm = m.vocoder.harmonic(acoustic.float()); nz = m.vocoder.noise(acoustic.float())
arrays = {"ids": np.array(ph.ids, np.int32), "stress": np.array(ph.stress, np.int32),
"encoder": hidden[0].numpy(), "duration_raw": dur_raw[0].numpy(), "durations": durations[0].numpy().astype(np.int32),
"frames": frames[0].numpy(), "prosody_hidden": prosody_hidden[0].numpy(), "prosody_raw": prosody_raw[0].numpy(),
"prosody": prosody[0].numpy(), "acoustic": acoustic[0].numpy(), "f0": f0[0].numpy(),
"harmonic_head": harm[0].numpy(), "noise_head": nz[0].numpy(), "noise": noise[0].numpy(), "wave": wave[0].numpy()}
out.mkdir(parents=True, exist_ok=True)
meta = {"text": text, "style": style, "style_id": int(speakers[0]), "seed": seed, "symbols": ph.symbols,
"octave_fix": OCTAVE_FIX, "arrays": {}}
for k, v in arrays.items():
v = np.ascontiguousarray(v)
v.astype(v.dtype).tofile(out / f"{k}.bin")
meta["arrays"][k] = {"shape": list(v.shape), "dtype": str(v.dtype)}
(out / "meta.json").write_text(json.dumps(meta, ensure_ascii=False, indent=1), encoding="utf-8")
(out / "inputs.txt").write_text(f"{int(speakers[0])} {seed} {OCTAVE_FIX}\n") # for the C++ checker
return len(ph), int(frame_lengths[0])
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--checkpoint", default="models/tiny_voca_hindi.pt")
ap.add_argument("--out", default="cpp/ref")
a = ap.parse_args()
m = load_model(a.checkpoint, "cpu").eval()
for i, (style, text) in enumerate(CASES):
n, t = run(m, text, style, seed=i, out=Path(a.out) / f"case{i}")
print(json.dumps({"case": i, "style": style, "tokens": n, "frames": t}))
if __name__ == "__main__":
main()
|