"""Reference intermediates from the PyTorch Hindi tiny-voca model, for checking the C++ engine (tts_cpp/) stage by stage. python scripts/reference.py --out cpp/ref Writes .bin (raw float32 or int32, row-major) and meta.json (shapes, dtypes, inputs) per test case. The step-by-step pipeline below must reproduce model.synthesize exactly (asserted), so the dumps are the real model's. """ import argparse import json import math import sys from pathlib import Path import numpy as np import torch from torch import nn ROOT = Path(__file__).resolve().parent.parent sys.path.insert(0, str(ROOT)) from tiny_voca.infer import OCTAVE_FIX, load_model # noqa: E402 from tiny_voca.text import phonemize # noqa: E402 CASES = [("neutral", "भारत एक विशाल देश है, जहाँ अनेक भाषाएँ बोली जाती हैं."), ("surprise", "क्या? तुमने यह सब अकेले बनाया?"), ("conv", "अरे यार, तुम कल आए क्यों नहीं?")] @torch.inference_mode() def run(m, text, style, seed, out): c = m.config ph = phonemize(text) tokens = torch.tensor([ph.ids]); stress = torch.tensor([ph.stress]); lengths = torch.tensor([len(ph)]) speakers = torch.tensor([c.speakers.index(style)]) hidden, mask = m.encode(tokens, stress, lengths, speakers) dur_raw = m.duration(hidden, mask).float() durations = m.predict_durations(hidden, mask, tokens) frames, frame_mask, frame_lengths = m.expand(hidden, durations, mask, speakers) prosody_hidden, prosody_raw = m.predict_prosody(frames, frame_mask) # same post-processing as synthesize(voiced_vowels=True, octave_fix=OCTAVE_FIX) ends = durations.cumsum(1) frame = torch.arange(frames.shape[1])[None].contiguous() owner = torch.searchsorted(ends, frame, right=True).clamp(max=tokens.shape[1] - 1) vowel = m.vowel_tokens[tokens].gather(1, owner) & frame_mask prosody = torch.cat((prosody_raw[..., :1], torch.where(vowel, prosody_raw[..., 1].clamp_min(2.0), prosody_raw[..., 1])[..., None], prosody_raw[..., 2:]), dim=-1) voiced = (prosody[..., 1] > 0) & frame_mask logf0 = prosody[..., 0] median = torch.where(voiced, logf0, torch.full_like(logf0, float("nan"))).nanmedian(1, keepdim=True).values.nan_to_num(0.0) limit = math.log(OCTAVE_FIX) excess = logf0 - median prosody = torch.cat((torch.where(excess > 0, median + limit * torch.tanh(excess / limit), logf0)[..., None], prosody[..., 1:]), dim=-1) acoustic = m.decode(prosody_hidden, m.hard_prosody(prosody), frame_mask) noise = nn.utils.rnn.pad_sequence([torch.randn(int(frame_lengths[0]) * c.hop, generator=torch.Generator().manual_seed(seed))], batch_first=True) f0 = m.frequency(prosody[..., 0]) wave = m.vocoder(acoustic, f0, frame_lengths, noise=noise) ref, _, _ = m.synthesize(tokens, stress, lengths, speakers, seed=seed, octave_fix=OCTAVE_FIX) assert torch.equal(wave, ref), float((wave - ref).abs().max()) harm = m.vocoder.harmonic(acoustic.float()); nz = m.vocoder.noise(acoustic.float()) arrays = {"ids": np.array(ph.ids, np.int32), "stress": np.array(ph.stress, np.int32), "encoder": hidden[0].numpy(), "duration_raw": dur_raw[0].numpy(), "durations": durations[0].numpy().astype(np.int32), "frames": frames[0].numpy(), "prosody_hidden": prosody_hidden[0].numpy(), "prosody_raw": prosody_raw[0].numpy(), "prosody": prosody[0].numpy(), "acoustic": acoustic[0].numpy(), "f0": f0[0].numpy(), "harmonic_head": harm[0].numpy(), "noise_head": nz[0].numpy(), "noise": noise[0].numpy(), "wave": wave[0].numpy()} out.mkdir(parents=True, exist_ok=True) meta = {"text": text, "style": style, "style_id": int(speakers[0]), "seed": seed, "symbols": ph.symbols, "octave_fix": OCTAVE_FIX, "arrays": {}} for k, v in arrays.items(): v = np.ascontiguousarray(v) v.astype(v.dtype).tofile(out / f"{k}.bin") meta["arrays"][k] = {"shape": list(v.shape), "dtype": str(v.dtype)} (out / "meta.json").write_text(json.dumps(meta, ensure_ascii=False, indent=1), encoding="utf-8") (out / "inputs.txt").write_text(f"{int(speakers[0])} {seed} {OCTAVE_FIX}\n") # for the C++ checker return len(ph), int(frame_lengths[0]) def main(): ap = argparse.ArgumentParser() ap.add_argument("--checkpoint", default="models/tiny_voca_hindi.pt") ap.add_argument("--out", default="cpp/ref") a = ap.parse_args() m = load_model(a.checkpoint, "cpu").eval() for i, (style, text) in enumerate(CASES): n, t = run(m, text, style, seed=i, out=Path(a.out) / f"case{i}") print(json.dumps({"case": i, "style": style, "tokens": n, "frames": t})) if __name__ == "__main__": main()