little-tts / scripts /validate_batched_onnx.py
TuringLogic's picture
Add PyTorch-free FP32 GPU batch graphs, queue, validation and RTX 5090 benchmarks
550ce03 verified
Raw History Blame Contribute Delete
6.32 kB
"""Compare batched FP32 audio with the released single-request ONNX graph."""
import argparse
import json
from pathlib import Path
import sys
import tempfile
import numpy as np
import onnx
from onnx import helper as h, TensorProto as T
import onnxruntime as ort
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT))
from batched_onnx import BatchedSynthesizer
TEXTS = [
"The morning sunlight streamed through the window, warming the quiet room.",
"Hello, welcome to Little TTS.",
"She had never seen the sea before, and for a long while she stood at the edge of the water without saying a word.",
"After a long and difficult journey across the mountains, the travellers finally reached the small village, where they were welcomed with warm food, a place to sleep and many questions about the road behind them.",
]
def main():
p = argparse.ArgumentParser(description=__doc__)
p.add_argument("--model-dir", type=Path, default=ROOT)
p.add_argument("--device", choices=["cpu", "cuda"], default="cpu")
p.add_argument("--output", type=Path, default=ROOT / "evaluation/batched_validation.json")
args = p.parse_args()
synth = BatchedSynthesizer(args.model_dir, args.device, max_batch=4)
# Expose only the original graph's random noise, leaving all learned operators unchanged.
model = onnx.load(args.model_dir / "onnx/model.onnx")
for node in model.graph.node:
if node.op_type == "RandomNormalLike":
del node.input[:]
node.input.append("reference_noise")
node.op_type = "Identity"
del node.attribute[:]
model.graph.input.append(h.make_tensor_value_info("reference_noise", T.FLOAT, [1, "samples"]))
onnx.checker.check_model(model)
options = ort.SessionOptions()
options.intra_op_num_threads = 2
with tempfile.TemporaryDirectory() as temp:
path = Path(temp) / "reference.onnx"
onnx.save(model, path)
reference = ort.InferenceSession(str(path), options, providers=["CPUExecutionProvider"])
rows = []
cases = [(["92", "6097", "92", "6097"], [1, 1.1, .95, 1], [0]*4, [1]*4),
(["6097", "92", "6097", "92"], [1.1, 1, 1, .95], [0]*4, [1]*4),
(["92", "6097", "92", "6097"], [1]*4, [.05, -.03, .02, 0], [1.2, .8, 1.1, 1])]
for voices, speeds, shifts, ranges in cases:
result = synth.speak_batch(TEXTS, voices=voices, speeds=speeds, seeds=[10, 11, 12, 13],
pitch_shifts=shifts, pitch_ranges=ranges)
for j, ((wave, durations), text, voice, speed) in enumerate(zip(result, TEXTS, voices, speeds)):
phones = synth.tokenize(synth.phonemizer.pieces(text))
noise = np.random.default_rng(10+j).standard_normal(len(wave), dtype=np.float32)[None]
one = lambda v: np.array([v], np.float32)
feeds = dict(tokens=np.array([phones.ids], np.int64), stress=np.array([phones.stress], np.int64),
voice=np.array([synth.voices.index(voice)], np.int64), speed=one(speed),
pitch_shift=one(shifts[j]), pitch_range=one(ranges[j]), reference_noise=noise)
original, original_durations = reference.run(None, feeds)
assert durations.tolist() == original_durations[0].tolist(), (j, "durations differ")
assert len(wave) == original.shape[1], (j, "length differs")
original = original[0]
peak = float(np.max(np.abs(original)))
error = wave - original
row = dict(voice=voice, text=text, speed=speed, pitch_shift=shifts[j], pitch_range=ranges[j], seconds=len(wave)/24000,
same_durations=True, same_length=True,
max_error_over_peak=float(np.max(np.abs(error))/peak),
rms_error_over_rms=float(np.sqrt(np.mean(error**2))/np.sqrt(np.mean(original**2))))
rows.append(row)
print(json.dumps(row), flush=True)
# Batch invariance, mixed settings and input ordering are independently checked.
batch = synth.speak_batch(TEXTS[:3], voices=["6097", "92", "6097"], seeds=[5, 6, 7])
invariance = []
for j in range(3):
single = synth.speak_batch([TEXTS[j]], voices=[["6097", "92", "6097"][j]], seeds=[5+j])[0]
assert np.array_equal(batch[j][1], single[1])
delta = np.max(np.abs(batch[j][0] - single[0]))
peak = np.max(np.abs(single[0]))
invariance.append(float(delta/peak))
# Force a padded-frame-budget split and verify output ordering/durations.
limited = BatchedSynthesizer(args.model_dir, args.device, max_batch=4, max_padded_frames=3000)
limited_result = limited.speak_batch(TEXTS, seeds=[10, 11, 12, 13])
assert len(limited.last_stats["batches"]) > 1
assert all(b["batch_size"] * b["padded_frames"] <= 3000 for b in limited.last_stats["batches"])
for j, (wave, durations) in enumerate(limited_result):
single = synth.speak_batch([TEXTS[j]], seeds=[10+j])[0]
assert np.array_equal(durations, single[1])
assert np.max(np.abs(wave - single[0])) / np.max(np.abs(single[0])) < .005
report = dict(device=args.device, precision="float32", quantized=False, pytorch_imported="torch" in sys.modules,
reference="Released FP32 single-request ONNX, with identical externally supplied noise",
comparisons=rows, batch_invariance_max_error_over_peak=invariance,
padded_frame_budget_checked=True,
max_error_over_peak=max(r["max_error_over_peak"] for r in rows),
max_rms_error_over_rms=max(r["rms_error_over_rms"] for r in rows))
args.output.parent.mkdir(parents=True, exist_ok=True)
args.output.write_text(json.dumps(report, indent=2) + "\n")
assert not report["pytorch_imported"]
assert report["max_error_over_peak"] < .005, report
assert report["max_rms_error_over_rms"] < .002, report
assert max(invariance) < .005, report
print("BATCHED_PARITY_PASSED", flush=True)
if __name__ == "__main__":
main()