Download scripts/validate_batched_onnx.py from TuringLogic/little-tts: direct link, hf CLI and curl.
- Browser
- Download file 6.32 kB
-
https://huggingface.co/TuringLogic/little-tts/resolve/main/scripts/validate_batched_onnx.py
- Command line
-
hf download hf://TuringLogic/little-tts/scripts/validate_batched_onnx.py
-
curl -L -o validate_batched_onnx.py https://huggingface.co/TuringLogic/little-tts/resolve/main/scripts/validate_batched_onnx.py
6.32 kB
| """Compare batched FP32 audio with the released single-request ONNX graph.""" | |
| import argparse | |
| import json | |
| from pathlib import Path | |
| import sys | |
| import tempfile | |
| import numpy as np | |
| import onnx | |
| from onnx import helper as h, TensorProto as T | |
| import onnxruntime as ort | |
| ROOT = Path(__file__).resolve().parents[1] | |
| sys.path.insert(0, str(ROOT)) | |
| from batched_onnx import BatchedSynthesizer | |
| TEXTS = [ | |
| "The morning sunlight streamed through the window, warming the quiet room.", | |
| "Hello, welcome to Little TTS.", | |
| "She had never seen the sea before, and for a long while she stood at the edge of the water without saying a word.", | |
| "After a long and difficult journey across the mountains, the travellers finally reached the small village, where they were welcomed with warm food, a place to sleep and many questions about the road behind them.", | |
| ] | |
| def main(): | |
| p = argparse.ArgumentParser(description=__doc__) | |
| p.add_argument("--model-dir", type=Path, default=ROOT) | |
| p.add_argument("--device", choices=["cpu", "cuda"], default="cpu") | |
| p.add_argument("--output", type=Path, default=ROOT / "evaluation/batched_validation.json") | |
| args = p.parse_args() | |
| synth = BatchedSynthesizer(args.model_dir, args.device, max_batch=4) | |
| # Expose only the original graph's random noise, leaving all learned operators unchanged. | |
| model = onnx.load(args.model_dir / "onnx/model.onnx") | |
| for node in model.graph.node: | |
| if node.op_type == "RandomNormalLike": | |
| del node.input[:] | |
| node.input.append("reference_noise") | |
| node.op_type = "Identity" | |
| del node.attribute[:] | |
| model.graph.input.append(h.make_tensor_value_info("reference_noise", T.FLOAT, [1, "samples"])) | |
| onnx.checker.check_model(model) | |
| options = ort.SessionOptions() | |
| options.intra_op_num_threads = 2 | |
| with tempfile.TemporaryDirectory() as temp: | |
| path = Path(temp) / "reference.onnx" | |
| onnx.save(model, path) | |
| reference = ort.InferenceSession(str(path), options, providers=["CPUExecutionProvider"]) | |
| rows = [] | |
| cases = [(["92", "6097", "92", "6097"], [1, 1.1, .95, 1], [0]*4, [1]*4), | |
| (["6097", "92", "6097", "92"], [1.1, 1, 1, .95], [0]*4, [1]*4), | |
| (["92", "6097", "92", "6097"], [1]*4, [.05, -.03, .02, 0], [1.2, .8, 1.1, 1])] | |
| for voices, speeds, shifts, ranges in cases: | |
| result = synth.speak_batch(TEXTS, voices=voices, speeds=speeds, seeds=[10, 11, 12, 13], | |
| pitch_shifts=shifts, pitch_ranges=ranges) | |
| for j, ((wave, durations), text, voice, speed) in enumerate(zip(result, TEXTS, voices, speeds)): | |
| phones = synth.tokenize(synth.phonemizer.pieces(text)) | |
| noise = np.random.default_rng(10+j).standard_normal(len(wave), dtype=np.float32)[None] | |
| one = lambda v: np.array([v], np.float32) | |
| feeds = dict(tokens=np.array([phones.ids], np.int64), stress=np.array([phones.stress], np.int64), | |
| voice=np.array([synth.voices.index(voice)], np.int64), speed=one(speed), | |
| pitch_shift=one(shifts[j]), pitch_range=one(ranges[j]), reference_noise=noise) | |
| original, original_durations = reference.run(None, feeds) | |
| assert durations.tolist() == original_durations[0].tolist(), (j, "durations differ") | |
| assert len(wave) == original.shape[1], (j, "length differs") | |
| original = original[0] | |
| peak = float(np.max(np.abs(original))) | |
| error = wave - original | |
| row = dict(voice=voice, text=text, speed=speed, pitch_shift=shifts[j], pitch_range=ranges[j], seconds=len(wave)/24000, | |
| same_durations=True, same_length=True, | |
| max_error_over_peak=float(np.max(np.abs(error))/peak), | |
| rms_error_over_rms=float(np.sqrt(np.mean(error**2))/np.sqrt(np.mean(original**2)))) | |
| rows.append(row) | |
| print(json.dumps(row), flush=True) | |
| # Batch invariance, mixed settings and input ordering are independently checked. | |
| batch = synth.speak_batch(TEXTS[:3], voices=["6097", "92", "6097"], seeds=[5, 6, 7]) | |
| invariance = [] | |
| for j in range(3): | |
| single = synth.speak_batch([TEXTS[j]], voices=[["6097", "92", "6097"][j]], seeds=[5+j])[0] | |
| assert np.array_equal(batch[j][1], single[1]) | |
| delta = np.max(np.abs(batch[j][0] - single[0])) | |
| peak = np.max(np.abs(single[0])) | |
| invariance.append(float(delta/peak)) | |
| # Force a padded-frame-budget split and verify output ordering/durations. | |
| limited = BatchedSynthesizer(args.model_dir, args.device, max_batch=4, max_padded_frames=3000) | |
| limited_result = limited.speak_batch(TEXTS, seeds=[10, 11, 12, 13]) | |
| assert len(limited.last_stats["batches"]) > 1 | |
| assert all(b["batch_size"] * b["padded_frames"] <= 3000 for b in limited.last_stats["batches"]) | |
| for j, (wave, durations) in enumerate(limited_result): | |
| single = synth.speak_batch([TEXTS[j]], seeds=[10+j])[0] | |
| assert np.array_equal(durations, single[1]) | |
| assert np.max(np.abs(wave - single[0])) / np.max(np.abs(single[0])) < .005 | |
| report = dict(device=args.device, precision="float32", quantized=False, pytorch_imported="torch" in sys.modules, | |
| reference="Released FP32 single-request ONNX, with identical externally supplied noise", | |
| comparisons=rows, batch_invariance_max_error_over_peak=invariance, | |
| padded_frame_budget_checked=True, | |
| max_error_over_peak=max(r["max_error_over_peak"] for r in rows), | |
| max_rms_error_over_rms=max(r["rms_error_over_rms"] for r in rows)) | |
| args.output.parent.mkdir(parents=True, exist_ok=True) | |
| args.output.write_text(json.dumps(report, indent=2) + "\n") | |
| assert not report["pytorch_imported"] | |
| assert report["max_error_over_peak"] < .005, report | |
| assert report["max_rms_error_over_rms"] < .002, report | |
| assert max(invariance) < .005, report | |
| print("BATCHED_PARITY_PASSED", flush=True) | |
| if __name__ == "__main__": | |
| main() | |