File size: 6,016 Bytes
f21d6eb
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
# SPDX-License-Identifier: Apache-2.0
# Copyright 2026 Kale
"""export check (16 Nov): an exported release graph vs its float32 reference built through the SAME steps (one-file layout, float32 phase,
rewrites, end clamp) from the training export's float32 weights. Noise source seeded in both, fixed inputs (20 sentences x 3 voices).
Bars per precision (16 Nov):
  float graphs (fp32 / fp16, no DequantizeLinear): every sentence the same length AND median waveform SNR >= 30 dB;
  int8 / fp16-weight graphs (17 Nov): at most 1 sentence per check set off by exactly +-1 frame (25 ms) and none by more AND median
  log-mel L1 <= 0.20 AND Whisper clarity within noise of the
  float32 reference (whisper_check.py, a separate step: it needs the Whisper scorer on Modal). Waveform SNR is reported for int8 too, but
  small int8 rounding adds up in the phase path, so it isn't the int8 bar.
Fixed inputs: 20 sentences x every voice in voices.json.
usage: python export_check.py <exported model.onnx> <reference model.onnx> <model dir (voices.json, voices/)> [--json out]"""
import json, sys
import numpy as np, onnx, onnxruntime as ort
exp, ref, d = sys.argv[1:4]; out = sys.argv[sys.argv.index("--json") + 1] if "--json" in sys.argv else None
IDS = __import__("os").environ.get("CHECK_IDS", str(__import__("pathlib").Path(__file__).resolve().parent / "check_ids.json"))
ETTIN = __import__("os").environ.get("CHECK_ETTIN", "")   # "<en-us.json>,<en-gb.json>": per-sentence ids + Ettin inputs from the JS front end
NS = int(__import__("os").environ.get("CHECK_N", "20"))   # sentences per voice from the CHECK_ETTIN items (20 = the release check)
ET = dict(zip(("en-us", "en-gb"), [[s for it in json.load(open(f)) for s in it["sents"]][:NS] for f in ETTIN.split(",")])) if ETTIN else None
def seeded(p):
    m = onnx.load(p)
    if any(i.name == "ettin_ids" for i in m.graph.input) and not ETTIN:
        raise SystemExit("graph needs Ettin inputs: set CHECK_ETTIN=<en-us items.json>,<en-gb items.json> (ettin_items.mjs on the check texts)")
    for n in m.graph.node:
        if n.op_type in ("RandomNormalLike", "RandomNormal", "RandomUniformLike", "RandomUniform"):
            n.attribute.append(onnx.helper.make_attribute("seed", 1.0))
    so = ort.SessionOptions(); so.intra_op_num_threads = 4
    return ort.InferenceSession(m.SerializeToString(), so)
def logmel(y, n=1024, h=256):
    F = np.lib.stride_tricks.sliding_window_view(np.pad(y, (n // 2, n // 2)), n)[::h] * np.hanning(n)
    S = np.abs(np.fft.rfft(F, axis=1)) ** 2; fr = np.fft.rfftfreq(n, 1 / 24000)
    mel = lambda f: 2595 * np.log10(1 + f / 700)  # noqa: E731
    edges = np.linspace(mel(0), mel(12000), 82); hz = 700 * (10 ** (edges / 2595) - 1)
    fb = np.stack([np.clip(np.minimum((fr - hz[i]) / (hz[i + 1] - hz[i] + 1e-9), (hz[i + 2] - fr) / (hz[i + 2] - hz[i + 1] + 1e-9)), 0, None) for i in range(80)])
    return np.log(S @ fb.T + 1e-5)
A, B = seeded(exp), seeded(ref); vj = json.load(open(f"{d}/voices.json")); vs = [v["id"] for v in vj["voices"]]
L = [x for l in json.load(open(IDS)) for x in l["sents"]][:20]
snr, l1, mism, dlen = [], [], 0, []   # dlen: per mismatched sentence, the length difference in frames (600 samples = 25 ms)
for v in vs:
    st = np.fromfile(f"{d}/voices/{v}.bin", np.float32).reshape(-1, 256); ex = np.fromfile(f"{d}/voices/{v}.extra.bin", np.float32); e, o = {}, 0
    for vi in vj["voice_inputs"]: k = int(np.prod(vi["dims"])); e[vi["name"]] = ex[o:o + k].reshape(vi["dims"]); o += k
    acc = next((x.get("accent", "en-us") for x in vj["voices"] if x["id"] == v), "en-us")
    for k, ids in enumerate(L if not ET else [s["ids"] for s in ET[acc]]):
        f = {"input_ids": np.array([[0] + ids + [0]], np.int64), "style": st[min(len(ids), len(st)) - 1][None], "speed": np.ones(1, np.float32), **e}
        if ET:
            s_ = ET[acc][k]; w = s_["widx"]
            f.update(ettin_ids=np.array([s_["ettin_ids"]], np.int64), ettin_mask=np.ones((1, s_["S"]), np.int64),
                     wavg=np.array(s_["wavg"], np.float32).reshape(1, s_["W"], s_["S"]), widx=np.array([[w[0], *w, w[-1]]], np.int64))
        x, y = A.run(None, f)[0].ravel(), B.run(None, f)[0].ravel()
        if x.size != y.size:   # a one-frame length change also shifts how much seeded noise each graph draws: re-create both sessions
            mism += 1; dlen.append((x.size - y.size) / 600); A, B = seeded(exp), seeded(ref); continue   # so later sentences' noise lines up again (17 Nov)
        err = ((x - y) ** 2).sum(); snr.append(float("inf") if err == 0 else 10 * np.log10((y ** 2).sum() / err))
        l1.append(float(np.abs(logmel(x) - logmel(y)).mean()))
fin = [s for s in snr if np.isfinite(s)]
_m = onnx.load(exp)   # int8 = int8 weights or integer ops anywhere (DequantizeLinear, MatMulInteger, ...)
precision = "int8" if any(i.data_type in (onnx.TensorProto.INT8, onnx.TensorProto.UINT8) for i in _m.graph.initializer) or \
    any(n.op_type in ("DequantizeLinear", "MatMulInteger", "ConvInteger", "DynamicQuantizeLinear") for n in _m.graph.node) else "float"
r = dict(precision=precision, sentences=len(snr) + mism, length_mismatches=mism, snr_median_db=float(np.median(snr)) if snr else None, snr_min_db=float(min(snr)) if snr else None,
         logmel_l1_median=float(np.median(l1)) if l1 else None, identical=len(snr) - len(fin))
r["length_diff_frames"] = dlen
if precision == "float": r["bar"] = "same length on every sentence + median SNR >= 30 dB"; r["pass"] = bool(mism == 0 and snr and np.median(snr) >= 30.0)
else:   # 17 Nov: the length bar catches broken exports; one sentence off by exactly one frame is allowed, a second one or 2+ frames fails
    r["bar"] = "at most 1 sentence per check set off by exactly +-1 frame (25 ms), none by more + median log-mel L1 <= 0.20 (+ whisper_check.py)"
    r["pass"] = bool(mism <= 1 and all(abs(d) == 1.0 for d in dlen) and l1 and np.median(l1) <= 0.20)
print(json.dumps(r)); json.dump(r, open(out, "w"), indent=1) if out else None
sys.exit(0 if r["pass"] else 1)