Download export_watermarkers.py from codebucketsolutions/wavmark-onnx: direct link, hf CLI and curl.
- Browser
- Download file 11.9 kB
-
https://huggingface.co/codebucketsolutions/wavmark-onnx/resolve/main/export_watermarkers.py
- Command line
-
hf download hf://codebucketsolutions/wavmark-onnx/export_watermarkers.py
-
curl -L -o export_watermarkers.py https://huggingface.co/codebucketsolutions/wavmark-onnx/resolve/main/export_watermarkers.py
11.9 kB
| #!/usr/bin/env python3 | |
| """Export the Perth (Resemble AI) and WavMark watermarkers to ONNX, and check the exports against the PyTorch originals. | |
| python export_watermarkers.py perth OUT_DIR | |
| python export_watermarkers.py wavmark OUT_DIR | |
| Each model becomes two graphs that take and return plain audio, with their STFT and inverse STFT inside: | |
| * Perth (resemble-perth 1.0.1, implicit model ``perth_net_250000``), 32 kHz mono, any length: | |
| ``perth_embed.onnx`` audio ``[T]`` -> watermarked audio ``[T']`` (T' = T rounded down to the 320-sample hop) | |
| ``perth_detect.onnx`` audio ``[T]`` -> ``score`` ``[1]``: about 1 for a marked clip, 0 for an unmarked one | |
| * WavMark (``M4869/WavMark``, ``step59000_snr39.99_pesq4.35_BERP_none0.30``), 16 kHz mono, one second at a time: | |
| ``wavmark_embed.onnx`` audio ``[B, 16000]`` and bits ``[B, 32]`` (0/1) -> watermarked audio ``[B, 16000]`` | |
| ``wavmark_detect.onnx`` audio ``[B, 16000]`` -> bits ``[B, 32]`` (in -1..1; a bit is 1 from 0.5) | |
| torch.istft does not export to ONNX: the graphs use an equivalent made of matrix products, reshapes and pads | |
| (``OnnxISTFT``, max |difference| to torch.istft about 1.5e-6). The weights are unchanged. | |
| """ | |
| from __future__ import annotations | |
| import json | |
| import math | |
| import sys | |
| from pathlib import Path | |
| import numpy as np | |
| import torch | |
| import torch.nn.functional as F | |
| class OnnxISTFT(torch.nn.Module): | |
| """torch.istft(center=True, onesided, periodic Hann window, normalized=False) from exportable operations.""" | |
| def __init__(self, n_fft: int, hop: int): | |
| super().__init__() | |
| self.n_fft, self.hop = n_fft, hop | |
| self.m = -(-n_fft // hop) | |
| window = torch.hann_window(n_fft, dtype=torch.float64) | |
| k = torch.arange(n_fft // 2 + 1, dtype=torch.float64)[:, None] | |
| n = torch.arange(n_fft, dtype=torch.float64)[None, :] | |
| scale = torch.full((n_fft // 2 + 1, 1), 2.0, dtype=torch.float64) | |
| scale[0] = 1.0 | |
| if n_fft % 2 == 0: | |
| scale[-1] = 1.0 | |
| angle = 2 * math.pi * k * n / n_fft | |
| pad = self.m * hop - n_fft | |
| self.register_buffer("cos", F.pad(scale * torch.cos(angle) / n_fft * window, (0, pad)).float()) | |
| self.register_buffer("sin", F.pad(-scale * torch.sin(angle) / n_fft * window, (0, pad)).float()) | |
| self.register_buffer("wsq", F.pad(window**2, (0, pad)).float()) | |
| def _overlap_add(self, frames: torch.Tensor) -> torch.Tensor: | |
| b, _, t = frames.shape | |
| chunks = frames.reshape(b, self.m, self.hop, t).transpose(2, 3).reshape(b, self.m, t * self.hop) | |
| out = None | |
| for k in range(self.m): | |
| piece = F.pad(chunks[:, k], (k * self.hop, (self.m - 1 - k) * self.hop)) | |
| out = piece if out is None else out + piece | |
| assert out is not None | |
| return out | |
| def forward(self, re: torch.Tensor, im: torch.Tensor) -> torch.Tensor: | |
| frames = torch.einsum("bft,fn->bnt", re, self.cos) + torch.einsum("bft,fn->bnt", im, self.sin) | |
| y = self._overlap_add(frames) | |
| env = self._overlap_add(self.wsq[None, :, None].expand(1, -1, frames.shape[2])) | |
| half = self.n_fft // 2 | |
| n = self.hop * (frames.shape[2] - 1) | |
| y = y[:, half : half + n] | |
| env = env[:, half : half + n] | |
| return y / torch.where(env > 1e-11, env, torch.ones_like(env)) | |
| # -- Perth ------------------------------------------------------------------------------------------------------ | |
| def perth_models(): | |
| from perth.perth_net import PREPACKAGED_MODELS_DIR | |
| from perth.perth_net.perth_net_implicit.model.perth_net import PerthNet | |
| from perth.perth_net.perth_net_implicit.utils import denormalize_spectrogram | |
| net = PerthNet.load("implicit", PREPACKAGED_MODELS_DIR).eval() | |
| hp = net.hp | |
| class Embed(torch.nn.Module): | |
| def __init__(self) -> None: | |
| super().__init__() | |
| self.net = net | |
| self.istft = OnnxISTFT(hp.n_fft, hp.hop_size) | |
| def forward(self, audio: torch.Tensor) -> torch.Tensor: | |
| spec = self.net.ap.spectrogram(audio) # (F, T) complex | |
| mag = 20 * torch.log10(spec.abs().clip(hp.stft_magnitude_min)) | |
| min_db = 20 * math.log10(hp.stft_magnitude_min) | |
| mag = (mag - min_db) / (-min_db + 15) | |
| wm, _ = self.net.encoder(mag[None]) | |
| amp = 10.0 ** ((denormalize_spectrogram(hp, wm[0]) / 20).clip(max=10)) | |
| phase = torch.angle(spec) | |
| return self.istft((amp * torch.cos(phase))[None], (amp * torch.sin(phase))[None])[0] | |
| class Detect(torch.nn.Module): | |
| def __init__(self) -> None: | |
| super().__init__() | |
| self.net = net | |
| def forward(self, audio: torch.Tensor) -> torch.Tensor: | |
| mag, _ = self.net.ap.signal_to_magphase(audio) | |
| return self.net.decoder(mag[None]).clip(0.0, 1.0) | |
| return net, Embed().eval(), Detect().eval() | |
| def export_perth(out: Path) -> dict: | |
| _net, embed, detect = perth_models() | |
| x = torch.randn(32000 * 3) * 0.1 | |
| t = torch.export.Dim("samples", min=16000, max=32000 * 900) | |
| torch.onnx.export(embed, (x,), out / "perth_embed.onnx", dynamo=True, dynamic_shapes=({0: t},), | |
| input_names=["audio"], output_names=["audio_marked"], opset_version=18, external_data=False) # fmt: skip | |
| torch.onnx.export(detect, (x,), out / "perth_detect.onnx", dynamo=True, dynamic_shapes=({0: t},), | |
| input_names=["audio"], output_names=["score"], opset_version=18, external_data=False) # fmt: skip | |
| return {"perth_embed.onnx": "audio [T] at 32 kHz -> audio_marked", "perth_detect.onnx": "audio [T] -> score"} | |
| def parity_perth(out: Path, clips: list[np.ndarray]) -> list[dict]: | |
| import onnxruntime as ort | |
| from perth import PerthImplicitWatermarker | |
| torch_wm = PerthImplicitWatermarker() | |
| emb = ort.InferenceSession(str(out / "perth_embed.onnx"), providers=["CPUExecutionProvider"]) | |
| det = ort.InferenceSession(str(out / "perth_detect.onnx"), providers=["CPUExecutionProvider"]) | |
| rows = [] | |
| for x in clips: # 32 kHz | |
| x = x[: len(x) // 320 * 320].astype(np.float32) | |
| t_marked = np.asarray(torch_wm.apply_watermark(x, sample_rate=32000), dtype=np.float32) | |
| o_marked = emb.run(None, {"audio": x})[0] | |
| n = min(len(t_marked), len(o_marked)) | |
| def t_score(y: np.ndarray) -> float: | |
| return float(torch_wm.get_watermark(y, sample_rate=32000, round=False)) | |
| def o_score(y: np.ndarray) -> float: | |
| return float(det.run(None, {"audio": y.astype(np.float32)})[0].reshape(-1)[0]) | |
| rows.append({ | |
| "seconds": round(len(x) / 32000, 2), | |
| "embed_max_abs_diff": float(np.abs(t_marked[:n] - o_marked[:n]).max()), | |
| "unmarked": {"torch": t_score(x), "onnx": o_score(x)}, | |
| "marked_by_torch": {"torch": t_score(t_marked), "onnx": o_score(t_marked)}, | |
| "marked_by_onnx": {"torch": t_score(o_marked), "onnx": o_score(o_marked)}, | |
| }) # fmt: skip | |
| return rows | |
| # -- WavMark ------------------------------------------------------------------------------------------------------ | |
| def wavmark_models(): | |
| import wavmark | |
| model = wavmark.load_model().eval() | |
| class Embed(torch.nn.Module): | |
| def __init__(self) -> None: | |
| super().__init__() | |
| self.model = model | |
| self.istft = OnnxISTFT(model.n_fft, model.hop_length) | |
| def forward(self, audio: torch.Tensor, bits: torch.Tensor) -> torch.Tensor: | |
| m = self.model | |
| signal_fft = m.stft(audio) | |
| message_fft = m.stft(m.watermark_fc(bits)) | |
| marked_fft, _ = m.enc_dec(signal_fft, message_fft, rev=False) | |
| return self.istft(marked_fft[..., 0], marked_fft[..., 1]) | |
| class Detect(torch.nn.Module): | |
| def __init__(self) -> None: | |
| super().__init__() | |
| self.model = model | |
| self.istft = OnnxISTFT(model.n_fft, model.hop_length) | |
| def forward(self, audio: torch.Tensor) -> torch.Tensor: | |
| m = self.model | |
| signal_fft = m.stft(audio) | |
| _, restored = m.enc_dec(signal_fft, signal_fft, rev=True) | |
| expanded = self.istft(restored[..., 0], restored[..., 1]) | |
| return m.watermark_fc_back(expanded).clamp(-1, 1) | |
| return model, Embed().eval(), Detect().eval() | |
| def export_wavmark(out: Path) -> dict: | |
| _model, embed, detect = wavmark_models() | |
| x = torch.randn(2, 16000) * 0.1 | |
| bits = torch.randint(0, 2, (2, 32)).float() | |
| b = torch.export.Dim("batch", min=1, max=256) | |
| torch.onnx.export(embed, (x, bits), out / "wavmark_embed.onnx", dynamo=True, | |
| dynamic_shapes=({0: b}, {0: b}), input_names=["audio", "bits"], output_names=["audio_marked"], | |
| opset_version=18, external_data=False) # fmt: skip | |
| torch.onnx.export(detect, (x,), out / "wavmark_detect.onnx", dynamo=True, dynamic_shapes=({0: b},), | |
| input_names=["audio"], output_names=["bits"], opset_version=18, external_data=False) # fmt: skip | |
| return {"wavmark_embed.onnx": "audio [B, 16000], bits [B, 32] -> audio_marked", "wavmark_detect.onnx": "audio -> bits"} | |
| def parity_wavmark(out: Path, clips: list[np.ndarray]) -> list[dict]: | |
| import onnxruntime as ort | |
| model, _e, _d = wavmark_models() | |
| emb = ort.InferenceSession(str(out / "wavmark_embed.onnx"), providers=["CPUExecutionProvider"]) | |
| det = ort.InferenceSession(str(out / "wavmark_detect.onnx"), providers=["CPUExecutionProvider"]) | |
| rng = np.random.default_rng(0) | |
| rows = [] | |
| for x in clips: # 16 kHz | |
| segs = np.stack([x[i : i + 16000] for i in range(0, len(x) - 16000 + 1, 17600)]).astype(np.float32) | |
| bits = rng.integers(0, 2, (len(segs), 32)).astype(np.float32) | |
| with torch.no_grad(): | |
| t_marked = model.encode(torch.from_numpy(segs), torch.from_numpy(bits)).numpy() | |
| o_marked = emb.run(None, {"audio": segs, "bits": bits})[0] | |
| def t_bits(y: np.ndarray) -> np.ndarray: | |
| with torch.no_grad(): | |
| return model.decode(torch.from_numpy(y.astype(np.float32))).numpy() | |
| def o_bits(y: np.ndarray) -> np.ndarray: | |
| return det.run(None, {"audio": y.astype(np.float32)})[0] | |
| def right(found: np.ndarray) -> float: | |
| return float(((found >= 0.5) == (bits >= 0.5)).mean()) | |
| rows.append({ | |
| "seconds": round(len(x) / 16000, 2), "segments": len(segs), | |
| "embed_max_abs_diff": float(np.abs(t_marked - o_marked).max()), | |
| "detect_max_abs_diff": float(np.abs(t_bits(t_marked) - o_bits(t_marked)).max()), | |
| "bits_right": { | |
| "marked_by_torch": {"torch": right(t_bits(t_marked)), "onnx": right(o_bits(t_marked))}, | |
| "marked_by_onnx": {"torch": right(t_bits(o_marked)), "onnx": right(o_bits(o_marked))}, | |
| "unmarked": {"torch": right(t_bits(segs)), "onnx": right(o_bits(segs))}, | |
| }, | |
| }) # fmt: skip | |
| return rows | |
| def main() -> int: | |
| import soundfile as sf | |
| which, out = sys.argv[1], Path(sys.argv[2]) | |
| out.mkdir(parents=True, exist_ok=True) | |
| rate = 32000 if which == "perth" else 16000 | |
| clips = [] | |
| for path in sys.argv[3:]: | |
| x, sr = sf.read(path, dtype="float32", always_2d=True) | |
| x = x.mean(axis=1) | |
| if sr != rate: | |
| import librosa | |
| x = librosa.resample(x, orig_sr=sr, target_sr=rate) | |
| clips.append(x[: rate * 20]) | |
| files = export_perth(out) if which == "perth" else export_wavmark(out) | |
| parity = parity_perth(out, clips) if which == "perth" else parity_wavmark(out, clips) | |
| report = {"model": which, "torch": torch.__version__, "files": files, "parity": parity} | |
| (out / "parity.json").write_text(json.dumps(report, indent=1) + "\n") | |
| print(json.dumps(report, indent=1)) | |
| return 0 | |
| if __name__ == "__main__": | |
| raise SystemExit(main()) | |