ONNX
audio-watermarking
wavmark
wavmark-onnx / export_watermarkers.py
gotham13's picture
Wavmark watermarker as ONNX, with the export script and parity results
4eb7998 verified
Raw History Blame Contribute Delete
11.9 kB
#!/usr/bin/env python3
"""Export the Perth (Resemble AI) and WavMark watermarkers to ONNX, and check the exports against the PyTorch originals.
python export_watermarkers.py perth OUT_DIR
python export_watermarkers.py wavmark OUT_DIR
Each model becomes two graphs that take and return plain audio, with their STFT and inverse STFT inside:
* Perth (resemble-perth 1.0.1, implicit model ``perth_net_250000``), 32 kHz mono, any length:
``perth_embed.onnx`` audio ``[T]`` -> watermarked audio ``[T']`` (T' = T rounded down to the 320-sample hop)
``perth_detect.onnx`` audio ``[T]`` -> ``score`` ``[1]``: about 1 for a marked clip, 0 for an unmarked one
* WavMark (``M4869/WavMark``, ``step59000_snr39.99_pesq4.35_BERP_none0.30``), 16 kHz mono, one second at a time:
``wavmark_embed.onnx`` audio ``[B, 16000]`` and bits ``[B, 32]`` (0/1) -> watermarked audio ``[B, 16000]``
``wavmark_detect.onnx`` audio ``[B, 16000]`` -> bits ``[B, 32]`` (in -1..1; a bit is 1 from 0.5)
torch.istft does not export to ONNX: the graphs use an equivalent made of matrix products, reshapes and pads
(``OnnxISTFT``, max |difference| to torch.istft about 1.5e-6). The weights are unchanged.
"""
from __future__ import annotations
import json
import math
import sys
from pathlib import Path
import numpy as np
import torch
import torch.nn.functional as F
class OnnxISTFT(torch.nn.Module):
"""torch.istft(center=True, onesided, periodic Hann window, normalized=False) from exportable operations."""
def __init__(self, n_fft: int, hop: int):
super().__init__()
self.n_fft, self.hop = n_fft, hop
self.m = -(-n_fft // hop)
window = torch.hann_window(n_fft, dtype=torch.float64)
k = torch.arange(n_fft // 2 + 1, dtype=torch.float64)[:, None]
n = torch.arange(n_fft, dtype=torch.float64)[None, :]
scale = torch.full((n_fft // 2 + 1, 1), 2.0, dtype=torch.float64)
scale[0] = 1.0
if n_fft % 2 == 0:
scale[-1] = 1.0
angle = 2 * math.pi * k * n / n_fft
pad = self.m * hop - n_fft
self.register_buffer("cos", F.pad(scale * torch.cos(angle) / n_fft * window, (0, pad)).float())
self.register_buffer("sin", F.pad(-scale * torch.sin(angle) / n_fft * window, (0, pad)).float())
self.register_buffer("wsq", F.pad(window**2, (0, pad)).float())
def _overlap_add(self, frames: torch.Tensor) -> torch.Tensor:
b, _, t = frames.shape
chunks = frames.reshape(b, self.m, self.hop, t).transpose(2, 3).reshape(b, self.m, t * self.hop)
out = None
for k in range(self.m):
piece = F.pad(chunks[:, k], (k * self.hop, (self.m - 1 - k) * self.hop))
out = piece if out is None else out + piece
assert out is not None
return out
def forward(self, re: torch.Tensor, im: torch.Tensor) -> torch.Tensor:
frames = torch.einsum("bft,fn->bnt", re, self.cos) + torch.einsum("bft,fn->bnt", im, self.sin)
y = self._overlap_add(frames)
env = self._overlap_add(self.wsq[None, :, None].expand(1, -1, frames.shape[2]))
half = self.n_fft // 2
n = self.hop * (frames.shape[2] - 1)
y = y[:, half : half + n]
env = env[:, half : half + n]
return y / torch.where(env > 1e-11, env, torch.ones_like(env))
# -- Perth ------------------------------------------------------------------------------------------------------
def perth_models():
from perth.perth_net import PREPACKAGED_MODELS_DIR
from perth.perth_net.perth_net_implicit.model.perth_net import PerthNet
from perth.perth_net.perth_net_implicit.utils import denormalize_spectrogram
net = PerthNet.load("implicit", PREPACKAGED_MODELS_DIR).eval()
hp = net.hp
class Embed(torch.nn.Module):
def __init__(self) -> None:
super().__init__()
self.net = net
self.istft = OnnxISTFT(hp.n_fft, hp.hop_size)
def forward(self, audio: torch.Tensor) -> torch.Tensor:
spec = self.net.ap.spectrogram(audio) # (F, T) complex
mag = 20 * torch.log10(spec.abs().clip(hp.stft_magnitude_min))
min_db = 20 * math.log10(hp.stft_magnitude_min)
mag = (mag - min_db) / (-min_db + 15)
wm, _ = self.net.encoder(mag[None])
amp = 10.0 ** ((denormalize_spectrogram(hp, wm[0]) / 20).clip(max=10))
phase = torch.angle(spec)
return self.istft((amp * torch.cos(phase))[None], (amp * torch.sin(phase))[None])[0]
class Detect(torch.nn.Module):
def __init__(self) -> None:
super().__init__()
self.net = net
def forward(self, audio: torch.Tensor) -> torch.Tensor:
mag, _ = self.net.ap.signal_to_magphase(audio)
return self.net.decoder(mag[None]).clip(0.0, 1.0)
return net, Embed().eval(), Detect().eval()
def export_perth(out: Path) -> dict:
_net, embed, detect = perth_models()
x = torch.randn(32000 * 3) * 0.1
t = torch.export.Dim("samples", min=16000, max=32000 * 900)
torch.onnx.export(embed, (x,), out / "perth_embed.onnx", dynamo=True, dynamic_shapes=({0: t},),
input_names=["audio"], output_names=["audio_marked"], opset_version=18, external_data=False) # fmt: skip
torch.onnx.export(detect, (x,), out / "perth_detect.onnx", dynamo=True, dynamic_shapes=({0: t},),
input_names=["audio"], output_names=["score"], opset_version=18, external_data=False) # fmt: skip
return {"perth_embed.onnx": "audio [T] at 32 kHz -> audio_marked", "perth_detect.onnx": "audio [T] -> score"}
def parity_perth(out: Path, clips: list[np.ndarray]) -> list[dict]:
import onnxruntime as ort
from perth import PerthImplicitWatermarker
torch_wm = PerthImplicitWatermarker()
emb = ort.InferenceSession(str(out / "perth_embed.onnx"), providers=["CPUExecutionProvider"])
det = ort.InferenceSession(str(out / "perth_detect.onnx"), providers=["CPUExecutionProvider"])
rows = []
for x in clips: # 32 kHz
x = x[: len(x) // 320 * 320].astype(np.float32)
t_marked = np.asarray(torch_wm.apply_watermark(x, sample_rate=32000), dtype=np.float32)
o_marked = emb.run(None, {"audio": x})[0]
n = min(len(t_marked), len(o_marked))
def t_score(y: np.ndarray) -> float:
return float(torch_wm.get_watermark(y, sample_rate=32000, round=False))
def o_score(y: np.ndarray) -> float:
return float(det.run(None, {"audio": y.astype(np.float32)})[0].reshape(-1)[0])
rows.append({
"seconds": round(len(x) / 32000, 2),
"embed_max_abs_diff": float(np.abs(t_marked[:n] - o_marked[:n]).max()),
"unmarked": {"torch": t_score(x), "onnx": o_score(x)},
"marked_by_torch": {"torch": t_score(t_marked), "onnx": o_score(t_marked)},
"marked_by_onnx": {"torch": t_score(o_marked), "onnx": o_score(o_marked)},
}) # fmt: skip
return rows
# -- WavMark ------------------------------------------------------------------------------------------------------
def wavmark_models():
import wavmark
model = wavmark.load_model().eval()
class Embed(torch.nn.Module):
def __init__(self) -> None:
super().__init__()
self.model = model
self.istft = OnnxISTFT(model.n_fft, model.hop_length)
def forward(self, audio: torch.Tensor, bits: torch.Tensor) -> torch.Tensor:
m = self.model
signal_fft = m.stft(audio)
message_fft = m.stft(m.watermark_fc(bits))
marked_fft, _ = m.enc_dec(signal_fft, message_fft, rev=False)
return self.istft(marked_fft[..., 0], marked_fft[..., 1])
class Detect(torch.nn.Module):
def __init__(self) -> None:
super().__init__()
self.model = model
self.istft = OnnxISTFT(model.n_fft, model.hop_length)
def forward(self, audio: torch.Tensor) -> torch.Tensor:
m = self.model
signal_fft = m.stft(audio)
_, restored = m.enc_dec(signal_fft, signal_fft, rev=True)
expanded = self.istft(restored[..., 0], restored[..., 1])
return m.watermark_fc_back(expanded).clamp(-1, 1)
return model, Embed().eval(), Detect().eval()
def export_wavmark(out: Path) -> dict:
_model, embed, detect = wavmark_models()
x = torch.randn(2, 16000) * 0.1
bits = torch.randint(0, 2, (2, 32)).float()
b = torch.export.Dim("batch", min=1, max=256)
torch.onnx.export(embed, (x, bits), out / "wavmark_embed.onnx", dynamo=True,
dynamic_shapes=({0: b}, {0: b}), input_names=["audio", "bits"], output_names=["audio_marked"],
opset_version=18, external_data=False) # fmt: skip
torch.onnx.export(detect, (x,), out / "wavmark_detect.onnx", dynamo=True, dynamic_shapes=({0: b},),
input_names=["audio"], output_names=["bits"], opset_version=18, external_data=False) # fmt: skip
return {"wavmark_embed.onnx": "audio [B, 16000], bits [B, 32] -> audio_marked", "wavmark_detect.onnx": "audio -> bits"}
def parity_wavmark(out: Path, clips: list[np.ndarray]) -> list[dict]:
import onnxruntime as ort
model, _e, _d = wavmark_models()
emb = ort.InferenceSession(str(out / "wavmark_embed.onnx"), providers=["CPUExecutionProvider"])
det = ort.InferenceSession(str(out / "wavmark_detect.onnx"), providers=["CPUExecutionProvider"])
rng = np.random.default_rng(0)
rows = []
for x in clips: # 16 kHz
segs = np.stack([x[i : i + 16000] for i in range(0, len(x) - 16000 + 1, 17600)]).astype(np.float32)
bits = rng.integers(0, 2, (len(segs), 32)).astype(np.float32)
with torch.no_grad():
t_marked = model.encode(torch.from_numpy(segs), torch.from_numpy(bits)).numpy()
o_marked = emb.run(None, {"audio": segs, "bits": bits})[0]
def t_bits(y: np.ndarray) -> np.ndarray:
with torch.no_grad():
return model.decode(torch.from_numpy(y.astype(np.float32))).numpy()
def o_bits(y: np.ndarray) -> np.ndarray:
return det.run(None, {"audio": y.astype(np.float32)})[0]
def right(found: np.ndarray) -> float:
return float(((found >= 0.5) == (bits >= 0.5)).mean())
rows.append({
"seconds": round(len(x) / 16000, 2), "segments": len(segs),
"embed_max_abs_diff": float(np.abs(t_marked - o_marked).max()),
"detect_max_abs_diff": float(np.abs(t_bits(t_marked) - o_bits(t_marked)).max()),
"bits_right": {
"marked_by_torch": {"torch": right(t_bits(t_marked)), "onnx": right(o_bits(t_marked))},
"marked_by_onnx": {"torch": right(t_bits(o_marked)), "onnx": right(o_bits(o_marked))},
"unmarked": {"torch": right(t_bits(segs)), "onnx": right(o_bits(segs))},
},
}) # fmt: skip
return rows
def main() -> int:
import soundfile as sf
which, out = sys.argv[1], Path(sys.argv[2])
out.mkdir(parents=True, exist_ok=True)
rate = 32000 if which == "perth" else 16000
clips = []
for path in sys.argv[3:]:
x, sr = sf.read(path, dtype="float32", always_2d=True)
x = x.mean(axis=1)
if sr != rate:
import librosa
x = librosa.resample(x, orig_sr=sr, target_sr=rate)
clips.append(x[: rate * 20])
files = export_perth(out) if which == "perth" else export_wavmark(out)
parity = parity_perth(out, clips) if which == "perth" else parity_wavmark(out, clips)
report = {"model": which, "torch": torch.__version__, "files": files, "parity": parity}
(out / "parity.json").write_text(json.dumps(report, indent=1) + "\n")
print(json.dumps(report, indent=1))
return 0
if __name__ == "__main__":
raise SystemExit(main())