Perth watermarker as ONNX, with the export script and parity results
Browse files- LICENSE +21 -0
- README.md +51 -0
- export_watermarkers.py +261 -0
- parity.json +42 -0
- perth_detect.onnx +3 -0
- perth_embed.onnx +3 -0
LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
MIT License
|
| 2 |
+
|
| 3 |
+
Copyright (c) 2025 Resemble AI
|
| 4 |
+
|
| 5 |
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
| 6 |
+
of this software and associated documentation files (the "Software"), to deal
|
| 7 |
+
in the Software without restriction, including without limitation the rights
|
| 8 |
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
| 9 |
+
copies of the Software, and to permit persons to whom the Software is
|
| 10 |
+
furnished to do so, subject to the following conditions:
|
| 11 |
+
|
| 12 |
+
The above copyright notice and this permission notice shall be included in all
|
| 13 |
+
copies or substantial portions of the Software.
|
| 14 |
+
|
| 15 |
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
| 16 |
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
| 17 |
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
| 18 |
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
| 19 |
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
| 20 |
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
| 21 |
+
SOFTWARE.
|
README.md
ADDED
|
@@ -0,0 +1,51 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
---
|
| 2 |
+
license: mit
|
| 3 |
+
library_name: onnx
|
| 4 |
+
base_model: resemble-ai/Perth
|
| 5 |
+
tags:
|
| 6 |
+
- audio-watermarking
|
| 7 |
+
- perth
|
| 8 |
+
- onnx
|
| 9 |
+
inference: false
|
| 10 |
+
---
|
| 11 |
+
|
| 12 |
+
# Perth (implicit) watermarker, ONNX
|
| 13 |
+
|
| 14 |
+
An ONNX export of Resemble AI's [Perth](https://github.com/resemble-ai/Perth) implicit watermarker, as shipped in
|
| 15 |
+
the `resemble-perth` 1.0.1 package (checkpoint `perth_net_250000.pth.tar`, SHA-256
|
| 16 |
+
`a15bce457ebc53ce5e6c9c3f11df78cf7ee2bf9cdab0a798902135b4c4027670`). The weights are unchanged. This is not an
|
| 17 |
+
official Resemble AI release.
|
| 18 |
+
|
| 19 |
+
Perth adds an imperceptible watermark to the magnitude spectrum below 2 kHz and detects it from the whole clip.
|
| 20 |
+
|
| 21 |
+
## Files
|
| 22 |
+
|
| 23 |
+
| File | Input | Output |
|
| 24 |
+
| --- | --- | --- |
|
| 25 |
+
| `perth_embed.onnx` | `audio`: float32 `[T]`, mono, 32 kHz | `audio_marked`: float32 `[320 * (T // 320)]` |
|
| 26 |
+
| `perth_detect.onnx` | `audio`: float32 `[T]`, mono, 32 kHz | `score`: float32 `[1]`, 0 to 1 (about 1 when marked) |
|
| 27 |
+
|
| 28 |
+
Both graphs are opset 18 and contain the whole pipeline: STFT (2048 points, hop 320, Hann window), the network,
|
| 29 |
+
and for the embedder the inverse STFT. `torch.istft` does not export to ONNX, so the embedder uses an equivalent
|
| 30 |
+
built from matrix products, reshapes and pads (`OnnxISTFT` in the script; at most 1.5e-6 from `torch.istft`).
|
| 31 |
+
Resample other rates to 32 kHz first.
|
| 32 |
+
|
| 33 |
+
## Parity with the PyTorch original
|
| 34 |
+
|
| 35 |
+
`export_watermarkers.py perth OUT clip.wav ...` exports the graphs and compares them with
|
| 36 |
+
`perth.PerthImplicitWatermarker` (`apply_watermark`, `get_watermark(round=False)`), torch 2.15 on CPU, on two
|
| 37 |
+
speech clips (11.8 s and 4.1 s) resampled to 32 kHz:
|
| 38 |
+
|
| 39 |
+
| | 11.8 s clip | 4.1 s clip |
|
| 40 |
+
| --- | --- | --- |
|
| 41 |
+
| embedded audio, max abs difference (PyTorch vs ONNX) | 5.4e-7 | 5.1e-7 |
|
| 42 |
+
| unmarked clip, score PyTorch / ONNX | 5.1e-5 / 5.3e-5 | 6.5e-5 / 6.4e-5 |
|
| 43 |
+
| marked by PyTorch, score PyTorch / ONNX | 1.0 / 1.0 | 1.0 / 1.0 |
|
| 44 |
+
| marked by ONNX, score PyTorch / ONNX | 1.0 / 1.0 | 1.0 / 1.0 |
|
| 45 |
+
|
| 46 |
+
`parity.json` has the exact numbers.
|
| 47 |
+
|
| 48 |
+
## Licence and attribution
|
| 49 |
+
|
| 50 |
+
MIT, Copyright (c) 2025 Resemble AI (see `LICENSE`, the licence of the `resemble-perth` package that contains the
|
| 51 |
+
weights). The export script is MIT as well.
|
export_watermarkers.py
ADDED
|
@@ -0,0 +1,261 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
"""Export the Perth (Resemble AI) and WavMark watermarkers to ONNX, and check the exports against the PyTorch originals.
|
| 3 |
+
|
| 4 |
+
python export_watermarkers.py perth OUT_DIR
|
| 5 |
+
python export_watermarkers.py wavmark OUT_DIR
|
| 6 |
+
|
| 7 |
+
Each model becomes two graphs that take and return plain audio, with their STFT and inverse STFT inside:
|
| 8 |
+
|
| 9 |
+
* Perth (resemble-perth 1.0.1, implicit model ``perth_net_250000``), 32 kHz mono, any length:
|
| 10 |
+
``perth_embed.onnx`` audio ``[T]`` -> watermarked audio ``[T']`` (T' = T rounded down to the 320-sample hop)
|
| 11 |
+
``perth_detect.onnx`` audio ``[T]`` -> ``score`` ``[1]``: about 1 for a marked clip, 0 for an unmarked one
|
| 12 |
+
* WavMark (``M4869/WavMark``, ``step59000_snr39.99_pesq4.35_BERP_none0.30``), 16 kHz mono, one second at a time:
|
| 13 |
+
``wavmark_embed.onnx`` audio ``[B, 16000]`` and bits ``[B, 32]`` (0/1) -> watermarked audio ``[B, 16000]``
|
| 14 |
+
``wavmark_detect.onnx`` audio ``[B, 16000]`` -> bits ``[B, 32]`` (in -1..1; a bit is 1 from 0.5)
|
| 15 |
+
|
| 16 |
+
torch.istft does not export to ONNX: the graphs use an equivalent made of matrix products, reshapes and pads
|
| 17 |
+
(``OnnxISTFT``, max |difference| to torch.istft about 1.5e-6). The weights are unchanged.
|
| 18 |
+
"""
|
| 19 |
+
|
| 20 |
+
from __future__ import annotations
|
| 21 |
+
|
| 22 |
+
import json
|
| 23 |
+
import math
|
| 24 |
+
import sys
|
| 25 |
+
from pathlib import Path
|
| 26 |
+
|
| 27 |
+
import numpy as np
|
| 28 |
+
import torch
|
| 29 |
+
import torch.nn.functional as F
|
| 30 |
+
|
| 31 |
+
|
| 32 |
+
class OnnxISTFT(torch.nn.Module):
|
| 33 |
+
"""torch.istft(center=True, onesided, periodic Hann window, normalized=False) from exportable operations."""
|
| 34 |
+
|
| 35 |
+
def __init__(self, n_fft: int, hop: int):
|
| 36 |
+
super().__init__()
|
| 37 |
+
self.n_fft, self.hop = n_fft, hop
|
| 38 |
+
self.m = -(-n_fft // hop)
|
| 39 |
+
window = torch.hann_window(n_fft, dtype=torch.float64)
|
| 40 |
+
k = torch.arange(n_fft // 2 + 1, dtype=torch.float64)[:, None]
|
| 41 |
+
n = torch.arange(n_fft, dtype=torch.float64)[None, :]
|
| 42 |
+
scale = torch.full((n_fft // 2 + 1, 1), 2.0, dtype=torch.float64)
|
| 43 |
+
scale[0] = 1.0
|
| 44 |
+
if n_fft % 2 == 0:
|
| 45 |
+
scale[-1] = 1.0
|
| 46 |
+
angle = 2 * math.pi * k * n / n_fft
|
| 47 |
+
pad = self.m * hop - n_fft
|
| 48 |
+
self.register_buffer("cos", F.pad(scale * torch.cos(angle) / n_fft * window, (0, pad)).float())
|
| 49 |
+
self.register_buffer("sin", F.pad(-scale * torch.sin(angle) / n_fft * window, (0, pad)).float())
|
| 50 |
+
self.register_buffer("wsq", F.pad(window**2, (0, pad)).float())
|
| 51 |
+
|
| 52 |
+
def _overlap_add(self, frames: torch.Tensor) -> torch.Tensor:
|
| 53 |
+
b, _, t = frames.shape
|
| 54 |
+
chunks = frames.reshape(b, self.m, self.hop, t).transpose(2, 3).reshape(b, self.m, t * self.hop)
|
| 55 |
+
out = None
|
| 56 |
+
for k in range(self.m):
|
| 57 |
+
piece = F.pad(chunks[:, k], (k * self.hop, (self.m - 1 - k) * self.hop))
|
| 58 |
+
out = piece if out is None else out + piece
|
| 59 |
+
assert out is not None
|
| 60 |
+
return out
|
| 61 |
+
|
| 62 |
+
def forward(self, re: torch.Tensor, im: torch.Tensor) -> torch.Tensor:
|
| 63 |
+
frames = torch.einsum("bft,fn->bnt", re, self.cos) + torch.einsum("bft,fn->bnt", im, self.sin)
|
| 64 |
+
y = self._overlap_add(frames)
|
| 65 |
+
env = self._overlap_add(self.wsq[None, :, None].expand(1, -1, frames.shape[2]))
|
| 66 |
+
half = self.n_fft // 2
|
| 67 |
+
n = self.hop * (frames.shape[2] - 1)
|
| 68 |
+
y = y[:, half : half + n]
|
| 69 |
+
env = env[:, half : half + n]
|
| 70 |
+
return y / torch.where(env > 1e-11, env, torch.ones_like(env))
|
| 71 |
+
|
| 72 |
+
|
| 73 |
+
# -- Perth ------------------------------------------------------------------------------------------------------
|
| 74 |
+
def perth_models():
|
| 75 |
+
from perth.perth_net import PREPACKAGED_MODELS_DIR
|
| 76 |
+
from perth.perth_net.perth_net_implicit.model.perth_net import PerthNet
|
| 77 |
+
from perth.perth_net.perth_net_implicit.utils import denormalize_spectrogram
|
| 78 |
+
|
| 79 |
+
net = PerthNet.load("implicit", PREPACKAGED_MODELS_DIR).eval()
|
| 80 |
+
hp = net.hp
|
| 81 |
+
|
| 82 |
+
class Embed(torch.nn.Module):
|
| 83 |
+
def __init__(self) -> None:
|
| 84 |
+
super().__init__()
|
| 85 |
+
self.net = net
|
| 86 |
+
self.istft = OnnxISTFT(hp.n_fft, hp.hop_size)
|
| 87 |
+
|
| 88 |
+
def forward(self, audio: torch.Tensor) -> torch.Tensor:
|
| 89 |
+
spec = self.net.ap.spectrogram(audio) # (F, T) complex
|
| 90 |
+
mag = 20 * torch.log10(spec.abs().clip(hp.stft_magnitude_min))
|
| 91 |
+
min_db = 20 * math.log10(hp.stft_magnitude_min)
|
| 92 |
+
mag = (mag - min_db) / (-min_db + 15)
|
| 93 |
+
wm, _ = self.net.encoder(mag[None])
|
| 94 |
+
amp = 10.0 ** ((denormalize_spectrogram(hp, wm[0]) / 20).clip(max=10))
|
| 95 |
+
phase = torch.angle(spec)
|
| 96 |
+
return self.istft((amp * torch.cos(phase))[None], (amp * torch.sin(phase))[None])[0]
|
| 97 |
+
|
| 98 |
+
class Detect(torch.nn.Module):
|
| 99 |
+
def __init__(self) -> None:
|
| 100 |
+
super().__init__()
|
| 101 |
+
self.net = net
|
| 102 |
+
|
| 103 |
+
def forward(self, audio: torch.Tensor) -> torch.Tensor:
|
| 104 |
+
mag, _ = self.net.ap.signal_to_magphase(audio)
|
| 105 |
+
return self.net.decoder(mag[None]).clip(0.0, 1.0)
|
| 106 |
+
|
| 107 |
+
return net, Embed().eval(), Detect().eval()
|
| 108 |
+
|
| 109 |
+
|
| 110 |
+
def export_perth(out: Path) -> dict:
|
| 111 |
+
_net, embed, detect = perth_models()
|
| 112 |
+
x = torch.randn(32000 * 3) * 0.1
|
| 113 |
+
t = torch.export.Dim("samples", min=16000, max=32000 * 900)
|
| 114 |
+
torch.onnx.export(embed, (x,), out / "perth_embed.onnx", dynamo=True, dynamic_shapes=({0: t},),
|
| 115 |
+
input_names=["audio"], output_names=["audio_marked"], opset_version=18, external_data=False) # fmt: skip
|
| 116 |
+
torch.onnx.export(detect, (x,), out / "perth_detect.onnx", dynamo=True, dynamic_shapes=({0: t},),
|
| 117 |
+
input_names=["audio"], output_names=["score"], opset_version=18, external_data=False) # fmt: skip
|
| 118 |
+
return {"perth_embed.onnx": "audio [T] at 32 kHz -> audio_marked", "perth_detect.onnx": "audio [T] -> score"}
|
| 119 |
+
|
| 120 |
+
|
| 121 |
+
def parity_perth(out: Path, clips: list[np.ndarray]) -> list[dict]:
|
| 122 |
+
import onnxruntime as ort
|
| 123 |
+
from perth import PerthImplicitWatermarker
|
| 124 |
+
|
| 125 |
+
torch_wm = PerthImplicitWatermarker()
|
| 126 |
+
emb = ort.InferenceSession(str(out / "perth_embed.onnx"), providers=["CPUExecutionProvider"])
|
| 127 |
+
det = ort.InferenceSession(str(out / "perth_detect.onnx"), providers=["CPUExecutionProvider"])
|
| 128 |
+
rows = []
|
| 129 |
+
for x in clips: # 32 kHz
|
| 130 |
+
x = x[: len(x) // 320 * 320].astype(np.float32)
|
| 131 |
+
t_marked = np.asarray(torch_wm.apply_watermark(x, sample_rate=32000), dtype=np.float32)
|
| 132 |
+
o_marked = emb.run(None, {"audio": x})[0]
|
| 133 |
+
n = min(len(t_marked), len(o_marked))
|
| 134 |
+
|
| 135 |
+
def t_score(y: np.ndarray) -> float:
|
| 136 |
+
return float(torch_wm.get_watermark(y, sample_rate=32000, round=False))
|
| 137 |
+
|
| 138 |
+
def o_score(y: np.ndarray) -> float:
|
| 139 |
+
return float(det.run(None, {"audio": y.astype(np.float32)})[0].reshape(-1)[0])
|
| 140 |
+
|
| 141 |
+
rows.append({
|
| 142 |
+
"seconds": round(len(x) / 32000, 2),
|
| 143 |
+
"embed_max_abs_diff": float(np.abs(t_marked[:n] - o_marked[:n]).max()),
|
| 144 |
+
"unmarked": {"torch": t_score(x), "onnx": o_score(x)},
|
| 145 |
+
"marked_by_torch": {"torch": t_score(t_marked), "onnx": o_score(t_marked)},
|
| 146 |
+
"marked_by_onnx": {"torch": t_score(o_marked), "onnx": o_score(o_marked)},
|
| 147 |
+
}) # fmt: skip
|
| 148 |
+
return rows
|
| 149 |
+
|
| 150 |
+
|
| 151 |
+
# -- WavMark ------------------------------------------------------------------------------------------------------
|
| 152 |
+
def wavmark_models():
|
| 153 |
+
import wavmark
|
| 154 |
+
|
| 155 |
+
model = wavmark.load_model().eval()
|
| 156 |
+
|
| 157 |
+
class Embed(torch.nn.Module):
|
| 158 |
+
def __init__(self) -> None:
|
| 159 |
+
super().__init__()
|
| 160 |
+
self.model = model
|
| 161 |
+
self.istft = OnnxISTFT(model.n_fft, model.hop_length)
|
| 162 |
+
|
| 163 |
+
def forward(self, audio: torch.Tensor, bits: torch.Tensor) -> torch.Tensor:
|
| 164 |
+
m = self.model
|
| 165 |
+
signal_fft = m.stft(audio)
|
| 166 |
+
message_fft = m.stft(m.watermark_fc(bits))
|
| 167 |
+
marked_fft, _ = m.enc_dec(signal_fft, message_fft, rev=False)
|
| 168 |
+
return self.istft(marked_fft[..., 0], marked_fft[..., 1])
|
| 169 |
+
|
| 170 |
+
class Detect(torch.nn.Module):
|
| 171 |
+
def __init__(self) -> None:
|
| 172 |
+
super().__init__()
|
| 173 |
+
self.model = model
|
| 174 |
+
self.istft = OnnxISTFT(model.n_fft, model.hop_length)
|
| 175 |
+
|
| 176 |
+
def forward(self, audio: torch.Tensor) -> torch.Tensor:
|
| 177 |
+
m = self.model
|
| 178 |
+
signal_fft = m.stft(audio)
|
| 179 |
+
_, restored = m.enc_dec(signal_fft, signal_fft, rev=True)
|
| 180 |
+
expanded = self.istft(restored[..., 0], restored[..., 1])
|
| 181 |
+
return m.watermark_fc_back(expanded).clamp(-1, 1)
|
| 182 |
+
|
| 183 |
+
return model, Embed().eval(), Detect().eval()
|
| 184 |
+
|
| 185 |
+
|
| 186 |
+
def export_wavmark(out: Path) -> dict:
|
| 187 |
+
_model, embed, detect = wavmark_models()
|
| 188 |
+
x = torch.randn(2, 16000) * 0.1
|
| 189 |
+
bits = torch.randint(0, 2, (2, 32)).float()
|
| 190 |
+
b = torch.export.Dim("batch", min=1, max=256)
|
| 191 |
+
torch.onnx.export(embed, (x, bits), out / "wavmark_embed.onnx", dynamo=True,
|
| 192 |
+
dynamic_shapes=({0: b}, {0: b}), input_names=["audio", "bits"], output_names=["audio_marked"],
|
| 193 |
+
opset_version=18, external_data=False) # fmt: skip
|
| 194 |
+
torch.onnx.export(detect, (x,), out / "wavmark_detect.onnx", dynamo=True, dynamic_shapes=({0: b},),
|
| 195 |
+
input_names=["audio"], output_names=["bits"], opset_version=18, external_data=False) # fmt: skip
|
| 196 |
+
return {"wavmark_embed.onnx": "audio [B, 16000], bits [B, 32] -> audio_marked", "wavmark_detect.onnx": "audio -> bits"}
|
| 197 |
+
|
| 198 |
+
|
| 199 |
+
def parity_wavmark(out: Path, clips: list[np.ndarray]) -> list[dict]:
|
| 200 |
+
import onnxruntime as ort
|
| 201 |
+
|
| 202 |
+
model, _e, _d = wavmark_models()
|
| 203 |
+
emb = ort.InferenceSession(str(out / "wavmark_embed.onnx"), providers=["CPUExecutionProvider"])
|
| 204 |
+
det = ort.InferenceSession(str(out / "wavmark_detect.onnx"), providers=["CPUExecutionProvider"])
|
| 205 |
+
rng = np.random.default_rng(0)
|
| 206 |
+
rows = []
|
| 207 |
+
for x in clips: # 16 kHz
|
| 208 |
+
segs = np.stack([x[i : i + 16000] for i in range(0, len(x) - 16000 + 1, 17600)]).astype(np.float32)
|
| 209 |
+
bits = rng.integers(0, 2, (len(segs), 32)).astype(np.float32)
|
| 210 |
+
with torch.no_grad():
|
| 211 |
+
t_marked = model.encode(torch.from_numpy(segs), torch.from_numpy(bits)).numpy()
|
| 212 |
+
o_marked = emb.run(None, {"audio": segs, "bits": bits})[0]
|
| 213 |
+
|
| 214 |
+
def t_bits(y: np.ndarray) -> np.ndarray:
|
| 215 |
+
with torch.no_grad():
|
| 216 |
+
return model.decode(torch.from_numpy(y.astype(np.float32))).numpy()
|
| 217 |
+
|
| 218 |
+
def o_bits(y: np.ndarray) -> np.ndarray:
|
| 219 |
+
return det.run(None, {"audio": y.astype(np.float32)})[0]
|
| 220 |
+
|
| 221 |
+
def right(found: np.ndarray) -> float:
|
| 222 |
+
return float(((found >= 0.5) == (bits >= 0.5)).mean())
|
| 223 |
+
|
| 224 |
+
rows.append({
|
| 225 |
+
"seconds": round(len(x) / 16000, 2), "segments": len(segs),
|
| 226 |
+
"embed_max_abs_diff": float(np.abs(t_marked - o_marked).max()),
|
| 227 |
+
"detect_max_abs_diff": float(np.abs(t_bits(t_marked) - o_bits(t_marked)).max()),
|
| 228 |
+
"bits_right": {
|
| 229 |
+
"marked_by_torch": {"torch": right(t_bits(t_marked)), "onnx": right(o_bits(t_marked))},
|
| 230 |
+
"marked_by_onnx": {"torch": right(t_bits(o_marked)), "onnx": right(o_bits(o_marked))},
|
| 231 |
+
"unmarked": {"torch": right(t_bits(segs)), "onnx": right(o_bits(segs))},
|
| 232 |
+
},
|
| 233 |
+
}) # fmt: skip
|
| 234 |
+
return rows
|
| 235 |
+
|
| 236 |
+
|
| 237 |
+
def main() -> int:
|
| 238 |
+
import soundfile as sf
|
| 239 |
+
|
| 240 |
+
which, out = sys.argv[1], Path(sys.argv[2])
|
| 241 |
+
out.mkdir(parents=True, exist_ok=True)
|
| 242 |
+
rate = 32000 if which == "perth" else 16000
|
| 243 |
+
clips = []
|
| 244 |
+
for path in sys.argv[3:]:
|
| 245 |
+
x, sr = sf.read(path, dtype="float32", always_2d=True)
|
| 246 |
+
x = x.mean(axis=1)
|
| 247 |
+
if sr != rate:
|
| 248 |
+
import librosa
|
| 249 |
+
|
| 250 |
+
x = librosa.resample(x, orig_sr=sr, target_sr=rate)
|
| 251 |
+
clips.append(x[: rate * 20])
|
| 252 |
+
files = export_perth(out) if which == "perth" else export_wavmark(out)
|
| 253 |
+
parity = parity_perth(out, clips) if which == "perth" else parity_wavmark(out, clips)
|
| 254 |
+
report = {"model": which, "torch": torch.__version__, "files": files, "parity": parity}
|
| 255 |
+
(out / "parity.json").write_text(json.dumps(report, indent=1) + "\n")
|
| 256 |
+
print(json.dumps(report, indent=1))
|
| 257 |
+
return 0
|
| 258 |
+
|
| 259 |
+
|
| 260 |
+
if __name__ == "__main__":
|
| 261 |
+
raise SystemExit(main())
|
parity.json
ADDED
|
@@ -0,0 +1,42 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"model": "perth",
|
| 3 |
+
"torch": "2.15.0+cpu",
|
| 4 |
+
"files": {
|
| 5 |
+
"perth_embed.onnx": "audio [T] at 32 kHz -> audio_marked",
|
| 6 |
+
"perth_detect.onnx": "audio [T] -> score"
|
| 7 |
+
},
|
| 8 |
+
"parity": [
|
| 9 |
+
{
|
| 10 |
+
"seconds": 11.81,
|
| 11 |
+
"embed_max_abs_diff": 5.364418029785156e-07,
|
| 12 |
+
"unmarked": {
|
| 13 |
+
"torch": 5.089864134788513e-05,
|
| 14 |
+
"onnx": 5.329865962266922e-05
|
| 15 |
+
},
|
| 16 |
+
"marked_by_torch": {
|
| 17 |
+
"torch": 1.0,
|
| 18 |
+
"onnx": 1.0
|
| 19 |
+
},
|
| 20 |
+
"marked_by_onnx": {
|
| 21 |
+
"torch": 1.0,
|
| 22 |
+
"onnx": 1.0
|
| 23 |
+
}
|
| 24 |
+
},
|
| 25 |
+
{
|
| 26 |
+
"seconds": 4.13,
|
| 27 |
+
"embed_max_abs_diff": 5.066394805908203e-07,
|
| 28 |
+
"unmarked": {
|
| 29 |
+
"torch": 6.537977606058121e-05,
|
| 30 |
+
"onnx": 6.363634020090103e-05
|
| 31 |
+
},
|
| 32 |
+
"marked_by_torch": {
|
| 33 |
+
"torch": 1.0,
|
| 34 |
+
"onnx": 1.0
|
| 35 |
+
},
|
| 36 |
+
"marked_by_onnx": {
|
| 37 |
+
"torch": 1.0,
|
| 38 |
+
"onnx": 1.0
|
| 39 |
+
}
|
| 40 |
+
}
|
| 41 |
+
]
|
| 42 |
+
}
|
perth_detect.onnx
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:c814c83e17dada320c845dbbbf20baa8d92075853243ba9f86a8ab2bab0effd3
|
| 3 |
+
size 28157039
|
perth_embed.onnx
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:02e3479e4f15d03cf84fccba1b35117b31f2e1d1276d7202ffdaa8f9790c12a8
|
| 3 |
+
size 27984295
|