Download tools/export_muscriptor_onnx.py from fpp-ableton/muscriptor-onnx: direct link, hf CLI and curl.
- Browser
- Download file 7.61 kB
-
https://huggingface.co/fpp-ableton/muscriptor-onnx/resolve/main/tools/export_muscriptor_onnx.py
- Command line
-
hf download hf://fpp-ableton/muscriptor-onnx/tools/export_muscriptor_onnx.py
-
curl -L -o export_muscriptor_onnx.py https://huggingface.co/fpp-ableton/muscriptor-onnx/resolve/main/tools/export_muscriptor_onnx.py
7.61 kB
| #!/usr/bin/env python3 | |
| """Export MuScriptor (small / medium) to two ONNX graphs for onnxruntime-node. | |
| Dev-time tool only; nothing here ships inside the extension. | |
| cond.onnx audio f32[1,80000] (5 s @ 16 kHz) + instrument i64[1,K] + dataset i64[1,1] | |
| -> prefix f32[1, 501+1+K, dim] | |
| The log-mel front end (reflect pad, STFT, mel bank, log) runs inside the graph | |
| using the *checkpoint's own* window and filterbank buffers, which differ from the | |
| textbook formulas by ~2.4e-4 and matter to the model. | |
| instrument ids are raw group ids, -1 = unconditional (same for dataset). | |
| lm.onnx tokens i64[1,S] + prefix f32[1,P,dim] + past_k/past_v f32[L,1,T,H,Dh] | |
| -> logits f32[1,card] (last position only) + present_k/present_v | |
| Explicit KV cache; positions continue from T. One graph serves prefill and decode. | |
| Usage (from a venv with torch, onnx, onnxruntime, einops, safetensors, mido, soundfile): | |
| python export_muscriptor_onnx.py --size small --checkpoint ckpt-small/model.safetensors \ | |
| --reference /path/to/muscriptor/repo --out out/small | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import math | |
| import sys | |
| from pathlib import Path | |
| import numpy as np | |
| import torch | |
| import torch.nn.functional as F | |
| from torch import nn | |
| SEGMENT = 80_000 | |
| N_FFT = 2048 | |
| HOP = 160 | |
| N_FREQ = N_FFT // 2 + 1 | |
| FRAMES = SEGMENT // HOP + 1 # 501 | |
| class Conditioner(nn.Module): | |
| def __init__(self, lm): | |
| super().__init__() | |
| cps = lm.condition_provider.conditioners | |
| wav = cps["self_wav"] | |
| self.proj = wav.output_proj | |
| self.inst_embed = cps["instrument_group"].embed | |
| self.data_embed = cps["dataset_name"].embed | |
| self.eps = wav.eps | |
| window = wav.mel_spec_transform.spectrogram.window.detach().double() | |
| fb = wav.mel_spec_transform.mel_scale.fb.detach().clone() | |
| n = torch.arange(N_FFT, dtype=torch.float64) | |
| k = torch.arange(N_FREQ, dtype=torch.float64) | |
| ang = 2 * math.pi * k[:, None] * n[None, :] / N_FFT | |
| kernel = torch.cat([window * torch.cos(ang), -window * torch.sin(ang)], 0) # [2*N_FREQ, N_FFT] | |
| self.register_buffer("dft", kernel.float().unsqueeze(1)) # [2*N_FREQ, 1, N_FFT] | |
| self.register_buffer("fb", fb) # [N_FREQ, n_mels] | |
| def forward(self, audio, instrument, dataset): | |
| x = F.pad(audio.unsqueeze(1), (N_FFT // 2, N_FFT // 2), mode="reflect") | |
| spec = F.conv1d(x, self.dft, stride=HOP) # [1, 2*N_FREQ, FRAMES] | |
| re, im = spec[:, :N_FREQ], spec[:, N_FREQ:] | |
| mag = torch.sqrt(re * re + im * im) | |
| mel = torch.matmul(mag.transpose(1, 2), self.fb) # [1, FRAMES, n_mels] | |
| emb = self.proj(torch.log(mel + self.eps)) | |
| # The reference masks frames >= length/hop (500 of 501): the last frame is zeroed. | |
| keep = (torch.arange(FRAMES) < SEGMENT // HOP).to(emb.dtype).view(1, -1, 1) | |
| emb = emb * keep | |
| # ClassConditioner: embedding row = id + 2 (id -1 -> row 1 = "unconditional") | |
| inst = self.inst_embed(instrument + 2) | |
| data = self.data_embed(dataset + 2) | |
| # Order in the LM input is [wav, dataset, instrument] | |
| return torch.cat([emb, data, inst], dim=1) | |
| class Decoder(nn.Module): | |
| def __init__(self, lm, heads: int): | |
| super().__init__() | |
| self.lm = lm | |
| self.heads = heads | |
| def forward(self, tokens, prefix, past_k, past_v): | |
| lm = self.lm | |
| x = lm.emb(tokens) | |
| x = torch.cat([prefix.to(x.dtype), x], dim=1) | |
| n_new = x.shape[1] | |
| past = past_k.shape[2] | |
| dim = x.shape[2] | |
| h = self.heads | |
| dh = dim // h | |
| pos = (torch.arange(n_new) + past).view(1, -1, 1).float() | |
| half = dim // 2 | |
| adim = torch.arange(half, dtype=torch.float32).view(1, 1, -1) | |
| phase = pos / (10000.0 ** (adim / (half - 1))) | |
| x = x + torch.cat([torch.cos(phase), torch.sin(phase)], dim=-1) | |
| q_idx = (torch.arange(n_new) + past).view(-1, 1) | |
| k_idx = torch.arange(past + n_new).view(1, -1) | |
| blocked = k_idx > q_idx # [n_new, past+n_new] | |
| ks, vs = [], [] | |
| for i, layer in enumerate(lm.transformer.layers): | |
| attn = layer.self_attn | |
| proj = F.linear(layer.norm1(x), attn.in_proj_weight).view(1, n_new, 3, h, dh) | |
| q, k, v = proj[:, :, 0], proj[:, :, 1], proj[:, :, 2] | |
| k_all = torch.cat([past_k[i], k], dim=1) # [1, T', h, dh] | |
| v_all = torch.cat([past_v[i], v], dim=1) | |
| ks.append(k_all) | |
| vs.append(v_all) | |
| scores = torch.matmul(q.transpose(1, 2), k_all.permute(0, 2, 3, 1)) / math.sqrt(dh) | |
| scores = scores.masked_fill(blocked, float("-inf")) | |
| out = torch.matmul(torch.softmax(scores, dim=-1), v_all.transpose(1, 2)) | |
| out = out.transpose(1, 2).reshape(1, n_new, dim) | |
| x = x + attn.out_proj(out) | |
| x = x + layer.linear2(F.gelu(layer.linear1(layer.norm2(x)))) | |
| last = lm.out_norm(x[:, -1:]) | |
| logits = lm.linear(last)[:, 0] | |
| return logits, torch.stack(ks), torch.stack(vs) | |
| def load_reference(reference: Path, checkpoint: Path): | |
| sys.path.insert(0, str(reference)) | |
| import muscriptor.transcription_model as tm # noqa: WPS433 | |
| model = tm.TranscriptionModel.load_model(str(checkpoint), device="cpu", dtype="float32") | |
| return tm, model | |
| def main() -> None: | |
| ap = argparse.ArgumentParser() | |
| ap.add_argument("--size", choices=["small", "medium"], required=True) | |
| ap.add_argument("--checkpoint", type=Path, required=True) | |
| ap.add_argument("--reference", type=Path, required=True, help="clone of github.com/muscriptor/muscriptor") | |
| ap.add_argument("--out", type=Path, required=True) | |
| ap.add_argument("--opset", type=int, default=17) | |
| args = ap.parse_args() | |
| args.out.mkdir(parents=True, exist_ok=True) | |
| tm, ref = load_reference(args.reference, args.checkpoint) | |
| lm = ref._model | |
| cfg = tm._CONFIGS[args.size] | |
| layers, heads, dim = cfg.num_layers, cfg.num_heads, cfg.dim | |
| dh = dim // heads | |
| lm.eval() | |
| cond = Conditioner(lm).eval() | |
| dec = Decoder(lm, heads).eval() | |
| # ---- export conditioner ------------------------------------------------- | |
| audio = torch.randn(1, SEGMENT) * 0.1 | |
| inst = torch.tensor([[-1]]) | |
| dset = torch.tensor([[-1]]) | |
| cond_path = args.out / "cond.onnx" | |
| torch.onnx.export( | |
| cond, (audio, inst, dset), str(cond_path), opset_version=args.opset, dynamo=False, | |
| input_names=["audio", "instrument", "dataset"], output_names=["prefix"], | |
| dynamic_axes={"instrument": {1: "k"}, "prefix": {1: "p"}}, | |
| ) | |
| # ---- export decoder ----------------------------------------------------- | |
| n_prefix = FRAMES + 1 + 1 | |
| tokens = torch.tensor([[card_initial(lm), 5, 6]]) | |
| prefix = torch.randn(1, n_prefix, dim) | |
| pk = torch.randn(layers, 1, 4, heads, dh) * 0.1 | |
| pv = torch.randn(layers, 1, 4, heads, dh) * 0.1 | |
| lm_path = args.out / "lm.onnx" | |
| torch.onnx.export( | |
| dec, (tokens, prefix, pk, pv), str(lm_path), opset_version=args.opset, dynamo=False, | |
| input_names=["tokens", "prefix", "past_k", "past_v"], | |
| output_names=["logits", "present_k", "present_v"], | |
| dynamic_axes={ | |
| "tokens": {1: "s"}, "prefix": {1: "p"}, | |
| "past_k": {2: "t"}, "past_v": {2: "t"}, | |
| "present_k": {2: "t2"}, "present_v": {2: "t2"}, | |
| }, | |
| ) | |
| print("exported", cond_path, lm_path) | |
| def card_initial(lm) -> int: | |
| return lm.card | |
| if __name__ == "__main__": | |
| main() | |