muscriptor-onnx / tools /export_muscriptor_onnx.py
fpp-ableton's picture
Add MuScriptor small + medium ONNX (INT8), tools, notice
bedfc1d verified
Raw History Blame Contribute Delete
7.61 kB
#!/usr/bin/env python3
"""Export MuScriptor (small / medium) to two ONNX graphs for onnxruntime-node.
Dev-time tool only; nothing here ships inside the extension.
cond.onnx audio f32[1,80000] (5 s @ 16 kHz) + instrument i64[1,K] + dataset i64[1,1]
-> prefix f32[1, 501+1+K, dim]
The log-mel front end (reflect pad, STFT, mel bank, log) runs inside the graph
using the *checkpoint's own* window and filterbank buffers, which differ from the
textbook formulas by ~2.4e-4 and matter to the model.
instrument ids are raw group ids, -1 = unconditional (same for dataset).
lm.onnx tokens i64[1,S] + prefix f32[1,P,dim] + past_k/past_v f32[L,1,T,H,Dh]
-> logits f32[1,card] (last position only) + present_k/present_v
Explicit KV cache; positions continue from T. One graph serves prefill and decode.
Usage (from a venv with torch, onnx, onnxruntime, einops, safetensors, mido, soundfile):
python export_muscriptor_onnx.py --size small --checkpoint ckpt-small/model.safetensors \
--reference /path/to/muscriptor/repo --out out/small
"""
from __future__ import annotations
import argparse
import math
import sys
from pathlib import Path
import numpy as np
import torch
import torch.nn.functional as F
from torch import nn
SEGMENT = 80_000
N_FFT = 2048
HOP = 160
N_FREQ = N_FFT // 2 + 1
FRAMES = SEGMENT // HOP + 1 # 501
class Conditioner(nn.Module):
def __init__(self, lm):
super().__init__()
cps = lm.condition_provider.conditioners
wav = cps["self_wav"]
self.proj = wav.output_proj
self.inst_embed = cps["instrument_group"].embed
self.data_embed = cps["dataset_name"].embed
self.eps = wav.eps
window = wav.mel_spec_transform.spectrogram.window.detach().double()
fb = wav.mel_spec_transform.mel_scale.fb.detach().clone()
n = torch.arange(N_FFT, dtype=torch.float64)
k = torch.arange(N_FREQ, dtype=torch.float64)
ang = 2 * math.pi * k[:, None] * n[None, :] / N_FFT
kernel = torch.cat([window * torch.cos(ang), -window * torch.sin(ang)], 0) # [2*N_FREQ, N_FFT]
self.register_buffer("dft", kernel.float().unsqueeze(1)) # [2*N_FREQ, 1, N_FFT]
self.register_buffer("fb", fb) # [N_FREQ, n_mels]
def forward(self, audio, instrument, dataset):
x = F.pad(audio.unsqueeze(1), (N_FFT // 2, N_FFT // 2), mode="reflect")
spec = F.conv1d(x, self.dft, stride=HOP) # [1, 2*N_FREQ, FRAMES]
re, im = spec[:, :N_FREQ], spec[:, N_FREQ:]
mag = torch.sqrt(re * re + im * im)
mel = torch.matmul(mag.transpose(1, 2), self.fb) # [1, FRAMES, n_mels]
emb = self.proj(torch.log(mel + self.eps))
# The reference masks frames >= length/hop (500 of 501): the last frame is zeroed.
keep = (torch.arange(FRAMES) < SEGMENT // HOP).to(emb.dtype).view(1, -1, 1)
emb = emb * keep
# ClassConditioner: embedding row = id + 2 (id -1 -> row 1 = "unconditional")
inst = self.inst_embed(instrument + 2)
data = self.data_embed(dataset + 2)
# Order in the LM input is [wav, dataset, instrument]
return torch.cat([emb, data, inst], dim=1)
class Decoder(nn.Module):
def __init__(self, lm, heads: int):
super().__init__()
self.lm = lm
self.heads = heads
def forward(self, tokens, prefix, past_k, past_v):
lm = self.lm
x = lm.emb(tokens)
x = torch.cat([prefix.to(x.dtype), x], dim=1)
n_new = x.shape[1]
past = past_k.shape[2]
dim = x.shape[2]
h = self.heads
dh = dim // h
pos = (torch.arange(n_new) + past).view(1, -1, 1).float()
half = dim // 2
adim = torch.arange(half, dtype=torch.float32).view(1, 1, -1)
phase = pos / (10000.0 ** (adim / (half - 1)))
x = x + torch.cat([torch.cos(phase), torch.sin(phase)], dim=-1)
q_idx = (torch.arange(n_new) + past).view(-1, 1)
k_idx = torch.arange(past + n_new).view(1, -1)
blocked = k_idx > q_idx # [n_new, past+n_new]
ks, vs = [], []
for i, layer in enumerate(lm.transformer.layers):
attn = layer.self_attn
proj = F.linear(layer.norm1(x), attn.in_proj_weight).view(1, n_new, 3, h, dh)
q, k, v = proj[:, :, 0], proj[:, :, 1], proj[:, :, 2]
k_all = torch.cat([past_k[i], k], dim=1) # [1, T', h, dh]
v_all = torch.cat([past_v[i], v], dim=1)
ks.append(k_all)
vs.append(v_all)
scores = torch.matmul(q.transpose(1, 2), k_all.permute(0, 2, 3, 1)) / math.sqrt(dh)
scores = scores.masked_fill(blocked, float("-inf"))
out = torch.matmul(torch.softmax(scores, dim=-1), v_all.transpose(1, 2))
out = out.transpose(1, 2).reshape(1, n_new, dim)
x = x + attn.out_proj(out)
x = x + layer.linear2(F.gelu(layer.linear1(layer.norm2(x))))
last = lm.out_norm(x[:, -1:])
logits = lm.linear(last)[:, 0]
return logits, torch.stack(ks), torch.stack(vs)
def load_reference(reference: Path, checkpoint: Path):
sys.path.insert(0, str(reference))
import muscriptor.transcription_model as tm # noqa: WPS433
model = tm.TranscriptionModel.load_model(str(checkpoint), device="cpu", dtype="float32")
return tm, model
def main() -> None:
ap = argparse.ArgumentParser()
ap.add_argument("--size", choices=["small", "medium"], required=True)
ap.add_argument("--checkpoint", type=Path, required=True)
ap.add_argument("--reference", type=Path, required=True, help="clone of github.com/muscriptor/muscriptor")
ap.add_argument("--out", type=Path, required=True)
ap.add_argument("--opset", type=int, default=17)
args = ap.parse_args()
args.out.mkdir(parents=True, exist_ok=True)
tm, ref = load_reference(args.reference, args.checkpoint)
lm = ref._model
cfg = tm._CONFIGS[args.size]
layers, heads, dim = cfg.num_layers, cfg.num_heads, cfg.dim
dh = dim // heads
lm.eval()
cond = Conditioner(lm).eval()
dec = Decoder(lm, heads).eval()
# ---- export conditioner -------------------------------------------------
audio = torch.randn(1, SEGMENT) * 0.1
inst = torch.tensor([[-1]])
dset = torch.tensor([[-1]])
cond_path = args.out / "cond.onnx"
torch.onnx.export(
cond, (audio, inst, dset), str(cond_path), opset_version=args.opset, dynamo=False,
input_names=["audio", "instrument", "dataset"], output_names=["prefix"],
dynamic_axes={"instrument": {1: "k"}, "prefix": {1: "p"}},
)
# ---- export decoder -----------------------------------------------------
n_prefix = FRAMES + 1 + 1
tokens = torch.tensor([[card_initial(lm), 5, 6]])
prefix = torch.randn(1, n_prefix, dim)
pk = torch.randn(layers, 1, 4, heads, dh) * 0.1
pv = torch.randn(layers, 1, 4, heads, dh) * 0.1
lm_path = args.out / "lm.onnx"
torch.onnx.export(
dec, (tokens, prefix, pk, pv), str(lm_path), opset_version=args.opset, dynamo=False,
input_names=["tokens", "prefix", "past_k", "past_v"],
output_names=["logits", "present_k", "present_v"],
dynamic_axes={
"tokens": {1: "s"}, "prefix": {1: "p"},
"past_k": {2: "t"}, "past_v": {2: "t"},
"present_k": {2: "t2"}, "present_v": {2: "t2"},
},
)
print("exported", cond_path, lm_path)
def card_initial(lm) -> int:
return lm.card
if __name__ == "__main__":
main()