#!/usr/bin/env python3 """Export MuScriptor (small / medium) to two ONNX graphs for onnxruntime-node. Dev-time tool only; nothing here ships inside the extension. cond.onnx audio f32[1,80000] (5 s @ 16 kHz) + instrument i64[1,K] + dataset i64[1,1] -> prefix f32[1, 501+1+K, dim] The log-mel front end (reflect pad, STFT, mel bank, log) runs inside the graph using the *checkpoint's own* window and filterbank buffers, which differ from the textbook formulas by ~2.4e-4 and matter to the model. instrument ids are raw group ids, -1 = unconditional (same for dataset). lm.onnx tokens i64[1,S] + prefix f32[1,P,dim] + past_k/past_v f32[L,1,T,H,Dh] -> logits f32[1,card] (last position only) + present_k/present_v Explicit KV cache; positions continue from T. One graph serves prefill and decode. Usage (from a venv with torch, onnx, onnxruntime, einops, safetensors, mido, soundfile): python export_muscriptor_onnx.py --size small --checkpoint ckpt-small/model.safetensors \ --reference /path/to/muscriptor/repo --out out/small """ from __future__ import annotations import argparse import math import sys from pathlib import Path import numpy as np import torch import torch.nn.functional as F from torch import nn SEGMENT = 80_000 N_FFT = 2048 HOP = 160 N_FREQ = N_FFT // 2 + 1 FRAMES = SEGMENT // HOP + 1 # 501 class Conditioner(nn.Module): def __init__(self, lm): super().__init__() cps = lm.condition_provider.conditioners wav = cps["self_wav"] self.proj = wav.output_proj self.inst_embed = cps["instrument_group"].embed self.data_embed = cps["dataset_name"].embed self.eps = wav.eps window = wav.mel_spec_transform.spectrogram.window.detach().double() fb = wav.mel_spec_transform.mel_scale.fb.detach().clone() n = torch.arange(N_FFT, dtype=torch.float64) k = torch.arange(N_FREQ, dtype=torch.float64) ang = 2 * math.pi * k[:, None] * n[None, :] / N_FFT kernel = torch.cat([window * torch.cos(ang), -window * torch.sin(ang)], 0) # [2*N_FREQ, N_FFT] self.register_buffer("dft", kernel.float().unsqueeze(1)) # [2*N_FREQ, 1, N_FFT] self.register_buffer("fb", fb) # [N_FREQ, n_mels] def forward(self, audio, instrument, dataset): x = F.pad(audio.unsqueeze(1), (N_FFT // 2, N_FFT // 2), mode="reflect") spec = F.conv1d(x, self.dft, stride=HOP) # [1, 2*N_FREQ, FRAMES] re, im = spec[:, :N_FREQ], spec[:, N_FREQ:] mag = torch.sqrt(re * re + im * im) mel = torch.matmul(mag.transpose(1, 2), self.fb) # [1, FRAMES, n_mels] emb = self.proj(torch.log(mel + self.eps)) # The reference masks frames >= length/hop (500 of 501): the last frame is zeroed. keep = (torch.arange(FRAMES) < SEGMENT // HOP).to(emb.dtype).view(1, -1, 1) emb = emb * keep # ClassConditioner: embedding row = id + 2 (id -1 -> row 1 = "unconditional") inst = self.inst_embed(instrument + 2) data = self.data_embed(dataset + 2) # Order in the LM input is [wav, dataset, instrument] return torch.cat([emb, data, inst], dim=1) class Decoder(nn.Module): def __init__(self, lm, heads: int): super().__init__() self.lm = lm self.heads = heads def forward(self, tokens, prefix, past_k, past_v): lm = self.lm x = lm.emb(tokens) x = torch.cat([prefix.to(x.dtype), x], dim=1) n_new = x.shape[1] past = past_k.shape[2] dim = x.shape[2] h = self.heads dh = dim // h pos = (torch.arange(n_new) + past).view(1, -1, 1).float() half = dim // 2 adim = torch.arange(half, dtype=torch.float32).view(1, 1, -1) phase = pos / (10000.0 ** (adim / (half - 1))) x = x + torch.cat([torch.cos(phase), torch.sin(phase)], dim=-1) q_idx = (torch.arange(n_new) + past).view(-1, 1) k_idx = torch.arange(past + n_new).view(1, -1) blocked = k_idx > q_idx # [n_new, past+n_new] ks, vs = [], [] for i, layer in enumerate(lm.transformer.layers): attn = layer.self_attn proj = F.linear(layer.norm1(x), attn.in_proj_weight).view(1, n_new, 3, h, dh) q, k, v = proj[:, :, 0], proj[:, :, 1], proj[:, :, 2] k_all = torch.cat([past_k[i], k], dim=1) # [1, T', h, dh] v_all = torch.cat([past_v[i], v], dim=1) ks.append(k_all) vs.append(v_all) scores = torch.matmul(q.transpose(1, 2), k_all.permute(0, 2, 3, 1)) / math.sqrt(dh) scores = scores.masked_fill(blocked, float("-inf")) out = torch.matmul(torch.softmax(scores, dim=-1), v_all.transpose(1, 2)) out = out.transpose(1, 2).reshape(1, n_new, dim) x = x + attn.out_proj(out) x = x + layer.linear2(F.gelu(layer.linear1(layer.norm2(x)))) last = lm.out_norm(x[:, -1:]) logits = lm.linear(last)[:, 0] return logits, torch.stack(ks), torch.stack(vs) def load_reference(reference: Path, checkpoint: Path): sys.path.insert(0, str(reference)) import muscriptor.transcription_model as tm # noqa: WPS433 model = tm.TranscriptionModel.load_model(str(checkpoint), device="cpu", dtype="float32") return tm, model def main() -> None: ap = argparse.ArgumentParser() ap.add_argument("--size", choices=["small", "medium"], required=True) ap.add_argument("--checkpoint", type=Path, required=True) ap.add_argument("--reference", type=Path, required=True, help="clone of github.com/muscriptor/muscriptor") ap.add_argument("--out", type=Path, required=True) ap.add_argument("--opset", type=int, default=17) args = ap.parse_args() args.out.mkdir(parents=True, exist_ok=True) tm, ref = load_reference(args.reference, args.checkpoint) lm = ref._model cfg = tm._CONFIGS[args.size] layers, heads, dim = cfg.num_layers, cfg.num_heads, cfg.dim dh = dim // heads lm.eval() cond = Conditioner(lm).eval() dec = Decoder(lm, heads).eval() # ---- export conditioner ------------------------------------------------- audio = torch.randn(1, SEGMENT) * 0.1 inst = torch.tensor([[-1]]) dset = torch.tensor([[-1]]) cond_path = args.out / "cond.onnx" torch.onnx.export( cond, (audio, inst, dset), str(cond_path), opset_version=args.opset, dynamo=False, input_names=["audio", "instrument", "dataset"], output_names=["prefix"], dynamic_axes={"instrument": {1: "k"}, "prefix": {1: "p"}}, ) # ---- export decoder ----------------------------------------------------- n_prefix = FRAMES + 1 + 1 tokens = torch.tensor([[card_initial(lm), 5, 6]]) prefix = torch.randn(1, n_prefix, dim) pk = torch.randn(layers, 1, 4, heads, dh) * 0.1 pv = torch.randn(layers, 1, 4, heads, dh) * 0.1 lm_path = args.out / "lm.onnx" torch.onnx.export( dec, (tokens, prefix, pk, pv), str(lm_path), opset_version=args.opset, dynamo=False, input_names=["tokens", "prefix", "past_k", "past_v"], output_names=["logits", "present_k", "present_v"], dynamic_axes={ "tokens": {1: "s"}, "prefix": {1: "p"}, "past_k": {2: "t"}, "past_v": {2: "t"}, "present_k": {2: "t2"}, "present_v": {2: "t2"}, }, ) print("exported", cond_path, lm_path) def card_initial(lm) -> int: return lm.card if __name__ == "__main__": main()