File size: 7,606 Bytes
bedfc1d | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 | #!/usr/bin/env python3
"""Export MuScriptor (small / medium) to two ONNX graphs for onnxruntime-node.
Dev-time tool only; nothing here ships inside the extension.
cond.onnx audio f32[1,80000] (5 s @ 16 kHz) + instrument i64[1,K] + dataset i64[1,1]
-> prefix f32[1, 501+1+K, dim]
The log-mel front end (reflect pad, STFT, mel bank, log) runs inside the graph
using the *checkpoint's own* window and filterbank buffers, which differ from the
textbook formulas by ~2.4e-4 and matter to the model.
instrument ids are raw group ids, -1 = unconditional (same for dataset).
lm.onnx tokens i64[1,S] + prefix f32[1,P,dim] + past_k/past_v f32[L,1,T,H,Dh]
-> logits f32[1,card] (last position only) + present_k/present_v
Explicit KV cache; positions continue from T. One graph serves prefill and decode.
Usage (from a venv with torch, onnx, onnxruntime, einops, safetensors, mido, soundfile):
python export_muscriptor_onnx.py --size small --checkpoint ckpt-small/model.safetensors \
--reference /path/to/muscriptor/repo --out out/small
"""
from __future__ import annotations
import argparse
import math
import sys
from pathlib import Path
import numpy as np
import torch
import torch.nn.functional as F
from torch import nn
SEGMENT = 80_000
N_FFT = 2048
HOP = 160
N_FREQ = N_FFT // 2 + 1
FRAMES = SEGMENT // HOP + 1 # 501
class Conditioner(nn.Module):
def __init__(self, lm):
super().__init__()
cps = lm.condition_provider.conditioners
wav = cps["self_wav"]
self.proj = wav.output_proj
self.inst_embed = cps["instrument_group"].embed
self.data_embed = cps["dataset_name"].embed
self.eps = wav.eps
window = wav.mel_spec_transform.spectrogram.window.detach().double()
fb = wav.mel_spec_transform.mel_scale.fb.detach().clone()
n = torch.arange(N_FFT, dtype=torch.float64)
k = torch.arange(N_FREQ, dtype=torch.float64)
ang = 2 * math.pi * k[:, None] * n[None, :] / N_FFT
kernel = torch.cat([window * torch.cos(ang), -window * torch.sin(ang)], 0) # [2*N_FREQ, N_FFT]
self.register_buffer("dft", kernel.float().unsqueeze(1)) # [2*N_FREQ, 1, N_FFT]
self.register_buffer("fb", fb) # [N_FREQ, n_mels]
def forward(self, audio, instrument, dataset):
x = F.pad(audio.unsqueeze(1), (N_FFT // 2, N_FFT // 2), mode="reflect")
spec = F.conv1d(x, self.dft, stride=HOP) # [1, 2*N_FREQ, FRAMES]
re, im = spec[:, :N_FREQ], spec[:, N_FREQ:]
mag = torch.sqrt(re * re + im * im)
mel = torch.matmul(mag.transpose(1, 2), self.fb) # [1, FRAMES, n_mels]
emb = self.proj(torch.log(mel + self.eps))
# The reference masks frames >= length/hop (500 of 501): the last frame is zeroed.
keep = (torch.arange(FRAMES) < SEGMENT // HOP).to(emb.dtype).view(1, -1, 1)
emb = emb * keep
# ClassConditioner: embedding row = id + 2 (id -1 -> row 1 = "unconditional")
inst = self.inst_embed(instrument + 2)
data = self.data_embed(dataset + 2)
# Order in the LM input is [wav, dataset, instrument]
return torch.cat([emb, data, inst], dim=1)
class Decoder(nn.Module):
def __init__(self, lm, heads: int):
super().__init__()
self.lm = lm
self.heads = heads
def forward(self, tokens, prefix, past_k, past_v):
lm = self.lm
x = lm.emb(tokens)
x = torch.cat([prefix.to(x.dtype), x], dim=1)
n_new = x.shape[1]
past = past_k.shape[2]
dim = x.shape[2]
h = self.heads
dh = dim // h
pos = (torch.arange(n_new) + past).view(1, -1, 1).float()
half = dim // 2
adim = torch.arange(half, dtype=torch.float32).view(1, 1, -1)
phase = pos / (10000.0 ** (adim / (half - 1)))
x = x + torch.cat([torch.cos(phase), torch.sin(phase)], dim=-1)
q_idx = (torch.arange(n_new) + past).view(-1, 1)
k_idx = torch.arange(past + n_new).view(1, -1)
blocked = k_idx > q_idx # [n_new, past+n_new]
ks, vs = [], []
for i, layer in enumerate(lm.transformer.layers):
attn = layer.self_attn
proj = F.linear(layer.norm1(x), attn.in_proj_weight).view(1, n_new, 3, h, dh)
q, k, v = proj[:, :, 0], proj[:, :, 1], proj[:, :, 2]
k_all = torch.cat([past_k[i], k], dim=1) # [1, T', h, dh]
v_all = torch.cat([past_v[i], v], dim=1)
ks.append(k_all)
vs.append(v_all)
scores = torch.matmul(q.transpose(1, 2), k_all.permute(0, 2, 3, 1)) / math.sqrt(dh)
scores = scores.masked_fill(blocked, float("-inf"))
out = torch.matmul(torch.softmax(scores, dim=-1), v_all.transpose(1, 2))
out = out.transpose(1, 2).reshape(1, n_new, dim)
x = x + attn.out_proj(out)
x = x + layer.linear2(F.gelu(layer.linear1(layer.norm2(x))))
last = lm.out_norm(x[:, -1:])
logits = lm.linear(last)[:, 0]
return logits, torch.stack(ks), torch.stack(vs)
def load_reference(reference: Path, checkpoint: Path):
sys.path.insert(0, str(reference))
import muscriptor.transcription_model as tm # noqa: WPS433
model = tm.TranscriptionModel.load_model(str(checkpoint), device="cpu", dtype="float32")
return tm, model
def main() -> None:
ap = argparse.ArgumentParser()
ap.add_argument("--size", choices=["small", "medium"], required=True)
ap.add_argument("--checkpoint", type=Path, required=True)
ap.add_argument("--reference", type=Path, required=True, help="clone of github.com/muscriptor/muscriptor")
ap.add_argument("--out", type=Path, required=True)
ap.add_argument("--opset", type=int, default=17)
args = ap.parse_args()
args.out.mkdir(parents=True, exist_ok=True)
tm, ref = load_reference(args.reference, args.checkpoint)
lm = ref._model
cfg = tm._CONFIGS[args.size]
layers, heads, dim = cfg.num_layers, cfg.num_heads, cfg.dim
dh = dim // heads
lm.eval()
cond = Conditioner(lm).eval()
dec = Decoder(lm, heads).eval()
# ---- export conditioner -------------------------------------------------
audio = torch.randn(1, SEGMENT) * 0.1
inst = torch.tensor([[-1]])
dset = torch.tensor([[-1]])
cond_path = args.out / "cond.onnx"
torch.onnx.export(
cond, (audio, inst, dset), str(cond_path), opset_version=args.opset, dynamo=False,
input_names=["audio", "instrument", "dataset"], output_names=["prefix"],
dynamic_axes={"instrument": {1: "k"}, "prefix": {1: "p"}},
)
# ---- export decoder -----------------------------------------------------
n_prefix = FRAMES + 1 + 1
tokens = torch.tensor([[card_initial(lm), 5, 6]])
prefix = torch.randn(1, n_prefix, dim)
pk = torch.randn(layers, 1, 4, heads, dh) * 0.1
pv = torch.randn(layers, 1, 4, heads, dh) * 0.1
lm_path = args.out / "lm.onnx"
torch.onnx.export(
dec, (tokens, prefix, pk, pv), str(lm_path), opset_version=args.opset, dynamo=False,
input_names=["tokens", "prefix", "past_k", "past_v"],
output_names=["logits", "present_k", "present_v"],
dynamic_axes={
"tokens": {1: "s"}, "prefix": {1: "p"},
"past_k": {2: "t"}, "past_v": {2: "t"},
"present_k": {2: "t2"}, "present_v": {2: "t2"},
},
)
print("exported", cond_path, lm_path)
def card_initial(lm) -> int:
return lm.card
if __name__ == "__main__":
main()
|