File size: 7,606 Bytes
bedfc1d
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
#!/usr/bin/env python3
"""Export MuScriptor (small / medium) to two ONNX graphs for onnxruntime-node.

Dev-time tool only; nothing here ships inside the extension.

  cond.onnx  audio f32[1,80000] (5 s @ 16 kHz) + instrument i64[1,K] + dataset i64[1,1]
             -> prefix f32[1, 501+1+K, dim]
             The log-mel front end (reflect pad, STFT, mel bank, log) runs inside the graph
             using the *checkpoint's own* window and filterbank buffers, which differ from the
             textbook formulas by ~2.4e-4 and matter to the model.
             instrument ids are raw group ids, -1 = unconditional (same for dataset).
  lm.onnx    tokens i64[1,S] + prefix f32[1,P,dim] + past_k/past_v f32[L,1,T,H,Dh]
             -> logits f32[1,card] (last position only) + present_k/present_v
             Explicit KV cache; positions continue from T. One graph serves prefill and decode.

Usage (from a venv with torch, onnx, onnxruntime, einops, safetensors, mido, soundfile):
  python export_muscriptor_onnx.py --size small  --checkpoint ckpt-small/model.safetensors \
      --reference /path/to/muscriptor/repo --out out/small
"""

from __future__ import annotations

import argparse
import math
import sys
from pathlib import Path

import numpy as np
import torch
import torch.nn.functional as F
from torch import nn

SEGMENT = 80_000
N_FFT = 2048
HOP = 160
N_FREQ = N_FFT // 2 + 1
FRAMES = SEGMENT // HOP + 1  # 501


class Conditioner(nn.Module):
    def __init__(self, lm):
        super().__init__()
        cps = lm.condition_provider.conditioners
        wav = cps["self_wav"]
        self.proj = wav.output_proj
        self.inst_embed = cps["instrument_group"].embed
        self.data_embed = cps["dataset_name"].embed
        self.eps = wav.eps

        window = wav.mel_spec_transform.spectrogram.window.detach().double()
        fb = wav.mel_spec_transform.mel_scale.fb.detach().clone()
        n = torch.arange(N_FFT, dtype=torch.float64)
        k = torch.arange(N_FREQ, dtype=torch.float64)
        ang = 2 * math.pi * k[:, None] * n[None, :] / N_FFT
        kernel = torch.cat([window * torch.cos(ang), -window * torch.sin(ang)], 0)  # [2*N_FREQ, N_FFT]
        self.register_buffer("dft", kernel.float().unsqueeze(1))  # [2*N_FREQ, 1, N_FFT]
        self.register_buffer("fb", fb)  # [N_FREQ, n_mels]

    def forward(self, audio, instrument, dataset):
        x = F.pad(audio.unsqueeze(1), (N_FFT // 2, N_FFT // 2), mode="reflect")
        spec = F.conv1d(x, self.dft, stride=HOP)  # [1, 2*N_FREQ, FRAMES]
        re, im = spec[:, :N_FREQ], spec[:, N_FREQ:]
        mag = torch.sqrt(re * re + im * im)
        mel = torch.matmul(mag.transpose(1, 2), self.fb)  # [1, FRAMES, n_mels]
        emb = self.proj(torch.log(mel + self.eps))
        # The reference masks frames >= length/hop (500 of 501): the last frame is zeroed.
        keep = (torch.arange(FRAMES) < SEGMENT // HOP).to(emb.dtype).view(1, -1, 1)
        emb = emb * keep
        # ClassConditioner: embedding row = id + 2 (id -1 -> row 1 = "unconditional")
        inst = self.inst_embed(instrument + 2)
        data = self.data_embed(dataset + 2)
        # Order in the LM input is [wav, dataset, instrument]
        return torch.cat([emb, data, inst], dim=1)


class Decoder(nn.Module):
    def __init__(self, lm, heads: int):
        super().__init__()
        self.lm = lm
        self.heads = heads

    def forward(self, tokens, prefix, past_k, past_v):
        lm = self.lm
        x = lm.emb(tokens)
        x = torch.cat([prefix.to(x.dtype), x], dim=1)
        n_new = x.shape[1]
        past = past_k.shape[2]
        dim = x.shape[2]
        h = self.heads
        dh = dim // h

        pos = (torch.arange(n_new) + past).view(1, -1, 1).float()
        half = dim // 2
        adim = torch.arange(half, dtype=torch.float32).view(1, 1, -1)
        phase = pos / (10000.0 ** (adim / (half - 1)))
        x = x + torch.cat([torch.cos(phase), torch.sin(phase)], dim=-1)

        q_idx = (torch.arange(n_new) + past).view(-1, 1)
        k_idx = torch.arange(past + n_new).view(1, -1)
        blocked = k_idx > q_idx  # [n_new, past+n_new]

        ks, vs = [], []
        for i, layer in enumerate(lm.transformer.layers):
            attn = layer.self_attn
            proj = F.linear(layer.norm1(x), attn.in_proj_weight).view(1, n_new, 3, h, dh)
            q, k, v = proj[:, :, 0], proj[:, :, 1], proj[:, :, 2]
            k_all = torch.cat([past_k[i], k], dim=1)  # [1, T', h, dh]
            v_all = torch.cat([past_v[i], v], dim=1)
            ks.append(k_all)
            vs.append(v_all)
            scores = torch.matmul(q.transpose(1, 2), k_all.permute(0, 2, 3, 1)) / math.sqrt(dh)
            scores = scores.masked_fill(blocked, float("-inf"))
            out = torch.matmul(torch.softmax(scores, dim=-1), v_all.transpose(1, 2))
            out = out.transpose(1, 2).reshape(1, n_new, dim)
            x = x + attn.out_proj(out)
            x = x + layer.linear2(F.gelu(layer.linear1(layer.norm2(x))))

        last = lm.out_norm(x[:, -1:])
        logits = lm.linear(last)[:, 0]
        return logits, torch.stack(ks), torch.stack(vs)


def load_reference(reference: Path, checkpoint: Path):
    sys.path.insert(0, str(reference))
    import muscriptor.transcription_model as tm  # noqa: WPS433

    model = tm.TranscriptionModel.load_model(str(checkpoint), device="cpu", dtype="float32")
    return tm, model


def main() -> None:
    ap = argparse.ArgumentParser()
    ap.add_argument("--size", choices=["small", "medium"], required=True)
    ap.add_argument("--checkpoint", type=Path, required=True)
    ap.add_argument("--reference", type=Path, required=True, help="clone of github.com/muscriptor/muscriptor")
    ap.add_argument("--out", type=Path, required=True)
    ap.add_argument("--opset", type=int, default=17)
    args = ap.parse_args()
    args.out.mkdir(parents=True, exist_ok=True)

    tm, ref = load_reference(args.reference, args.checkpoint)
    lm = ref._model
    cfg = tm._CONFIGS[args.size]
    layers, heads, dim = cfg.num_layers, cfg.num_heads, cfg.dim
    dh = dim // heads
    lm.eval()

    cond = Conditioner(lm).eval()
    dec = Decoder(lm, heads).eval()

    # ---- export conditioner -------------------------------------------------
    audio = torch.randn(1, SEGMENT) * 0.1
    inst = torch.tensor([[-1]])
    dset = torch.tensor([[-1]])
    cond_path = args.out / "cond.onnx"
    torch.onnx.export(
        cond, (audio, inst, dset), str(cond_path), opset_version=args.opset, dynamo=False,
        input_names=["audio", "instrument", "dataset"], output_names=["prefix"],
        dynamic_axes={"instrument": {1: "k"}, "prefix": {1: "p"}},
    )

    # ---- export decoder -----------------------------------------------------
    n_prefix = FRAMES + 1 + 1
    tokens = torch.tensor([[card_initial(lm), 5, 6]])
    prefix = torch.randn(1, n_prefix, dim)
    pk = torch.randn(layers, 1, 4, heads, dh) * 0.1
    pv = torch.randn(layers, 1, 4, heads, dh) * 0.1
    lm_path = args.out / "lm.onnx"
    torch.onnx.export(
        dec, (tokens, prefix, pk, pv), str(lm_path), opset_version=args.opset, dynamo=False,
        input_names=["tokens", "prefix", "past_k", "past_v"],
        output_names=["logits", "present_k", "present_v"],
        dynamic_axes={
            "tokens": {1: "s"}, "prefix": {1: "p"},
            "past_k": {2: "t"}, "past_v": {2: "t"},
            "present_k": {2: "t2"}, "present_v": {2: "t2"},
        },
    )
    print("exported", cond_path, lm_path)


def card_initial(lm) -> int:
    return lm.card


if __name__ == "__main__":
    main()