Reverb-TTS-onnx / scripts /export_net.py
melser's picture
Add ONNX export of Reverb-TTS
ca95b32 verified
Raw History Blame Contribute Delete
5.98 kB
"""Exportable Reverb network: `prefix` (text + reference -> KV) and `step` (target tokens + KV -> log-probs).
Mirrors reverb_tts.sampler.Batch for batch size 1 (no padding, so no padding masks are needed).
"""
import math
import torch
from reverb_tts.model import CODEBOOKS, CODES, MASK
def rms(x, w, eps):
x32 = x.float()
x32 = x32 * torch.rsqrt(x32.pow(2).mean(-1, keepdim=True) + eps)
return w * x32.to(x.dtype)
def rotate_half(x):
h = x.shape[-1] // 2
return torch.cat([-x[..., h:], x[..., :h]], dim=-1)
class Core(torch.nn.Module):
def __init__(self, network):
super().__init__()
self.n = network
cfg = network.llm.config
self.heads, self.kv, self.hd = cfg.num_attention_heads, cfg.num_key_value_heads, cfg.head_dim
self.eps = cfg.rms_norm_eps
theta = cfg.rope_parameters["rope_theta"] if getattr(cfg, "rope_parameters", None) else cfg.rope_theta
self.register_buffer("inv_freq", 1.0 / (theta ** (torch.arange(0, self.hd, 2).float() / self.hd)), persistent=False)
def rope(self, positions): # positions [S] -> cos/sin [1,1,S,hd]
freqs = positions.float()[:, None] * self.inv_freq[None]
emb = torch.cat([freqs, freqs], -1)
return emb.cos()[None, None], emb.sin()[None, None]
def layer(self, i, x, cos, sin, past, bias):
L = self.n.llm.layers[i]
a = L.self_attn
B, S, _ = x.shape
h = rms(x, L.input_layernorm.weight, self.eps)
q = rms(a.q_proj(h).view(B, S, self.heads, self.hd), a.q_norm.weight, self.eps).transpose(1, 2)
k = rms(a.k_proj(h).view(B, S, self.kv, self.hd), a.k_norm.weight, self.eps).transpose(1, 2)
v = a.v_proj(h).view(B, S, self.kv, self.hd).transpose(1, 2)
q = q * cos + rotate_half(q) * sin
k = k * cos + rotate_half(k) * sin
new_k, new_v = k, v
if past is not None:
k = torch.cat([past[0], k], dim=2)
v = torch.cat([past[1], v], dim=2)
rep = self.heads // self.kv
kk = k.repeat_interleave(rep, dim=1)
vv = v.repeat_interleave(rep, dim=1)
scores = (q @ kk.transpose(-1, -2)) / math.sqrt(self.hd)
if bias is not None:
scores = scores + bias
att = torch.softmax(scores.float(), dim=-1).to(q.dtype) @ vv
att = att.transpose(1, 2).reshape(B, S, self.heads * self.hd)
x = x + a.o_proj(att)
h = rms(x, L.post_attention_layernorm.weight, self.eps)
m = L.mlp
x = x + m.down_proj(torch.nn.functional.silu(m.gate_proj(h)) * m.up_proj(h))
return x, new_k, new_v
class Prefix(Core):
"""text_ids [1,Pt], ref_codes [1,8,R] -> keys, values [24,1,6,P,128] (rotated keys, as HF caches them)."""
def forward(self, text_ids, ref_codes):
n = self.n
offsets = CODES * torch.arange(CODEBOOKS)[None, :, None]
text = n.llm.embed_tokens(text_ids)
ref = n.audio_embeddings(ref_codes + offsets).sum(1)
x = torch.cat([text, ref], dim=1)
S = x.shape[1]
cos, sin = self.rope(torch.arange(S))
ks, vs = [], []
for i in range(len(self.n.llm.layers)):
x, k, v = self.layer(i, x, cos, sin, None, None)
ks.append(k)
vs.append(v)
return torch.stack(ks), torch.stack(vs)
class Step(Core):
"""tokens [1,8,F], keys/values [24,1,6,P,128], prefix_bias [P] -> log-probs [1,8,F,1024] (fp32)."""
def forward(self, tokens, keys, values, prefix_bias):
n = self.n
F = tokens.shape[2]
P = keys.shape[3]
offsets = CODES * torch.arange(CODEBOOKS)[None, :, None]
x = n.audio_embeddings(tokens + offsets).sum(1)
cos, sin = self.rope(P + torch.arange(F))
bias = torch.cat([prefix_bias, torch.zeros(F, dtype=prefix_bias.dtype)])[None, None, None, :]
for i in range(len(self.n.llm.layers)):
x, _, _ = self.layer(i, x, cos, sin, (keys[i], values[i]), bias)
x = rms(x, n.llm.norm.weight, self.eps)
logits = (x @ n.audio_heads.weight.T).float().view(1, F, CODEBOOKS, CODES).transpose(1, 2)
return logits.log_softmax(-1)[..., :MASK]
class CoreGraph(Core):
"""One graph for both prefix and target passes.
x [1,S,1024]; past_k/past_v [24,1,6,Pp,128] (Pp may be 0); bias [S, Pp+S] additive attention bias.
Positions are Pp..Pp+S-1. Returns final-normed hidden [1,S,1024] and this call's rotated keys/values.
"""
def forward(self, x, past_k, past_v, bias):
S = x.shape[1]
Pp = past_k.shape[3]
cos, sin = self.rope(Pp + torch.arange(S))
mask = bias[None, None]
ks, vs = [], []
for i in range(len(self.n.llm.layers)):
x, k, v = self.layer(i, x, cos, sin, (past_k[i], past_v[i]), mask)
ks.append(k)
vs.append(v)
return rms(x, self.n.llm.norm.weight, self.eps), torch.stack(ks), torch.stack(vs)
class EmbedPrefix(torch.nn.Module):
def __init__(self, network):
super().__init__()
self.n = network
def forward(self, text_ids, ref_codes):
offsets = CODES * torch.arange(CODEBOOKS)[None, :, None]
return torch.cat([self.n.llm.embed_tokens(text_ids), self.n.audio_embeddings(ref_codes + offsets).sum(1)], dim=1)
class EmbedTarget(torch.nn.Module):
def __init__(self, network):
super().__init__()
self.n = network
def forward(self, tokens):
offsets = CODES * torch.arange(CODEBOOKS)[None, :, None]
return self.n.audio_embeddings(tokens + offsets).sum(1)
class Heads(torch.nn.Module):
"""hidden [1,F,1024] -> log-probs [1,8,F,1024] fp32."""
def __init__(self, network):
super().__init__()
self.n = network
def forward(self, hidden):
F = hidden.shape[1]
logits = (hidden @ self.n.audio_heads.weight.T).float().view(1, F, CODEBOOKS, CODES).transpose(1, 2)
return logits.log_softmax(-1)[..., :MASK]