File size: 5,983 Bytes
ca95b32 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 | """Exportable Reverb network: `prefix` (text + reference -> KV) and `step` (target tokens + KV -> log-probs).
Mirrors reverb_tts.sampler.Batch for batch size 1 (no padding, so no padding masks are needed).
"""
import math
import torch
from reverb_tts.model import CODEBOOKS, CODES, MASK
def rms(x, w, eps):
x32 = x.float()
x32 = x32 * torch.rsqrt(x32.pow(2).mean(-1, keepdim=True) + eps)
return w * x32.to(x.dtype)
def rotate_half(x):
h = x.shape[-1] // 2
return torch.cat([-x[..., h:], x[..., :h]], dim=-1)
class Core(torch.nn.Module):
def __init__(self, network):
super().__init__()
self.n = network
cfg = network.llm.config
self.heads, self.kv, self.hd = cfg.num_attention_heads, cfg.num_key_value_heads, cfg.head_dim
self.eps = cfg.rms_norm_eps
theta = cfg.rope_parameters["rope_theta"] if getattr(cfg, "rope_parameters", None) else cfg.rope_theta
self.register_buffer("inv_freq", 1.0 / (theta ** (torch.arange(0, self.hd, 2).float() / self.hd)), persistent=False)
def rope(self, positions): # positions [S] -> cos/sin [1,1,S,hd]
freqs = positions.float()[:, None] * self.inv_freq[None]
emb = torch.cat([freqs, freqs], -1)
return emb.cos()[None, None], emb.sin()[None, None]
def layer(self, i, x, cos, sin, past, bias):
L = self.n.llm.layers[i]
a = L.self_attn
B, S, _ = x.shape
h = rms(x, L.input_layernorm.weight, self.eps)
q = rms(a.q_proj(h).view(B, S, self.heads, self.hd), a.q_norm.weight, self.eps).transpose(1, 2)
k = rms(a.k_proj(h).view(B, S, self.kv, self.hd), a.k_norm.weight, self.eps).transpose(1, 2)
v = a.v_proj(h).view(B, S, self.kv, self.hd).transpose(1, 2)
q = q * cos + rotate_half(q) * sin
k = k * cos + rotate_half(k) * sin
new_k, new_v = k, v
if past is not None:
k = torch.cat([past[0], k], dim=2)
v = torch.cat([past[1], v], dim=2)
rep = self.heads // self.kv
kk = k.repeat_interleave(rep, dim=1)
vv = v.repeat_interleave(rep, dim=1)
scores = (q @ kk.transpose(-1, -2)) / math.sqrt(self.hd)
if bias is not None:
scores = scores + bias
att = torch.softmax(scores.float(), dim=-1).to(q.dtype) @ vv
att = att.transpose(1, 2).reshape(B, S, self.heads * self.hd)
x = x + a.o_proj(att)
h = rms(x, L.post_attention_layernorm.weight, self.eps)
m = L.mlp
x = x + m.down_proj(torch.nn.functional.silu(m.gate_proj(h)) * m.up_proj(h))
return x, new_k, new_v
class Prefix(Core):
"""text_ids [1,Pt], ref_codes [1,8,R] -> keys, values [24,1,6,P,128] (rotated keys, as HF caches them)."""
def forward(self, text_ids, ref_codes):
n = self.n
offsets = CODES * torch.arange(CODEBOOKS)[None, :, None]
text = n.llm.embed_tokens(text_ids)
ref = n.audio_embeddings(ref_codes + offsets).sum(1)
x = torch.cat([text, ref], dim=1)
S = x.shape[1]
cos, sin = self.rope(torch.arange(S))
ks, vs = [], []
for i in range(len(self.n.llm.layers)):
x, k, v = self.layer(i, x, cos, sin, None, None)
ks.append(k)
vs.append(v)
return torch.stack(ks), torch.stack(vs)
class Step(Core):
"""tokens [1,8,F], keys/values [24,1,6,P,128], prefix_bias [P] -> log-probs [1,8,F,1024] (fp32)."""
def forward(self, tokens, keys, values, prefix_bias):
n = self.n
F = tokens.shape[2]
P = keys.shape[3]
offsets = CODES * torch.arange(CODEBOOKS)[None, :, None]
x = n.audio_embeddings(tokens + offsets).sum(1)
cos, sin = self.rope(P + torch.arange(F))
bias = torch.cat([prefix_bias, torch.zeros(F, dtype=prefix_bias.dtype)])[None, None, None, :]
for i in range(len(self.n.llm.layers)):
x, _, _ = self.layer(i, x, cos, sin, (keys[i], values[i]), bias)
x = rms(x, n.llm.norm.weight, self.eps)
logits = (x @ n.audio_heads.weight.T).float().view(1, F, CODEBOOKS, CODES).transpose(1, 2)
return logits.log_softmax(-1)[..., :MASK]
class CoreGraph(Core):
"""One graph for both prefix and target passes.
x [1,S,1024]; past_k/past_v [24,1,6,Pp,128] (Pp may be 0); bias [S, Pp+S] additive attention bias.
Positions are Pp..Pp+S-1. Returns final-normed hidden [1,S,1024] and this call's rotated keys/values.
"""
def forward(self, x, past_k, past_v, bias):
S = x.shape[1]
Pp = past_k.shape[3]
cos, sin = self.rope(Pp + torch.arange(S))
mask = bias[None, None]
ks, vs = [], []
for i in range(len(self.n.llm.layers)):
x, k, v = self.layer(i, x, cos, sin, (past_k[i], past_v[i]), mask)
ks.append(k)
vs.append(v)
return rms(x, self.n.llm.norm.weight, self.eps), torch.stack(ks), torch.stack(vs)
class EmbedPrefix(torch.nn.Module):
def __init__(self, network):
super().__init__()
self.n = network
def forward(self, text_ids, ref_codes):
offsets = CODES * torch.arange(CODEBOOKS)[None, :, None]
return torch.cat([self.n.llm.embed_tokens(text_ids), self.n.audio_embeddings(ref_codes + offsets).sum(1)], dim=1)
class EmbedTarget(torch.nn.Module):
def __init__(self, network):
super().__init__()
self.n = network
def forward(self, tokens):
offsets = CODES * torch.arange(CODEBOOKS)[None, :, None]
return self.n.audio_embeddings(tokens + offsets).sum(1)
class Heads(torch.nn.Module):
"""hidden [1,F,1024] -> log-probs [1,8,F,1024] fp32."""
def __init__(self, network):
super().__init__()
self.n = network
def forward(self, hidden):
F = hidden.shape[1]
logits = (hidden @ self.n.audio_heads.weight.T).float().view(1, F, CODEBOOKS, CODES).transpose(1, 2)
return logits.log_softmax(-1)[..., :MASK]
|