Julia-2 / runtime /julia2 /kernels.py
Julia Klee
Julia-2: julia2_audio_v1 (text = base_v2 byte-identical, + pruned audio tower, stem p9), int8/int4 variants, lean runtime 0.1.0 (audio, LiveSession, modalities), card with speech, live calls and variants
1d1c6db
Raw History Blame Contribute Delete
4.28 kB
"""Fast kernels with pure-torch fallbacks (no CUDA-only code at import).
causal_conv1d(x [B,C,T], w [C,K], b [C], state [B,C,K-1] | None) -> (y [B,C,T], final_state [B,C,K-1])
gated_delta_rule(q,k,v [B,T,H,D], g [B,T,H] fp32 log-decay, beta [B,T,H], state [B,H,Dk,Dv] | None) -> (o, final_state)
On CUDA with `flash-linear-attention` / `causal-conv1d` installed the fused kernels are used (same maths as the
reference training/eval code); otherwise the chunked torch implementation below (fp32) runs on any device.
"""
import torch
import torch.nn.functional as F
try: # optional fast kernels
from fla.ops.gated_delta_rule import chunk_gated_delta_rule as _fla_chunk
except Exception: # pragma: no cover
_fla_chunk = None
try:
from causal_conv1d import causal_conv1d_fn as _cc1d
except Exception: # pragma: no cover
_cc1d = None
USE_FAST = True
def have_fast(device):
return USE_FAST and device.type == "cuda" and _fla_chunk is not None and _cc1d is not None
def causal_conv1d(x, w, b, state=None, activation="silu"):
"""Depthwise causal conv with optional initial state (the last K-1 inputs of the previous segment)."""
K = w.shape[-1]
if have_fast(x.device):
if state is None:
y = _cc1d(x, w, b, activation=activation)
fin = F.pad(x, (K - 1 - x.shape[-1], 0))[:, :, -(K - 1):] if x.shape[-1] < K - 1 else x[:, :, -(K - 1):]
return y, fin.contiguous()
y, fin = _cc1d(x, w, b, initial_states=state, return_final_states=True, activation=activation)
return y, fin
xin = x if state is None else torch.cat([state.to(x.dtype), x], -1)
pad = (K - 1) if state is None else 0
y = F.conv1d(xin.to(w.dtype), w.unsqueeze(1), b.to(w.dtype) if b is not None else None, padding=pad, groups=x.shape[1])[:, :, : x.shape[-1]]
if activation == "silu":
y = F.silu(y)
fin = xin[:, :, -(K - 1):]
if fin.shape[-1] < K - 1:
fin = F.pad(fin, (K - 1 - fin.shape[-1], 0))
return y.to(x.dtype), fin.contiguous()
def _l2norm(x, eps=1e-6):
return x * torch.rsqrt((x * x).sum(-1, keepdim=True) + eps)
def gated_delta_rule(q, k, v, g, beta, state=None, chunk_size=64):
"""Chunked gated delta rule (port of the reference torch implementation; fp32; q/k l2-normalised; q scaled by D^-.5).
Returns output in q.dtype and the final recurrent state (fp32)."""
if have_fast(q.device):
o, s = _fla_chunk(q, k, v, g=g, beta=beta, initial_state=state, output_final_state=True, use_qk_l2norm_in_kernel=True)
return o, s
dt = q.dtype
B, T, H, Dk = k.shape
Dv = v.shape[-1]
q, k, v, beta, g = [x.transpose(1, 2).to(torch.float32).contiguous() for x in (q, k, v, beta, g)]
q = _l2norm(q) * (Dk ** -0.5); k = _l2norm(k)
pad = (chunk_size - T % chunk_size) % chunk_size
q, k, v = (F.pad(x, (0, 0, 0, pad)) for x in (q, k, v)); beta, g = (F.pad(x, (0, pad)) for x in (beta, g))
Tp = T + pad; n = Tp // chunk_size
v_beta = v * beta.unsqueeze(-1); k_beta = k * beta.unsqueeze(-1)
q, k, k_beta, v_beta = [x.reshape(B, H, n, chunk_size, x.shape[-1]) for x in (q, k, k_beta, v_beta)]
g = g.reshape(B, H, n, chunk_size)
up = torch.ones(chunk_size, chunk_size, dtype=torch.bool, device=q.device).triu(1)
cum = g.cumsum(3)
pw = (cum.unsqueeze(4) - cum.unsqueeze(3)).masked_fill(up, float("-inf")).exp()
ut = (k_beta @ k.transpose(-1, -2)) * pw
intra = (q @ k.transpose(-1, -2)) * pw
dk_beta = k_beta * cum.exp().unsqueeze(-1)
new_v = torch.linalg.solve_triangular(ut, v_beta, upper=False, unitriangular=True)
k_cum = torch.linalg.solve_triangular(ut, dk_beta, upper=False, unitriangular=True)
S = torch.zeros(B, H, Dk, Dv, dtype=torch.float32, device=q.device) if state is None else state.to(torch.float32)
out = torch.zeros_like(new_v)
q = q * cum.exp().unsqueeze(-1)
k = k * (cum[..., -1:] - cum).exp().unsqueeze(-1)
cd = cum[..., -1].exp()[..., None, None]
for i in range(n):
vn = new_v[:, :, i] - k_cum[:, :, i] @ S
out[:, :, i] = q[:, :, i] @ S + intra[:, :, i] @ vn
S = S * cd[:, :, i] + k[:, :, i].transpose(-1, -2) @ vn
out = out.reshape(B, H, Tp, Dv)[:, :, :T].transpose(1, 2).to(dt).contiguous()
return out, S