Julia-2-Flash / runtime /julia2 /kernels.py
vianahnz's picture
Julia 2 Flash
b61b4c4 verified
Raw History Blame Contribute Delete
4.82 kB
"""Fast kernels with pure-torch fallbacks (no CUDA-only code at import).
causal_conv1d(x [B,C,T], w [C,K], b [C], state [B,C,K-1] | None) -> (y [B,C,T], final_state [B,C,K-1])
gated_delta_rule(q,k,v [B,T,H,D], g [B,T,H] fp32 log-decay, beta [B,T,H], state [B,H,Dk,Dv] | None) -> (o, final_state)
On CUDA with `flash-linear-attention` / `causal-conv1d` installed the fused kernels are used (same maths as the
reference training/eval code); otherwise the chunked torch implementation below (fp32) runs on any device.
"""
import torch
import torch.nn.functional as F
try: # optional fast kernels
from fla.ops.gated_delta_rule import chunk_gated_delta_rule as _fla_chunk
except Exception: # pragma: no cover
_fla_chunk = None
try:
from causal_conv1d import causal_conv1d_fn as _cc1d
except Exception: # pragma: no cover
_cc1d = None
USE_FAST = True
def have_fast(device):
return USE_FAST and device.type == "cuda" and _fla_chunk is not None and _cc1d is not None
def causal_conv1d(x, w, b, state=None, activation="silu"):
"""Depthwise causal conv with optional initial state (the last K-1 inputs of the previous segment)."""
K = w.shape[-1]
if have_fast(x.device):
if state is None:
y = _cc1d(x, w, b, activation=activation)
fin = F.pad(x, (K - 1 - x.shape[-1], 0))[:, :, -(K - 1):] if x.shape[-1] < K - 1 else x[:, :, -(K - 1):]
return y, fin.contiguous()
y, fin = _cc1d(x, w, b, initial_states=state, return_final_states=True, activation=activation)
return y, fin
# Depthwise causal conv as K shifted multiply-adds: the grouped F.conv1d runs one channel at a time on the CPU
# (80 % of a CPU forward); this is the same sum, y[t] = b + sum_k w[k] * x[t - K + 1 + k].
# Work channel-last ([B,T,C]): x and the state buffers are transposed views of channel-last memory, and a
# concatenation along T in the [B,C,T] view costs 17x more on the CPU.
xt = x.transpose(1, 2)
xin = xt if state is None else torch.cat([state.transpose(1, 2).to(x.dtype), xt], 1)
xin_p = F.pad(xin, (0, 0, K - 1, 0)) if state is None else xin
T = xt.shape[1]
xw = xin_p.to(w.dtype); wt = w.t()
y = xw[:, :T] * wt[0]
for k in range(1, K):
y = y + xw[:, k:k + T] * wt[k]
if b is not None:
y = y + b.to(w.dtype)
if activation == "silu":
y = F.silu(y)
fin = xin[:, -(K - 1):]
if fin.shape[1] < K - 1:
fin = F.pad(fin, (0, 0, K - 1 - fin.shape[1], 0))
return y.to(x.dtype).transpose(1, 2), fin.transpose(1, 2)
def _l2norm(x, eps=1e-6):
return x * torch.rsqrt((x * x).sum(-1, keepdim=True) + eps)
def gated_delta_rule(q, k, v, g, beta, state=None, chunk_size=64):
"""Chunked gated delta rule (port of the reference torch implementation; fp32; q/k l2-normalised; q scaled by D^-.5).
Returns output in q.dtype and the final recurrent state (fp32)."""
if have_fast(q.device):
o, s = _fla_chunk(q, k, v, g=g, beta=beta, initial_state=state, output_final_state=True, use_qk_l2norm_in_kernel=True)
return o, s
dt = q.dtype
B, T, H, Dk = k.shape
Dv = v.shape[-1]
q, k, v, beta, g = [x.transpose(1, 2).to(torch.float32).contiguous() for x in (q, k, v, beta, g)]
q = _l2norm(q) * (Dk ** -0.5); k = _l2norm(k)
pad = (chunk_size - T % chunk_size) % chunk_size
q, k, v = (F.pad(x, (0, 0, 0, pad)) for x in (q, k, v)); beta, g = (F.pad(x, (0, pad)) for x in (beta, g))
Tp = T + pad; n = Tp // chunk_size
v_beta = v * beta.unsqueeze(-1); k_beta = k * beta.unsqueeze(-1)
q, k, k_beta, v_beta = [x.reshape(B, H, n, chunk_size, x.shape[-1]) for x in (q, k, k_beta, v_beta)]
g = g.reshape(B, H, n, chunk_size)
up = torch.ones(chunk_size, chunk_size, dtype=torch.bool, device=q.device).triu(1)
cum = g.cumsum(3)
pw = (cum.unsqueeze(4) - cum.unsqueeze(3)).masked_fill(up, float("-inf")).exp()
ut = (k_beta @ k.transpose(-1, -2)) * pw
intra = (q @ k.transpose(-1, -2)) * pw
dk_beta = k_beta * cum.exp().unsqueeze(-1)
new_v = torch.linalg.solve_triangular(ut, v_beta, upper=False, unitriangular=True)
k_cum = torch.linalg.solve_triangular(ut, dk_beta, upper=False, unitriangular=True)
S = torch.zeros(B, H, Dk, Dv, dtype=torch.float32, device=q.device) if state is None else state.to(torch.float32)
out = torch.zeros_like(new_v)
q = q * cum.exp().unsqueeze(-1)
k = k * (cum[..., -1:] - cum).exp().unsqueeze(-1)
cd = cum[..., -1].exp()[..., None, None]
for i in range(n):
vn = new_v[:, :, i] - k_cum[:, :, i] @ S
out[:, :, i] = q[:, :, i] @ S + intra[:, :, i] @ vn
S = S * cd[:, :, i] + k[:, :, i].transpose(-1, -2) @ vn
out = out.reshape(B, H, Tp, Dv)[:, :, :T].transpose(1, 2).to(dt).contiguous()
return out, S