"""Fast kernels with pure-torch fallbacks (no CUDA-only code at import). causal_conv1d(x [B,C,T], w [C,K], b [C], state [B,C,K-1] | None) -> (y [B,C,T], final_state [B,C,K-1]) gated_delta_rule(q,k,v [B,T,H,D], g [B,T,H] fp32 log-decay, beta [B,T,H], state [B,H,Dk,Dv] | None) -> (o, final_state) On CUDA with `flash-linear-attention` / `causal-conv1d` installed the fused kernels are used (same maths as the reference training/eval code); otherwise the chunked torch implementation below (fp32) runs on any device. """ import torch import torch.nn.functional as F try: # optional fast kernels from fla.ops.gated_delta_rule import chunk_gated_delta_rule as _fla_chunk except Exception: # pragma: no cover _fla_chunk = None try: from causal_conv1d import causal_conv1d_fn as _cc1d except Exception: # pragma: no cover _cc1d = None USE_FAST = True def have_fast(device): return USE_FAST and device.type == "cuda" and _fla_chunk is not None and _cc1d is not None def causal_conv1d(x, w, b, state=None, activation="silu"): """Depthwise causal conv with optional initial state (the last K-1 inputs of the previous segment).""" K = w.shape[-1] if have_fast(x.device): if state is None: y = _cc1d(x, w, b, activation=activation) fin = F.pad(x, (K - 1 - x.shape[-1], 0))[:, :, -(K - 1):] if x.shape[-1] < K - 1 else x[:, :, -(K - 1):] return y, fin.contiguous() y, fin = _cc1d(x, w, b, initial_states=state, return_final_states=True, activation=activation) return y, fin xin = x if state is None else torch.cat([state.to(x.dtype), x], -1) pad = (K - 1) if state is None else 0 y = F.conv1d(xin.to(w.dtype), w.unsqueeze(1), b.to(w.dtype) if b is not None else None, padding=pad, groups=x.shape[1])[:, :, : x.shape[-1]] if activation == "silu": y = F.silu(y) fin = xin[:, :, -(K - 1):] if fin.shape[-1] < K - 1: fin = F.pad(fin, (K - 1 - fin.shape[-1], 0)) return y.to(x.dtype), fin.contiguous() def _l2norm(x, eps=1e-6): return x * torch.rsqrt((x * x).sum(-1, keepdim=True) + eps) def gated_delta_rule(q, k, v, g, beta, state=None, chunk_size=64): """Chunked gated delta rule (port of the reference torch implementation; fp32; q/k l2-normalised; q scaled by D^-.5). Returns output in q.dtype and the final recurrent state (fp32).""" if have_fast(q.device): o, s = _fla_chunk(q, k, v, g=g, beta=beta, initial_state=state, output_final_state=True, use_qk_l2norm_in_kernel=True) return o, s dt = q.dtype B, T, H, Dk = k.shape Dv = v.shape[-1] q, k, v, beta, g = [x.transpose(1, 2).to(torch.float32).contiguous() for x in (q, k, v, beta, g)] q = _l2norm(q) * (Dk ** -0.5); k = _l2norm(k) pad = (chunk_size - T % chunk_size) % chunk_size q, k, v = (F.pad(x, (0, 0, 0, pad)) for x in (q, k, v)); beta, g = (F.pad(x, (0, pad)) for x in (beta, g)) Tp = T + pad; n = Tp // chunk_size v_beta = v * beta.unsqueeze(-1); k_beta = k * beta.unsqueeze(-1) q, k, k_beta, v_beta = [x.reshape(B, H, n, chunk_size, x.shape[-1]) for x in (q, k, k_beta, v_beta)] g = g.reshape(B, H, n, chunk_size) up = torch.ones(chunk_size, chunk_size, dtype=torch.bool, device=q.device).triu(1) cum = g.cumsum(3) pw = (cum.unsqueeze(4) - cum.unsqueeze(3)).masked_fill(up, float("-inf")).exp() ut = (k_beta @ k.transpose(-1, -2)) * pw intra = (q @ k.transpose(-1, -2)) * pw dk_beta = k_beta * cum.exp().unsqueeze(-1) new_v = torch.linalg.solve_triangular(ut, v_beta, upper=False, unitriangular=True) k_cum = torch.linalg.solve_triangular(ut, dk_beta, upper=False, unitriangular=True) S = torch.zeros(B, H, Dk, Dv, dtype=torch.float32, device=q.device) if state is None else state.to(torch.float32) out = torch.zeros_like(new_v) q = q * cum.exp().unsqueeze(-1) k = k * (cum[..., -1:] - cum).exp().unsqueeze(-1) cd = cum[..., -1].exp()[..., None, None] for i in range(n): vn = new_v[:, :, i] - k_cum[:, :, i] @ S out[:, :, i] = q[:, :, i] @ S + intra[:, :, i] @ vn S = S * cd[:, :, i] + k[:, :, i].transpose(-1, -2) @ vn out = out.reshape(B, H, Tp, Dv)[:, :, :T].transpose(1, 2).to(dt).contiguous() return out, S