Julia Klee
Julia-2: julia2_audio_v1 (text = base_v2 byte-identical, + pruned audio tower, stem p9), int8/int4 variants, lean runtime 0.1.0 (audio, LiveSession, modalities), card with speech, live calls and variants
1d1c6db Download runtime/julia2/kernels.py from SupersonicLabs/Julia-2: direct link, hf CLI and curl.
- Browser
- Download file 4.28 kB
-
https://huggingface.co/SupersonicLabs/Julia-2/resolve/main/runtime/julia2/kernels.py
- Command line
-
hf download hf://SupersonicLabs/Julia-2/runtime/julia2/kernels.py
-
curl -L -o kernels.py https://huggingface.co/SupersonicLabs/Julia-2/resolve/main/runtime/julia2/kernels.py
4.28 kB
| """Fast kernels with pure-torch fallbacks (no CUDA-only code at import). | |
| causal_conv1d(x [B,C,T], w [C,K], b [C], state [B,C,K-1] | None) -> (y [B,C,T], final_state [B,C,K-1]) | |
| gated_delta_rule(q,k,v [B,T,H,D], g [B,T,H] fp32 log-decay, beta [B,T,H], state [B,H,Dk,Dv] | None) -> (o, final_state) | |
| On CUDA with `flash-linear-attention` / `causal-conv1d` installed the fused kernels are used (same maths as the | |
| reference training/eval code); otherwise the chunked torch implementation below (fp32) runs on any device. | |
| """ | |
| import torch | |
| import torch.nn.functional as F | |
| try: # optional fast kernels | |
| from fla.ops.gated_delta_rule import chunk_gated_delta_rule as _fla_chunk | |
| except Exception: # pragma: no cover | |
| _fla_chunk = None | |
| try: | |
| from causal_conv1d import causal_conv1d_fn as _cc1d | |
| except Exception: # pragma: no cover | |
| _cc1d = None | |
| USE_FAST = True | |
| def have_fast(device): | |
| return USE_FAST and device.type == "cuda" and _fla_chunk is not None and _cc1d is not None | |
| def causal_conv1d(x, w, b, state=None, activation="silu"): | |
| """Depthwise causal conv with optional initial state (the last K-1 inputs of the previous segment).""" | |
| K = w.shape[-1] | |
| if have_fast(x.device): | |
| if state is None: | |
| y = _cc1d(x, w, b, activation=activation) | |
| fin = F.pad(x, (K - 1 - x.shape[-1], 0))[:, :, -(K - 1):] if x.shape[-1] < K - 1 else x[:, :, -(K - 1):] | |
| return y, fin.contiguous() | |
| y, fin = _cc1d(x, w, b, initial_states=state, return_final_states=True, activation=activation) | |
| return y, fin | |
| xin = x if state is None else torch.cat([state.to(x.dtype), x], -1) | |
| pad = (K - 1) if state is None else 0 | |
| y = F.conv1d(xin.to(w.dtype), w.unsqueeze(1), b.to(w.dtype) if b is not None else None, padding=pad, groups=x.shape[1])[:, :, : x.shape[-1]] | |
| if activation == "silu": | |
| y = F.silu(y) | |
| fin = xin[:, :, -(K - 1):] | |
| if fin.shape[-1] < K - 1: | |
| fin = F.pad(fin, (K - 1 - fin.shape[-1], 0)) | |
| return y.to(x.dtype), fin.contiguous() | |
| def _l2norm(x, eps=1e-6): | |
| return x * torch.rsqrt((x * x).sum(-1, keepdim=True) + eps) | |
| def gated_delta_rule(q, k, v, g, beta, state=None, chunk_size=64): | |
| """Chunked gated delta rule (port of the reference torch implementation; fp32; q/k l2-normalised; q scaled by D^-.5). | |
| Returns output in q.dtype and the final recurrent state (fp32).""" | |
| if have_fast(q.device): | |
| o, s = _fla_chunk(q, k, v, g=g, beta=beta, initial_state=state, output_final_state=True, use_qk_l2norm_in_kernel=True) | |
| return o, s | |
| dt = q.dtype | |
| B, T, H, Dk = k.shape | |
| Dv = v.shape[-1] | |
| q, k, v, beta, g = [x.transpose(1, 2).to(torch.float32).contiguous() for x in (q, k, v, beta, g)] | |
| q = _l2norm(q) * (Dk ** -0.5); k = _l2norm(k) | |
| pad = (chunk_size - T % chunk_size) % chunk_size | |
| q, k, v = (F.pad(x, (0, 0, 0, pad)) for x in (q, k, v)); beta, g = (F.pad(x, (0, pad)) for x in (beta, g)) | |
| Tp = T + pad; n = Tp // chunk_size | |
| v_beta = v * beta.unsqueeze(-1); k_beta = k * beta.unsqueeze(-1) | |
| q, k, k_beta, v_beta = [x.reshape(B, H, n, chunk_size, x.shape[-1]) for x in (q, k, k_beta, v_beta)] | |
| g = g.reshape(B, H, n, chunk_size) | |
| up = torch.ones(chunk_size, chunk_size, dtype=torch.bool, device=q.device).triu(1) | |
| cum = g.cumsum(3) | |
| pw = (cum.unsqueeze(4) - cum.unsqueeze(3)).masked_fill(up, float("-inf")).exp() | |
| ut = (k_beta @ k.transpose(-1, -2)) * pw | |
| intra = (q @ k.transpose(-1, -2)) * pw | |
| dk_beta = k_beta * cum.exp().unsqueeze(-1) | |
| new_v = torch.linalg.solve_triangular(ut, v_beta, upper=False, unitriangular=True) | |
| k_cum = torch.linalg.solve_triangular(ut, dk_beta, upper=False, unitriangular=True) | |
| S = torch.zeros(B, H, Dk, Dv, dtype=torch.float32, device=q.device) if state is None else state.to(torch.float32) | |
| out = torch.zeros_like(new_v) | |
| q = q * cum.exp().unsqueeze(-1) | |
| k = k * (cum[..., -1:] - cum).exp().unsqueeze(-1) | |
| cd = cum[..., -1].exp()[..., None, None] | |
| for i in range(n): | |
| vn = new_v[:, :, i] - k_cum[:, :, i] @ S | |
| out[:, :, i] = q[:, :, i] @ S + intra[:, :, i] @ vn | |
| S = S * cd[:, :, i] + k[:, :, i].transpose(-1, -2) @ vn | |
| out = out.reshape(B, H, Tp, Dv)[:, :, :T].transpose(1, 2).to(dt).contiguous() | |
| return out, S | |