Uploaded using `kernel-builder`.
Browse files- build/torch-cuda/__init__.py +140 -0
- build/torch-cuda/_ops.py +38 -0
- build/torch-cuda/ldsa/__init__.py +26 -0
- build/torch-cuda/metadata.json +29 -0
build/torch-cuda/__init__.py
ADDED
|
@@ -0,0 +1,140 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""ldsa — fused Local Dense Synthesizer Attention window op as a noarch Triton kernel.
|
| 2 |
+
|
| 3 |
+
Given synthesized per-position window logits `a_logits [B, H, W, T]` and values
|
| 4 |
+
`v [B, H, Dh, T]` with a window `(left, right)`, `W = left + right + 1`:
|
| 5 |
+
|
| 6 |
+
a[b,h,k,t] = softmax over k of a_logits[b,h,:,t] # softmax across the window
|
| 7 |
+
out[b,h,d,t] = sum_k a[b,h,k,t] * v[b,h,d, t-left+k] # weighted sum; zero outside [0,T)
|
| 8 |
+
|
| 9 |
+
i.e. local attention whose weights are SYNTHESIZED per frame (an MLP over the input) rather
|
| 10 |
+
than from query-key dot products (Synthesizer, arXiv 2005.00743). The kernel fuses the softmax
|
| 11 |
+
+ windowed weighted-sum into one launch (fp32 accumulation); it does NOT include the weight/
|
| 12 |
+
value/output projections — the caller supplies `a_logits` and `v`.
|
| 13 |
+
|
| 14 |
+
from kernels import get_kernel
|
| 15 |
+
k = get_kernel("futo-org/ldsa", version=1)
|
| 16 |
+
out = k.ldsa_local_attention(a_logits, v, left, right) # [B,H,Dh,T]; contiguous, any float
|
| 17 |
+
|
| 18 |
+
Inference kernel (no backward): the Triton path runs only on CUDA under `torch.no_grad()`;
|
| 19 |
+
grad-enabled / CPU calls take the autograd-safe `eager_ldsa` reference (also exported).
|
| 20 |
+
|
| 21 |
+
Reference: M. Xu, S. Li, X.-L. Zhang, "Transformer-based End-to-End Speech Recognition with
|
| 22 |
+
Local Dense Synthesizer Attention," ICASSP 2021 (arXiv:2010.12155;
|
| 23 |
+
github.com/mlxu995/multihead-LDSA). Differences from that reference are in the model card.
|
| 24 |
+
"""
|
| 25 |
+
|
| 26 |
+
from __future__ import annotations
|
| 27 |
+
|
| 28 |
+
import torch
|
| 29 |
+
import torch.nn.functional as F
|
| 30 |
+
|
| 31 |
+
# Hard dependency: this module IS the Triton kernel path (imported only when GPU kernels are
|
| 32 |
+
# opted into). A missing triton must fail LOUDLY here, not silently degrade to eager (which
|
| 33 |
+
# hides a broken serving setup). The eager reference stays for the legitimate RUNTIME fallbacks
|
| 34 |
+
# (CPU / grad-enabled), which are correctness, not error-hiding.
|
| 35 |
+
import triton
|
| 36 |
+
import triton.language as tl
|
| 37 |
+
|
| 38 |
+
|
| 39 |
+
def eager_ldsa(a_logits: torch.Tensor, v: torch.Tensor, left: int, right: int) -> torch.Tensor:
|
| 40 |
+
"""Autograd-safe reference — bit-matches LocalSynthAttention's softmax+pad+window loop.
|
| 41 |
+
a_logits [B,H,W,T], v [B,H,Dh,T] -> out [B,H,Dh,T]."""
|
| 42 |
+
a = a_logits.softmax(dim=2)
|
| 43 |
+
t = v.shape[3]
|
| 44 |
+
vp = F.pad(v, (left, right))
|
| 45 |
+
out = a[:, :, 0:1, :] * vp[:, :, :, 0:t]
|
| 46 |
+
for k in range(1, a.shape[2]):
|
| 47 |
+
out = out + a[:, :, k : k + 1, :] * vp[:, :, :, k : k + t]
|
| 48 |
+
return out
|
| 49 |
+
|
| 50 |
+
|
| 51 |
+
def _configs():
|
| 52 |
+
return [triton.Config({"BLOCK_T": bt}, num_warps=nw) for bt in (64, 128, 256) for nw in (4, 8)]
|
| 53 |
+
|
| 54 |
+
|
| 55 |
+
@triton.autotune(configs=_configs(), key=["T", "Dh", "W"])
|
| 56 |
+
@triton.jit
|
| 57 |
+
def _ldsa_kernel(
|
| 58 |
+
a_ptr,
|
| 59 |
+
v_ptr,
|
| 60 |
+
out_ptr,
|
| 61 |
+
T,
|
| 62 |
+
Dh,
|
| 63 |
+
left,
|
| 64 |
+
stride_a_bh,
|
| 65 |
+
stride_v_bh,
|
| 66 |
+
W: tl.constexpr,
|
| 67 |
+
BLOCK_T: tl.constexpr,
|
| 68 |
+
BLOCK_DH: tl.constexpr,
|
| 69 |
+
BLOCK_W: tl.constexpr,
|
| 70 |
+
):
|
| 71 |
+
bh = tl.program_id(0)
|
| 72 |
+
offs_t = tl.program_id(1) * BLOCK_T + tl.arange(0, BLOCK_T) # output frames
|
| 73 |
+
mt = offs_t < T
|
| 74 |
+
a_base = a_ptr + bh * stride_a_bh
|
| 75 |
+
|
| 76 |
+
# softmax stats over the window (axis 0), one max/sum per output column
|
| 77 |
+
offs_w = tl.arange(0, BLOCK_W)
|
| 78 |
+
a_tile = tl.load(
|
| 79 |
+
a_base + offs_w[:, None] * T + offs_t[None, :],
|
| 80 |
+
mask=(offs_w < W)[:, None] & mt[None, :],
|
| 81 |
+
other=float("-inf"),
|
| 82 |
+
).to(tl.float32) # [BLOCK_W, BLOCK_T]
|
| 83 |
+
m = tl.max(a_tile, axis=0) # [BLOCK_T]
|
| 84 |
+
z = tl.sum(tl.exp(a_tile - m[None, :]), axis=0) # [BLOCK_T]
|
| 85 |
+
|
| 86 |
+
# accumulate the windowed weighted sum into a [Dh, BLOCK_T] tile
|
| 87 |
+
offs_dh = tl.arange(0, BLOCK_DH)
|
| 88 |
+
md = offs_dh < Dh
|
| 89 |
+
v_base = v_ptr + bh * stride_v_bh
|
| 90 |
+
acc = tl.zeros((BLOCK_DH, BLOCK_T), dtype=tl.float32)
|
| 91 |
+
for k in tl.static_range(W):
|
| 92 |
+
a_k = tl.load(a_base + k * T + offs_t, mask=mt, other=float("-inf")).to(tl.float32)
|
| 93 |
+
wk = tl.exp(a_k - m) / z # softmax weight for window pos k, [BLOCK_T]
|
| 94 |
+
src = offs_t - left + k # source frame per output column
|
| 95 |
+
vk = tl.load(
|
| 96 |
+
v_base + offs_dh[:, None] * T + src[None, :],
|
| 97 |
+
mask=md[:, None] & (src >= 0)[None, :] & (src < T)[None, :] & mt[None, :],
|
| 98 |
+
other=0.0,
|
| 99 |
+
).to(tl.float32) # [BLOCK_DH, BLOCK_T]
|
| 100 |
+
acc += wk[None, :] * vk
|
| 101 |
+
tl.store(
|
| 102 |
+
out_ptr + bh * stride_v_bh + offs_dh[:, None] * T + offs_t[None, :],
|
| 103 |
+
acc,
|
| 104 |
+
mask=md[:, None] & mt[None, :],
|
| 105 |
+
)
|
| 106 |
+
|
| 107 |
+
|
| 108 |
+
def _triton_ldsa(a_logits: torch.Tensor, v: torch.Tensor, left: int) -> torch.Tensor:
|
| 109 |
+
b, h, w, t = a_logits.shape
|
| 110 |
+
dh = v.shape[2]
|
| 111 |
+
a_c, v_c = a_logits.contiguous(), v.contiguous()
|
| 112 |
+
out = torch.empty_like(v_c)
|
| 113 |
+
grid = lambda meta: (b * h, triton.cdiv(t, meta["BLOCK_T"])) # noqa: E731
|
| 114 |
+
_ldsa_kernel[grid](
|
| 115 |
+
a_c,
|
| 116 |
+
v_c,
|
| 117 |
+
out,
|
| 118 |
+
t,
|
| 119 |
+
dh,
|
| 120 |
+
left,
|
| 121 |
+
w * t,
|
| 122 |
+
dh * t,
|
| 123 |
+
W=w,
|
| 124 |
+
BLOCK_DH=triton.next_power_of_2(dh),
|
| 125 |
+
BLOCK_W=triton.next_power_of_2(w),
|
| 126 |
+
)
|
| 127 |
+
return out
|
| 128 |
+
|
| 129 |
+
|
| 130 |
+
def ldsa_local_attention(
|
| 131 |
+
a_logits: torch.Tensor, v: torch.Tensor, left: int, right: int
|
| 132 |
+
) -> torch.Tensor:
|
| 133 |
+
"""Fused LDSA window op; Triton on CUDA under no_grad, else the autograd-safe eager
|
| 134 |
+
reference (training, CPU). a_logits [B,H,W,T], v [B,H,Dh,T] -> [B,H,Dh,T]."""
|
| 135 |
+
if v.is_cuda and not torch.is_grad_enabled():
|
| 136 |
+
return _triton_ldsa(a_logits, v, left)
|
| 137 |
+
return eager_ldsa(a_logits, v, left, right)
|
| 138 |
+
|
| 139 |
+
|
| 140 |
+
__all__ = ["ldsa_local_attention", "eager_ldsa"]
|
build/torch-cuda/_ops.py
ADDED
|
@@ -0,0 +1,38 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import torch
|
| 2 |
+
|
| 3 |
+
def get_backend() -> str:
|
| 4 |
+
"""Detect the backend by inspecting torch."""
|
| 5 |
+
import torch
|
| 6 |
+
|
| 7 |
+
if hasattr(torch, "neuron"):
|
| 8 |
+
# Needs to be sorted before specific Torch builds, since Neuron
|
| 9 |
+
# extension can be loaded into e.g. CUDA Torch builds.
|
| 10 |
+
return "neuron"
|
| 11 |
+
elif torch.version.cuda is not None:
|
| 12 |
+
return "cuda"
|
| 13 |
+
elif torch.version.hip is not None:
|
| 14 |
+
return "rocm"
|
| 15 |
+
elif torch.backends.mps.is_available():
|
| 16 |
+
return "metal"
|
| 17 |
+
elif hasattr(torch.version, "xpu") and torch.version.xpu is not None:
|
| 18 |
+
return "xpu"
|
| 19 |
+
else:
|
| 20 |
+
return "cpu"
|
| 21 |
+
|
| 22 |
+
|
| 23 |
+
def _find_ops_name() -> str:
|
| 24 |
+
kernel_name = "ldsa"
|
| 25 |
+
unique_id = "9763df8"
|
| 26 |
+
backend = get_backend()
|
| 27 |
+
return f"_{kernel_name}_{backend}_{unique_id}"
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
_OPS_NAME = _find_ops_name()
|
| 31 |
+
|
| 32 |
+
ops = getattr(torch.ops, _OPS_NAME)
|
| 33 |
+
|
| 34 |
+
def add_op_namespace_prefix(op_name: str) -> str:
|
| 35 |
+
"""
|
| 36 |
+
Prefix op by namespace.
|
| 37 |
+
"""
|
| 38 |
+
return f"{_OPS_NAME}::{op_name}"
|
build/torch-cuda/ldsa/__init__.py
ADDED
|
@@ -0,0 +1,26 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import ctypes
|
| 2 |
+
import importlib.util
|
| 3 |
+
import sys
|
| 4 |
+
from pathlib import Path
|
| 5 |
+
from types import ModuleType
|
| 6 |
+
|
| 7 |
+
|
| 8 |
+
def _import_from_path(file_path: Path) -> ModuleType:
|
| 9 |
+
# We cannot use the module name as-is, after adding it to `sys.modules`,
|
| 10 |
+
# it would also be used for other imports. So, we make a module name that
|
| 11 |
+
# depends on the path for it to be unique using the hex-encoded hash of
|
| 12 |
+
# the path.
|
| 13 |
+
path_hash = "{:x}".format(ctypes.c_size_t(hash(file_path.absolute())).value)
|
| 14 |
+
module_name = path_hash
|
| 15 |
+
spec = importlib.util.spec_from_file_location(module_name, file_path)
|
| 16 |
+
if spec is None:
|
| 17 |
+
raise ImportError(f"Cannot load spec for {module_name} from {file_path}")
|
| 18 |
+
module = importlib.util.module_from_spec(spec)
|
| 19 |
+
if module is None:
|
| 20 |
+
raise ImportError(f"Cannot load module {module_name} from spec")
|
| 21 |
+
sys.modules[module_name] = module
|
| 22 |
+
spec.loader.exec_module(module) # type: ignore
|
| 23 |
+
return module
|
| 24 |
+
|
| 25 |
+
|
| 26 |
+
globals().update(vars(_import_from_path(Path(__file__).parent.parent / "__init__.py")))
|
build/torch-cuda/metadata.json
ADDED
|
@@ -0,0 +1,29 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"name": "ldsa",
|
| 3 |
+
"id": "_ldsa_cuda_9763df8",
|
| 4 |
+
"version": 1,
|
| 5 |
+
"license": "Apache-2.0",
|
| 6 |
+
"python-depends": [],
|
| 7 |
+
"backend": {
|
| 8 |
+
"type": "cuda"
|
| 9 |
+
},
|
| 10 |
+
"digest": {
|
| 11 |
+
"algorithm": "sha256",
|
| 12 |
+
"files": {
|
| 13 |
+
"__init__.py": "NEgktlyHTKkRCwaMYX8w/ekaSbpjqLPHVmWtBTlDe6Q=",
|
| 14 |
+
"_ops.py": "KddM++gF3qGbBFs9PMP3HLS6KFeK8NEJWiwEH4aX15Y=",
|
| 15 |
+
"ldsa/__init__.py": "DFYPlrhXwYjEqCl/8n0SmWGZV8NFml5DPhMjKfv98GY="
|
| 16 |
+
}
|
| 17 |
+
},
|
| 18 |
+
"provenance": {
|
| 19 |
+
"kernel-builder": {
|
| 20 |
+
"version": "0.17.0-dev0",
|
| 21 |
+
"sha": "7262b99a509c116a4661e07eba15663ae19cddb5",
|
| 22 |
+
"dirty": false
|
| 23 |
+
},
|
| 24 |
+
"kernel": {
|
| 25 |
+
"sha": "9763df80731b1bd5b031b311793df01d91f59554",
|
| 26 |
+
"dirty": false
|
| 27 |
+
}
|
| 28 |
+
}
|
| 29 |
+
}
|