kernel
triton
lee-futo commited on
Commit
47d2a94
·
verified ·
1 Parent(s): a98583b

Uploaded using `kernel-builder`.

Browse files
build/torch-cuda/__init__.py ADDED
@@ -0,0 +1,140 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """ldsa — fused Local Dense Synthesizer Attention window op as a noarch Triton kernel.
2
+
3
+ Given synthesized per-position window logits `a_logits [B, H, W, T]` and values
4
+ `v [B, H, Dh, T]` with a window `(left, right)`, `W = left + right + 1`:
5
+
6
+ a[b,h,k,t] = softmax over k of a_logits[b,h,:,t] # softmax across the window
7
+ out[b,h,d,t] = sum_k a[b,h,k,t] * v[b,h,d, t-left+k] # weighted sum; zero outside [0,T)
8
+
9
+ i.e. local attention whose weights are SYNTHESIZED per frame (an MLP over the input) rather
10
+ than from query-key dot products (Synthesizer, arXiv 2005.00743). The kernel fuses the softmax
11
+ + windowed weighted-sum into one launch (fp32 accumulation); it does NOT include the weight/
12
+ value/output projections — the caller supplies `a_logits` and `v`.
13
+
14
+ from kernels import get_kernel
15
+ k = get_kernel("futo-org/ldsa", version=1)
16
+ out = k.ldsa_local_attention(a_logits, v, left, right) # [B,H,Dh,T]; contiguous, any float
17
+
18
+ Inference kernel (no backward): the Triton path runs only on CUDA under `torch.no_grad()`;
19
+ grad-enabled / CPU calls take the autograd-safe `eager_ldsa` reference (also exported).
20
+
21
+ Reference: M. Xu, S. Li, X.-L. Zhang, "Transformer-based End-to-End Speech Recognition with
22
+ Local Dense Synthesizer Attention," ICASSP 2021 (arXiv:2010.12155;
23
+ github.com/mlxu995/multihead-LDSA). Differences from that reference are in the model card.
24
+ """
25
+
26
+ from __future__ import annotations
27
+
28
+ import torch
29
+ import torch.nn.functional as F
30
+
31
+ # Hard dependency: this module IS the Triton kernel path (imported only when GPU kernels are
32
+ # opted into). A missing triton must fail LOUDLY here, not silently degrade to eager (which
33
+ # hides a broken serving setup). The eager reference stays for the legitimate RUNTIME fallbacks
34
+ # (CPU / grad-enabled), which are correctness, not error-hiding.
35
+ import triton
36
+ import triton.language as tl
37
+
38
+
39
+ def eager_ldsa(a_logits: torch.Tensor, v: torch.Tensor, left: int, right: int) -> torch.Tensor:
40
+ """Autograd-safe reference — bit-matches LocalSynthAttention's softmax+pad+window loop.
41
+ a_logits [B,H,W,T], v [B,H,Dh,T] -> out [B,H,Dh,T]."""
42
+ a = a_logits.softmax(dim=2)
43
+ t = v.shape[3]
44
+ vp = F.pad(v, (left, right))
45
+ out = a[:, :, 0:1, :] * vp[:, :, :, 0:t]
46
+ for k in range(1, a.shape[2]):
47
+ out = out + a[:, :, k : k + 1, :] * vp[:, :, :, k : k + t]
48
+ return out
49
+
50
+
51
+ def _configs():
52
+ return [triton.Config({"BLOCK_T": bt}, num_warps=nw) for bt in (64, 128, 256) for nw in (4, 8)]
53
+
54
+
55
+ @triton.autotune(configs=_configs(), key=["T", "Dh", "W"])
56
+ @triton.jit
57
+ def _ldsa_kernel(
58
+ a_ptr,
59
+ v_ptr,
60
+ out_ptr,
61
+ T,
62
+ Dh,
63
+ left,
64
+ stride_a_bh,
65
+ stride_v_bh,
66
+ W: tl.constexpr,
67
+ BLOCK_T: tl.constexpr,
68
+ BLOCK_DH: tl.constexpr,
69
+ BLOCK_W: tl.constexpr,
70
+ ):
71
+ bh = tl.program_id(0)
72
+ offs_t = tl.program_id(1) * BLOCK_T + tl.arange(0, BLOCK_T) # output frames
73
+ mt = offs_t < T
74
+ a_base = a_ptr + bh * stride_a_bh
75
+
76
+ # softmax stats over the window (axis 0), one max/sum per output column
77
+ offs_w = tl.arange(0, BLOCK_W)
78
+ a_tile = tl.load(
79
+ a_base + offs_w[:, None] * T + offs_t[None, :],
80
+ mask=(offs_w < W)[:, None] & mt[None, :],
81
+ other=float("-inf"),
82
+ ).to(tl.float32) # [BLOCK_W, BLOCK_T]
83
+ m = tl.max(a_tile, axis=0) # [BLOCK_T]
84
+ z = tl.sum(tl.exp(a_tile - m[None, :]), axis=0) # [BLOCK_T]
85
+
86
+ # accumulate the windowed weighted sum into a [Dh, BLOCK_T] tile
87
+ offs_dh = tl.arange(0, BLOCK_DH)
88
+ md = offs_dh < Dh
89
+ v_base = v_ptr + bh * stride_v_bh
90
+ acc = tl.zeros((BLOCK_DH, BLOCK_T), dtype=tl.float32)
91
+ for k in tl.static_range(W):
92
+ a_k = tl.load(a_base + k * T + offs_t, mask=mt, other=float("-inf")).to(tl.float32)
93
+ wk = tl.exp(a_k - m) / z # softmax weight for window pos k, [BLOCK_T]
94
+ src = offs_t - left + k # source frame per output column
95
+ vk = tl.load(
96
+ v_base + offs_dh[:, None] * T + src[None, :],
97
+ mask=md[:, None] & (src >= 0)[None, :] & (src < T)[None, :] & mt[None, :],
98
+ other=0.0,
99
+ ).to(tl.float32) # [BLOCK_DH, BLOCK_T]
100
+ acc += wk[None, :] * vk
101
+ tl.store(
102
+ out_ptr + bh * stride_v_bh + offs_dh[:, None] * T + offs_t[None, :],
103
+ acc,
104
+ mask=md[:, None] & mt[None, :],
105
+ )
106
+
107
+
108
+ def _triton_ldsa(a_logits: torch.Tensor, v: torch.Tensor, left: int) -> torch.Tensor:
109
+ b, h, w, t = a_logits.shape
110
+ dh = v.shape[2]
111
+ a_c, v_c = a_logits.contiguous(), v.contiguous()
112
+ out = torch.empty_like(v_c)
113
+ grid = lambda meta: (b * h, triton.cdiv(t, meta["BLOCK_T"])) # noqa: E731
114
+ _ldsa_kernel[grid](
115
+ a_c,
116
+ v_c,
117
+ out,
118
+ t,
119
+ dh,
120
+ left,
121
+ w * t,
122
+ dh * t,
123
+ W=w,
124
+ BLOCK_DH=triton.next_power_of_2(dh),
125
+ BLOCK_W=triton.next_power_of_2(w),
126
+ )
127
+ return out
128
+
129
+
130
+ def ldsa_local_attention(
131
+ a_logits: torch.Tensor, v: torch.Tensor, left: int, right: int
132
+ ) -> torch.Tensor:
133
+ """Fused LDSA window op; Triton on CUDA under no_grad, else the autograd-safe eager
134
+ reference (training, CPU). a_logits [B,H,W,T], v [B,H,Dh,T] -> [B,H,Dh,T]."""
135
+ if v.is_cuda and not torch.is_grad_enabled():
136
+ return _triton_ldsa(a_logits, v, left)
137
+ return eager_ldsa(a_logits, v, left, right)
138
+
139
+
140
+ __all__ = ["ldsa_local_attention", "eager_ldsa"]
build/torch-cuda/_ops.py ADDED
@@ -0,0 +1,38 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import torch
2
+
3
+ def get_backend() -> str:
4
+ """Detect the backend by inspecting torch."""
5
+ import torch
6
+
7
+ if hasattr(torch, "neuron"):
8
+ # Needs to be sorted before specific Torch builds, since Neuron
9
+ # extension can be loaded into e.g. CUDA Torch builds.
10
+ return "neuron"
11
+ elif torch.version.cuda is not None:
12
+ return "cuda"
13
+ elif torch.version.hip is not None:
14
+ return "rocm"
15
+ elif torch.backends.mps.is_available():
16
+ return "metal"
17
+ elif hasattr(torch.version, "xpu") and torch.version.xpu is not None:
18
+ return "xpu"
19
+ else:
20
+ return "cpu"
21
+
22
+
23
+ def _find_ops_name() -> str:
24
+ kernel_name = "ldsa"
25
+ unique_id = "9763df8"
26
+ backend = get_backend()
27
+ return f"_{kernel_name}_{backend}_{unique_id}"
28
+
29
+
30
+ _OPS_NAME = _find_ops_name()
31
+
32
+ ops = getattr(torch.ops, _OPS_NAME)
33
+
34
+ def add_op_namespace_prefix(op_name: str) -> str:
35
+ """
36
+ Prefix op by namespace.
37
+ """
38
+ return f"{_OPS_NAME}::{op_name}"
build/torch-cuda/ldsa/__init__.py ADDED
@@ -0,0 +1,26 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import ctypes
2
+ import importlib.util
3
+ import sys
4
+ from pathlib import Path
5
+ from types import ModuleType
6
+
7
+
8
+ def _import_from_path(file_path: Path) -> ModuleType:
9
+ # We cannot use the module name as-is, after adding it to `sys.modules`,
10
+ # it would also be used for other imports. So, we make a module name that
11
+ # depends on the path for it to be unique using the hex-encoded hash of
12
+ # the path.
13
+ path_hash = "{:x}".format(ctypes.c_size_t(hash(file_path.absolute())).value)
14
+ module_name = path_hash
15
+ spec = importlib.util.spec_from_file_location(module_name, file_path)
16
+ if spec is None:
17
+ raise ImportError(f"Cannot load spec for {module_name} from {file_path}")
18
+ module = importlib.util.module_from_spec(spec)
19
+ if module is None:
20
+ raise ImportError(f"Cannot load module {module_name} from spec")
21
+ sys.modules[module_name] = module
22
+ spec.loader.exec_module(module) # type: ignore
23
+ return module
24
+
25
+
26
+ globals().update(vars(_import_from_path(Path(__file__).parent.parent / "__init__.py")))
build/torch-cuda/metadata.json ADDED
@@ -0,0 +1,29 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "name": "ldsa",
3
+ "id": "_ldsa_cuda_9763df8",
4
+ "version": 1,
5
+ "license": "Apache-2.0",
6
+ "python-depends": [],
7
+ "backend": {
8
+ "type": "cuda"
9
+ },
10
+ "digest": {
11
+ "algorithm": "sha256",
12
+ "files": {
13
+ "__init__.py": "NEgktlyHTKkRCwaMYX8w/ekaSbpjqLPHVmWtBTlDe6Q=",
14
+ "_ops.py": "KddM++gF3qGbBFs9PMP3HLS6KFeK8NEJWiwEH4aX15Y=",
15
+ "ldsa/__init__.py": "DFYPlrhXwYjEqCl/8n0SmWGZV8NFml5DPhMjKfv98GY="
16
+ }
17
+ },
18
+ "provenance": {
19
+ "kernel-builder": {
20
+ "version": "0.17.0-dev0",
21
+ "sha": "7262b99a509c116a4661e07eba15663ae19cddb5",
22
+ "dirty": false
23
+ },
24
+ "kernel": {
25
+ "sha": "9763df80731b1bd5b031b311793df01d91f59554",
26
+ "dirty": false
27
+ }
28
+ }
29
+ }