Ashiedu commited on
Commit
40ef767
·
verified ·
1 Parent(s): b419fb0

Add files using upload-large-folder tool

Browse files
CARD.md ADDED
@@ -0,0 +1,26 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ library_name: kernels
3
+ license: apache-2.0
4
+ ---
5
+
6
+ This is the repository card of Ashiedu/selective-update that has been pushed on the Hub. It was built to be used with the [`kernels` library](https://github.com/huggingface/kernels). This card was automatically generated.
7
+
8
+ ## How to use
9
+
10
+ ```python
11
+ # make sure `kernels` is installed: `pip install -U kernels`
12
+ from kernels import get_kernel
13
+
14
+ kernel_module = get_kernel("Ashiedu/selective-update", version=1)
15
+ selective_update = kernel_module.selective_update
16
+
17
+ selective_update(...)
18
+ ```
19
+
20
+ ## Available functions
21
+ - `selective_update`
22
+ - `selective_state_update`
23
+
24
+ ## Benchmarks
25
+
26
+ Benchmarking script is available for this kernel. Run `kernels benchmark Ashiedu/selective-update --version 1`.
benchmarks/benchmark.py ADDED
@@ -0,0 +1,17 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import torch
2
+
3
+ from kernels.benchmark import Benchmark
4
+
5
+
6
+ class SelectiveUpdateBenchmark(Benchmark):
7
+ def setup(self):
8
+ self.size = 1024
9
+ self.input = torch.randn(self.size, self.size, device=self.device)
10
+ self.out = torch.empty_like(self.input)
11
+
12
+ def benchmark_base(self):
13
+ self.kernel.selective_update(self.out, self.input)
14
+
15
+ def verify_base(self) -> torch.Tensor:
16
+ # Reference implementation
17
+ return self.input.clone()
torch-rocm/__init__.py ADDED
@@ -0,0 +1,16 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from typing import Optional
2
+
3
+ import torch
4
+
5
+ from .functional import selective_state_update
6
+
7
+
8
+ def selective_update(x: torch.Tensor, out: Optional[torch.Tensor] = None) -> torch.Tensor:
9
+ """Legacy scaffold kernel — adds 1.0 to each element."""
10
+ if out is None:
11
+ out = torch.empty_like(x)
12
+ out.copy_(x + 1.0)
13
+ return out
14
+
15
+
16
+ __all__ = ["selective_update", "selective_state_update"]
torch-rocm/_autotune_configs.py ADDED
@@ -0,0 +1,101 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+ import warnings
3
+ from packaging import version
4
+
5
+ import torch
6
+
7
+ try:
8
+ import triton
9
+ TRITON_VERSION = version.parse(triton.__version__)
10
+ except ImportError:
11
+ TRITON_VERSION = version.parse("0.0.0")
12
+
13
+ TRITON_HAS_CACHE_RESULTS = TRITON_VERSION >= version.parse("3.4.0")
14
+ _autotune_warning_issued = False
15
+ _deterministic_override = None
16
+
17
+
18
+ def use_deterministic_mode():
19
+ if _deterministic_override is not None:
20
+ return _deterministic_override
21
+ env = os.environ.get("MAMBA_DETERMINISTIC")
22
+ if env:
23
+ return env[0] == "1"
24
+ return torch.are_deterministic_algorithms_enabled()
25
+
26
+
27
+ def set_deterministic_mode(value):
28
+ global _deterministic_override
29
+ _deterministic_override = value
30
+
31
+
32
+ def is_gfx1030():
33
+ """Identifies the HIP gfx1030 hardware."""
34
+ if not torch.cuda.is_available():
35
+ return False
36
+ try:
37
+ device_cap = torch.cuda.get_device_capability()
38
+ return device_cap == (10, 3)
39
+ except Exception:
40
+ return False
41
+
42
+
43
+ def gfx1030_selective_update_configs(configs):
44
+ """Pin safe tile configs for selective-update on gfx1030."""
45
+ if not is_gfx1030():
46
+ return configs
47
+
48
+ expected = {
49
+ "BLOCK_SIZE_M": 32,
50
+ "BLOCK_SIZE_DSTATE": 64,
51
+ }
52
+ selected = [
53
+ config
54
+ for config in configs
55
+ if all(config.kwargs.get(key) == value for key, value in expected.items())
56
+ ]
57
+ if not selected:
58
+ return [configs[0]]
59
+ return selected[:1]
60
+
61
+
62
+ def _estimate_config_cost(cfg):
63
+ block_product = 1
64
+ for key, val in cfg.kwargs.items():
65
+ if key.startswith("BLOCK_SIZE_"):
66
+ block_product *= val
67
+ return block_product * (getattr(cfg, "num_stages", 1) or 1)
68
+
69
+
70
+ def _filter_configs_by_block_sizes(configs):
71
+ env_filters = {}
72
+ for suffix in ("M", "N", "K", "DSTATE"):
73
+ env_val = os.environ.get(f"TRITON_AUTOTUNE_BLOCK_SIZE_{suffix}")
74
+ if env_val is not None:
75
+ env_filters[f"BLOCK_SIZE_{suffix}"] = int(env_val)
76
+ if not env_filters:
77
+ return None
78
+ matching = configs
79
+ for key, target in env_filters.items():
80
+ matching = [c for c in matching if c.kwargs.get(key) == target]
81
+ return matching[:1] if matching else None
82
+
83
+
84
+ def autotune_configs(configs):
85
+ if not configs or not use_deterministic_mode():
86
+ return configs
87
+ if TRITON_HAS_CACHE_RESULTS and os.environ.get("TRITON_CACHE_AUTOTUNING") == "1":
88
+ return configs
89
+ global _autotune_warning_issued
90
+ if not _autotune_warning_issued:
91
+ _autotune_warning_issued = True
92
+ msg = (
93
+ "Deterministic mode: set TRITON_CACHE_AUTOTUNING=1 for cached autotuning."
94
+ if TRITON_HAS_CACHE_RESULTS
95
+ else "Deterministic mode: upgrade to Triton >= 3.4.0 for cached autotuning."
96
+ )
97
+ warnings.warn(msg)
98
+ filtered = _filter_configs_by_block_sizes(configs)
99
+ if filtered:
100
+ return filtered
101
+ return [min(configs, key=_estimate_config_cost)]
torch-rocm/_ops.py ADDED
@@ -0,0 +1,38 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import torch
2
+
3
+ def get_backend() -> str:
4
+ """Detect the backend by inspecting torch."""
5
+ import torch
6
+
7
+ if hasattr(torch, "neuron"):
8
+ # Needs to be sorted before specific Torch builds, since Neuron
9
+ # extension can be loaded into e.g. CUDA Torch builds.
10
+ return "neuron"
11
+ elif torch.version.cuda is not None:
12
+ return "cuda"
13
+ elif torch.version.hip is not None:
14
+ return "rocm"
15
+ elif torch.backends.mps.is_available():
16
+ return "metal"
17
+ elif hasattr(torch.version, "xpu") and torch.version.xpu is not None:
18
+ return "xpu"
19
+ else:
20
+ return "cpu"
21
+
22
+
23
+ def _find_ops_name() -> str:
24
+ kernel_name = "selective_update"
25
+ unique_id = "8104c19_dirty"
26
+ backend = get_backend()
27
+ return f"_{kernel_name}_{backend}_{unique_id}"
28
+
29
+
30
+ _OPS_NAME = _find_ops_name()
31
+
32
+ ops = getattr(torch.ops, _OPS_NAME)
33
+
34
+ def add_op_namespace_prefix(op_name: str) -> str:
35
+ """
36
+ Prefix op by namespace.
37
+ """
38
+ return f"{_OPS_NAME}::{op_name}"
torch-rocm/functional.py ADDED
@@ -0,0 +1,165 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Python autograd wrapper for selective-update Triton kernel.
2
+
3
+ Matches upstream ``mamba_ssm.ops.triton.selective_state_update`` ABI exactly:
4
+ state: (B, D, N) or (B, H, D, N)
5
+ x: (B, D) or (B, H, D)
6
+ dt: (B, D) or (B, H, D)
7
+ A: (D, N) or (H, D, N)
8
+ B: (B, N) or (B, G, N)
9
+ C: (B, N) or (B, G, N)
10
+ D: (D,) or (H, D) or None
11
+ z: (B, D) or (B, H, D) or None
12
+ dt_bias: (D,) or (H, D) or None
13
+ state_batch_indices: (B,) or None
14
+ """
15
+
16
+ from __future__ import annotations
17
+
18
+ import importlib.util
19
+ from pathlib import Path
20
+
21
+ import torch
22
+ import triton
23
+
24
+ _TRITON_KERNELS_PATH = (
25
+ Path(__file__).resolve().parent / "triton_kernels.py"
26
+ )
27
+
28
+
29
+ def _load_triton_kernel():
30
+ spec = importlib.util.spec_from_file_location(
31
+ "selective_update.triton_kernels",
32
+ _TRITON_KERNELS_PATH,
33
+ )
34
+ module = importlib.util.module_from_spec(spec)
35
+ spec.loader.exec_module(module)
36
+ return module.selective_state_update_kernel
37
+
38
+
39
+ selective_state_update_kernel = None # populated on first call
40
+
41
+
42
+ def _get_kernel():
43
+ global selective_state_update_kernel
44
+ if selective_state_update_kernel is None:
45
+ selective_state_update_kernel = _load_triton_kernel()
46
+ return selective_state_update_kernel
47
+
48
+
49
+ def _resolve_tie_hdim(A, dt, dt_bias):
50
+ if A.dim() == 2:
51
+ a0 = A[0]
52
+ return a0.stride(0) == 0 and a0.stride(1) == 0 and dt.stride(-1) == 0 and (dt_bias.stride(-1) == 0 if dt_bias is not None else True)
53
+ return A.stride(-1) == 0 and A.stride(-2) == 0 and dt.stride(-1) == 0 and (dt_bias.stride(-1) == 0 if dt_bias is not None else True)
54
+
55
+
56
+ class SelectiveUpdateFunction(torch.autograd.Function):
57
+ """Differentiable single-token selective state update.
58
+
59
+ Mirrors upstream ``selective_state_update`` shape contract exactly.
60
+ Backward raises NotImplementedError — training uses the patched
61
+ selective_scan reverse path instead.
62
+ """
63
+
64
+ @staticmethod
65
+ def forward(ctx, state, x, dt, A, B, C, D=None, z=None, dt_bias=None, dt_softplus=False, state_batch_indices=None):
66
+ orig_state = state
67
+ has_heads = state.dim() > 3
68
+ if state.dim() == 3:
69
+ state = state.unsqueeze(1)
70
+ if x.dim() == 2:
71
+ x = x.unsqueeze(1)
72
+ if dt.dim() == 2:
73
+ dt = dt.unsqueeze(1)
74
+ if A.dim() == 2:
75
+ A = A.unsqueeze(0)
76
+ if B.dim() == 2:
77
+ B = B.unsqueeze(1)
78
+ if C.dim() == 2:
79
+ C = C.unsqueeze(1)
80
+ if D is not None and D.dim() == 1:
81
+ D = D.unsqueeze(0)
82
+ if z is not None and z.dim() == 2:
83
+ z = z.unsqueeze(1)
84
+ if dt_bias is not None and dt_bias.dim() == 1:
85
+ dt_bias = dt_bias.unsqueeze(0)
86
+
87
+ B_sz, H, D_sz, N_sz = state.shape
88
+ x_sz = x.shape[0]
89
+ if x.shape != (x_sz, H, D_sz):
90
+ raise ValueError(f"x shape {x.shape} does not match state batch/heads/dim")
91
+ if dt.shape != (x_sz, H, D_sz):
92
+ raise ValueError(f"dt shape {dt.shape} does not match x shape")
93
+ if A.shape != (H, D_sz, N_sz):
94
+ raise ValueError(f"A shape {A.shape} does not match (H, D, N)")
95
+ ngroups = B.shape[1]
96
+ if H % ngroups != 0:
97
+ raise ValueError(f"nheads {H} must be divisible by ngroups {ngroups}")
98
+ if B.shape[0] != x_sz or B.shape[2] != N_sz or C.shape[0] != x_sz or C.shape[2] != N_sz:
99
+ raise ValueError(f"B/C batch or dstate mismatch")
100
+ if D is not None and D.shape not in {(H, D_sz), (H,)}:
101
+ raise ValueError(f"D shape {D.shape} does not match (H, D) or (H,)")
102
+ if z is not None and z.shape != (x_sz, H, D_sz):
103
+ raise ValueError(f"z shape {z.shape} does not match x shape")
104
+ if dt_bias is not None and dt_bias.shape not in {(H, D_sz), (H,)}:
105
+ raise ValueError(f"dt_bias shape {dt_bias.shape} does not match (H, D) or (H,)")
106
+ if state_batch_indices is not None and state_batch_indices.shape != (x_sz,):
107
+ raise ValueError(f"state_batch_indices shape {state_batch_indices.shape} does not match (B,)")
108
+
109
+ tie_hdim = _resolve_tie_hdim(A, dt, dt_bias)
110
+ nheads_ratio = H // ngroups
111
+
112
+ out = torch.empty_like(x)
113
+ grid = lambda META: (triton.cdiv(D_sz, META["BLOCK_SIZE_M"]), x_sz, H)
114
+
115
+ BLOCK_M = 32 if N_sz <= 16 else (16 if N_sz <= 32 else (8 if N_sz <= 64 else (4 if N_sz <= 128 else 4)))
116
+ num_warps = 4 if N_sz <= 64 else 8
117
+
118
+ z_strides = ((z.stride(0), z.stride(1), z.stride(2)) if z is not None else (0, 0, 0))
119
+
120
+ _get_kernel()[grid](
121
+ state, x, dt, dt_bias, A, B, C, D, z, out, state_batch_indices,
122
+ x_sz, H, D_sz, N_sz, nheads_ratio,
123
+ state.stride(0), state.stride(1), state.stride(2), state.stride(3),
124
+ x.stride(0), x.stride(1), x.stride(2),
125
+ dt.stride(0), dt.stride(1), dt.stride(2),
126
+ dt_bias.stride(0) if dt_bias is not None else 0, dt_bias.stride(1) if dt_bias is not None else 0,
127
+ A.stride(0), A.stride(1), A.stride(2),
128
+ B.stride(0), B.stride(1), B.stride(2),
129
+ C.stride(0), C.stride(1), C.stride(2),
130
+ D.stride(0) if D is not None else 0, D.stride(1) if D is not None else 0,
131
+ z_strides[0], z_strides[1], z_strides[2],
132
+ out.stride(0), out.stride(1), out.stride(2),
133
+ dt_softplus,
134
+ tie_hdim,
135
+ BLOCK_M,
136
+ num_warps=num_warps,
137
+ num_stages=2,
138
+ )
139
+
140
+ if not has_heads:
141
+ out = out.squeeze(1)
142
+ state = state.squeeze(1)
143
+ ctx.save_for_backward(orig_state, x, dt, A, B, C, D, z, dt_bias, state, state_batch_indices)
144
+ ctx.tie_hdim = tie_hdim
145
+ ctx.dt_softplus = dt_softplus
146
+ ctx.has_heads = has_heads
147
+ ctx.nheads_ratio = nheads_ratio
148
+ return out
149
+
150
+ @staticmethod
151
+ def backward(ctx, grad_out):
152
+ raise NotImplementedError(
153
+ "SelectiveUpdateFunction backward is not implemented. "
154
+ "Use autograd through the patched selective_scan path for training."
155
+ )
156
+
157
+
158
+ def selective_state_update(state, x, dt, A, B, C, D=None, z=None, dt_bias=None, dt_softplus=False, state_batch_indices=None, **kwargs):
159
+ """Dispatches single-token state updates to the custom Triton backend.
160
+
161
+ Shapes mirror upstream ``mamba_ssm.ops.triton.selective_state_update`` exactly.
162
+ """
163
+ return SelectiveUpdateFunction.apply(
164
+ state, x, dt, A, B, C, D, z, dt_bias, dt_softplus, state_batch_indices
165
+ )
torch-rocm/metadata.json ADDED
@@ -0,0 +1,32 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "name": "selective-update",
3
+ "id": "_selective_update_rocm_8104c19_dirty",
4
+ "version": 1,
5
+ "license": "Apache-2.0",
6
+ "python-depends": [],
7
+ "backend": {
8
+ "type": "rocm"
9
+ },
10
+ "digest": {
11
+ "algorithm": "sha256",
12
+ "files": {
13
+ "__init__.py": "rfNR8lwmy5vE+tOcY3z9WlRB6oKnHDWFTKGl2a1da8Y=",
14
+ "_autotune_configs.py": "Fa+a4MPxNOJEh8C9Y5SiYmASML4kjwbqey31khACuHU=",
15
+ "_ops.py": "EZezVTAP8qYzjOXulaFH/WywDDnJD6HKCaxR5MyvM+w=",
16
+ "functional.py": "lCB9nsg0h4srtTQewSRtmSih351l92cxFQF75WLkGeA=",
17
+ "selective_update/__init__.py": "DFYPlrhXwYjEqCl/8n0SmWGZV8NFml5DPhMjKfv98GY=",
18
+ "triton_kernels.py": "l8cpfdk/bM2bmL8etGzhr4KZdAp9o0/ypRn2IyBkmOA="
19
+ }
20
+ },
21
+ "provenance": {
22
+ "kernel-builder": {
23
+ "version": "0.17.0-dev0",
24
+ "sha": "570dcf4a810e6bd9d2433716c2bc6962c6ae46bb",
25
+ "dirty": false
26
+ },
27
+ "kernel": {
28
+ "sha": "8104c195102619f0d0f1a8087307547318b78d57",
29
+ "dirty": true
30
+ }
31
+ }
32
+ }
torch-rocm/selective_update/__init__.py ADDED
@@ -0,0 +1,26 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import ctypes
2
+ import importlib.util
3
+ import sys
4
+ from pathlib import Path
5
+ from types import ModuleType
6
+
7
+
8
+ def _import_from_path(file_path: Path) -> ModuleType:
9
+ # We cannot use the module name as-is, after adding it to `sys.modules`,
10
+ # it would also be used for other imports. So, we make a module name that
11
+ # depends on the path for it to be unique using the hex-encoded hash of
12
+ # the path.
13
+ path_hash = "{:x}".format(ctypes.c_size_t(hash(file_path.absolute())).value)
14
+ module_name = path_hash
15
+ spec = importlib.util.spec_from_file_location(module_name, file_path)
16
+ if spec is None:
17
+ raise ImportError(f"Cannot load spec for {module_name} from {file_path}")
18
+ module = importlib.util.module_from_spec(spec)
19
+ if module is None:
20
+ raise ImportError(f"Cannot load module {module_name} from spec")
21
+ sys.modules[module_name] = module
22
+ spec.loader.exec_module(module) # type: ignore
23
+ return module
24
+
25
+
26
+ globals().update(vars(_import_from_path(Path(__file__).parent.parent / "__init__.py")))
torch-rocm/triton_kernels.py ADDED
@@ -0,0 +1,127 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Triton kernels for Mamba-2 single-token selective state updates.
2
+
3
+ Matches upstream ``mamba_ssm.ops.triton.selective_state_update`` ABI exactly:
4
+ state: (B, H, D, N) x/dt: (B, H, D) A: (H, D, N) B/C: (B, G, N)
5
+ """
6
+
7
+ import triton
8
+ import triton.language as tl
9
+
10
+
11
+ @triton.jit
12
+ def _softplus(dt):
13
+ return tl.math.log(tl.math.exp(dt) + 1)
14
+
15
+
16
+ @triton.heuristics({"HAS_DT_BIAS": lambda args: args["dt_bias_ptr"] is not None})
17
+ @triton.heuristics({"HAS_D": lambda args: args["D_ptr"] is not None})
18
+ @triton.heuristics({"HAS_Z": lambda args: args["z_ptr"] is not None})
19
+ @triton.heuristics({"HAS_STATE_BATCH_INDICES": lambda args: args["state_batch_indices_ptr"] is not None})
20
+ @triton.heuristics({"BLOCK_SIZE_DSTATE": lambda args: triton.next_power_of_2(args["dstate"])})
21
+ @triton.jit
22
+ def selective_state_update_kernel(
23
+ state_ptr, x_ptr, dt_ptr, dt_bias_ptr, A_ptr, B_ptr, C_ptr, D_ptr, z_ptr, out_ptr, state_batch_indices_ptr,
24
+ batch, nheads, dim, dstate, nheads_ngroups_ratio,
25
+ s_state_batch, s_state_head, s_state_dim, s_state_dstate,
26
+ s_x_batch, s_x_head, s_x_dim,
27
+ s_dt_batch, s_dt_head, s_dt_dim,
28
+ s_dt_bias_head, s_dt_bias_dim,
29
+ s_A_head, s_A_dim, s_A_dstate,
30
+ s_B_batch, s_B_group, s_B_dstate,
31
+ s_C_batch, s_C_group, s_C_dstate,
32
+ s_D_head, s_D_dim,
33
+ s_z_batch, s_z_head, s_z_dim,
34
+ s_out_batch, s_out_head, s_out_dim,
35
+ DT_SOFTPLUS: tl.constexpr,
36
+ TIE_HDIM: tl.constexpr,
37
+ BLOCK_SIZE_M: tl.constexpr,
38
+ HAS_DT_BIAS: tl.constexpr,
39
+ HAS_D: tl.constexpr,
40
+ HAS_Z: tl.constexpr,
41
+ HAS_STATE_BATCH_INDICES: tl.constexpr,
42
+ BLOCK_SIZE_DSTATE: tl.constexpr,
43
+ ):
44
+ pid_m = tl.program_id(axis=0)
45
+ pid_b = tl.program_id(axis=1)
46
+ pid_h = tl.program_id(axis=2)
47
+
48
+ offs_m = pid_m * BLOCK_SIZE_M + tl.arange(0, BLOCK_SIZE_M)
49
+ out_ptr += pid_b * s_out_batch + pid_h * s_out_head
50
+ out_ptrs = out_ptr + offs_m * s_out_dim
51
+
52
+ if HAS_STATE_BATCH_INDICES:
53
+ state_batch_indices_ptr += pid_b
54
+ state_batch_idx = tl.load(state_batch_indices_ptr)
55
+ if state_batch_idx < 0:
56
+ tl.store(out_ptrs, 0.0, mask=offs_m < dim)
57
+ return
58
+ state_ptr += state_batch_idx * s_state_batch + pid_h * s_state_head
59
+ else:
60
+ state_ptr += pid_b * s_state_batch + pid_h * s_state_head
61
+
62
+ x_ptr += pid_b * s_x_batch + pid_h * s_x_head
63
+ dt_ptr += pid_b * s_dt_batch + pid_h * s_dt_head
64
+ if HAS_DT_BIAS:
65
+ dt_bias_ptr += pid_h * s_dt_bias_head
66
+ A_ptr += pid_h * s_A_head
67
+ B_ptr += pid_b * s_B_batch + (pid_h // nheads_ngroups_ratio) * s_B_group
68
+ C_ptr += pid_b * s_C_batch + (pid_h // nheads_ngroups_ratio) * s_C_group
69
+ if HAS_Z:
70
+ z_ptr += pid_b * s_z_batch + pid_h * s_z_head
71
+
72
+ offs_n = tl.arange(0, BLOCK_SIZE_DSTATE)
73
+ state_ptrs = state_ptr + (offs_m[:, None] * s_state_dim + offs_n[None, :] * s_state_dstate)
74
+ x_ptrs = x_ptr + offs_m * s_x_dim
75
+ dt_ptrs = dt_ptr + offs_m * s_dt_dim
76
+ if HAS_DT_BIAS:
77
+ dt_bias_ptrs = dt_bias_ptr + offs_m * s_dt_bias_dim
78
+ if HAS_D:
79
+ D_ptr += pid_h * s_D_head
80
+ A_ptrs = A_ptr + (offs_m[:, None] * s_A_dim + offs_n[None, :] * s_A_dstate)
81
+ B_ptrs = B_ptr + offs_n * s_B_dstate
82
+ C_ptrs = C_ptr + offs_n * s_C_dstate
83
+ if HAS_D:
84
+ D_ptrs = D_ptr + offs_m * s_D_dim
85
+ if HAS_Z:
86
+ z_ptrs = z_ptr + offs_m * s_z_dim
87
+
88
+ state = tl.load(state_ptrs, mask=(offs_m[:, None] < dim) & (offs_n[None, :] < dstate), other=0.0)
89
+ x = tl.load(x_ptrs, mask=offs_m < dim, other=0.0).to(tl.float32)
90
+ if not TIE_HDIM:
91
+ dt = tl.load(dt_ptrs, mask=offs_m < dim, other=0.0).to(tl.float32)
92
+ if HAS_DT_BIAS:
93
+ dt += tl.load(dt_bias_ptrs, mask=offs_m < dim, other=0.0).to(tl.float32)
94
+ if DT_SOFTPLUS:
95
+ dt = tl.where(dt <= 20.0, _softplus(dt), dt)
96
+ A = tl.load(A_ptrs, mask=(offs_m[:, None] < dim) & (offs_n[None, :] < dstate), other=0.0).to(tl.float32)
97
+ dA = tl.exp(A * dt[:, None])
98
+ else:
99
+ dt = tl.load(dt_ptr).to(tl.float32)
100
+ if HAS_DT_BIAS:
101
+ dt += tl.load(dt_bias_ptr).to(tl.float32)
102
+ if DT_SOFTPLUS:
103
+ dt = tl.where(dt <= 20.0, _softplus(dt), dt)
104
+ A = tl.load(A_ptr).to(tl.float32)
105
+ dA = tl.exp(A * dt)
106
+
107
+ B = tl.load(B_ptrs, mask=offs_n < dstate, other=0.0).to(tl.float32)
108
+ C = tl.load(C_ptrs, mask=offs_n < dstate, other=0.0).to(tl.float32)
109
+ if HAS_D:
110
+ D = tl.load(D_ptrs, mask=offs_m < dim, other=0.0).to(tl.float32)
111
+ if HAS_Z:
112
+ z = tl.load(z_ptrs, mask=offs_m < dim, other=0.0).to(tl.float32)
113
+
114
+ if not TIE_HDIM:
115
+ dB = B[None, :] * dt[:, None]
116
+ else:
117
+ dB = B * dt
118
+
119
+ state = state * dA + dB * x[:, None]
120
+ tl.store(state_ptrs, state, mask=(offs_m[:, None] < dim) & (offs_n[None, :] < dstate))
121
+ out = tl.sum(state * C[None, :], axis=1)
122
+ if HAS_D:
123
+ out += x * D
124
+ if HAS_Z:
125
+ out *= z * tl.sigmoid(z)
126
+
127
+ tl.store(out_ptrs, out, mask=offs_m < dim)