Add files using upload-large-folder tool
Browse files- CARD.md +26 -0
- benchmarks/benchmark.py +17 -0
- torch-rocm/__init__.py +16 -0
- torch-rocm/_autotune_configs.py +101 -0
- torch-rocm/_ops.py +38 -0
- torch-rocm/functional.py +165 -0
- torch-rocm/metadata.json +32 -0
- torch-rocm/selective_update/__init__.py +26 -0
- torch-rocm/triton_kernels.py +127 -0
CARD.md
ADDED
|
@@ -0,0 +1,26 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
---
|
| 2 |
+
library_name: kernels
|
| 3 |
+
license: apache-2.0
|
| 4 |
+
---
|
| 5 |
+
|
| 6 |
+
This is the repository card of Ashiedu/selective-update that has been pushed on the Hub. It was built to be used with the [`kernels` library](https://github.com/huggingface/kernels). This card was automatically generated.
|
| 7 |
+
|
| 8 |
+
## How to use
|
| 9 |
+
|
| 10 |
+
```python
|
| 11 |
+
# make sure `kernels` is installed: `pip install -U kernels`
|
| 12 |
+
from kernels import get_kernel
|
| 13 |
+
|
| 14 |
+
kernel_module = get_kernel("Ashiedu/selective-update", version=1)
|
| 15 |
+
selective_update = kernel_module.selective_update
|
| 16 |
+
|
| 17 |
+
selective_update(...)
|
| 18 |
+
```
|
| 19 |
+
|
| 20 |
+
## Available functions
|
| 21 |
+
- `selective_update`
|
| 22 |
+
- `selective_state_update`
|
| 23 |
+
|
| 24 |
+
## Benchmarks
|
| 25 |
+
|
| 26 |
+
Benchmarking script is available for this kernel. Run `kernels benchmark Ashiedu/selective-update --version 1`.
|
benchmarks/benchmark.py
ADDED
|
@@ -0,0 +1,17 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import torch
|
| 2 |
+
|
| 3 |
+
from kernels.benchmark import Benchmark
|
| 4 |
+
|
| 5 |
+
|
| 6 |
+
class SelectiveUpdateBenchmark(Benchmark):
|
| 7 |
+
def setup(self):
|
| 8 |
+
self.size = 1024
|
| 9 |
+
self.input = torch.randn(self.size, self.size, device=self.device)
|
| 10 |
+
self.out = torch.empty_like(self.input)
|
| 11 |
+
|
| 12 |
+
def benchmark_base(self):
|
| 13 |
+
self.kernel.selective_update(self.out, self.input)
|
| 14 |
+
|
| 15 |
+
def verify_base(self) -> torch.Tensor:
|
| 16 |
+
# Reference implementation
|
| 17 |
+
return self.input.clone()
|
torch-rocm/__init__.py
ADDED
|
@@ -0,0 +1,16 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from typing import Optional
|
| 2 |
+
|
| 3 |
+
import torch
|
| 4 |
+
|
| 5 |
+
from .functional import selective_state_update
|
| 6 |
+
|
| 7 |
+
|
| 8 |
+
def selective_update(x: torch.Tensor, out: Optional[torch.Tensor] = None) -> torch.Tensor:
|
| 9 |
+
"""Legacy scaffold kernel — adds 1.0 to each element."""
|
| 10 |
+
if out is None:
|
| 11 |
+
out = torch.empty_like(x)
|
| 12 |
+
out.copy_(x + 1.0)
|
| 13 |
+
return out
|
| 14 |
+
|
| 15 |
+
|
| 16 |
+
__all__ = ["selective_update", "selective_state_update"]
|
torch-rocm/_autotune_configs.py
ADDED
|
@@ -0,0 +1,101 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import os
|
| 2 |
+
import warnings
|
| 3 |
+
from packaging import version
|
| 4 |
+
|
| 5 |
+
import torch
|
| 6 |
+
|
| 7 |
+
try:
|
| 8 |
+
import triton
|
| 9 |
+
TRITON_VERSION = version.parse(triton.__version__)
|
| 10 |
+
except ImportError:
|
| 11 |
+
TRITON_VERSION = version.parse("0.0.0")
|
| 12 |
+
|
| 13 |
+
TRITON_HAS_CACHE_RESULTS = TRITON_VERSION >= version.parse("3.4.0")
|
| 14 |
+
_autotune_warning_issued = False
|
| 15 |
+
_deterministic_override = None
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
def use_deterministic_mode():
|
| 19 |
+
if _deterministic_override is not None:
|
| 20 |
+
return _deterministic_override
|
| 21 |
+
env = os.environ.get("MAMBA_DETERMINISTIC")
|
| 22 |
+
if env:
|
| 23 |
+
return env[0] == "1"
|
| 24 |
+
return torch.are_deterministic_algorithms_enabled()
|
| 25 |
+
|
| 26 |
+
|
| 27 |
+
def set_deterministic_mode(value):
|
| 28 |
+
global _deterministic_override
|
| 29 |
+
_deterministic_override = value
|
| 30 |
+
|
| 31 |
+
|
| 32 |
+
def is_gfx1030():
|
| 33 |
+
"""Identifies the HIP gfx1030 hardware."""
|
| 34 |
+
if not torch.cuda.is_available():
|
| 35 |
+
return False
|
| 36 |
+
try:
|
| 37 |
+
device_cap = torch.cuda.get_device_capability()
|
| 38 |
+
return device_cap == (10, 3)
|
| 39 |
+
except Exception:
|
| 40 |
+
return False
|
| 41 |
+
|
| 42 |
+
|
| 43 |
+
def gfx1030_selective_update_configs(configs):
|
| 44 |
+
"""Pin safe tile configs for selective-update on gfx1030."""
|
| 45 |
+
if not is_gfx1030():
|
| 46 |
+
return configs
|
| 47 |
+
|
| 48 |
+
expected = {
|
| 49 |
+
"BLOCK_SIZE_M": 32,
|
| 50 |
+
"BLOCK_SIZE_DSTATE": 64,
|
| 51 |
+
}
|
| 52 |
+
selected = [
|
| 53 |
+
config
|
| 54 |
+
for config in configs
|
| 55 |
+
if all(config.kwargs.get(key) == value for key, value in expected.items())
|
| 56 |
+
]
|
| 57 |
+
if not selected:
|
| 58 |
+
return [configs[0]]
|
| 59 |
+
return selected[:1]
|
| 60 |
+
|
| 61 |
+
|
| 62 |
+
def _estimate_config_cost(cfg):
|
| 63 |
+
block_product = 1
|
| 64 |
+
for key, val in cfg.kwargs.items():
|
| 65 |
+
if key.startswith("BLOCK_SIZE_"):
|
| 66 |
+
block_product *= val
|
| 67 |
+
return block_product * (getattr(cfg, "num_stages", 1) or 1)
|
| 68 |
+
|
| 69 |
+
|
| 70 |
+
def _filter_configs_by_block_sizes(configs):
|
| 71 |
+
env_filters = {}
|
| 72 |
+
for suffix in ("M", "N", "K", "DSTATE"):
|
| 73 |
+
env_val = os.environ.get(f"TRITON_AUTOTUNE_BLOCK_SIZE_{suffix}")
|
| 74 |
+
if env_val is not None:
|
| 75 |
+
env_filters[f"BLOCK_SIZE_{suffix}"] = int(env_val)
|
| 76 |
+
if not env_filters:
|
| 77 |
+
return None
|
| 78 |
+
matching = configs
|
| 79 |
+
for key, target in env_filters.items():
|
| 80 |
+
matching = [c for c in matching if c.kwargs.get(key) == target]
|
| 81 |
+
return matching[:1] if matching else None
|
| 82 |
+
|
| 83 |
+
|
| 84 |
+
def autotune_configs(configs):
|
| 85 |
+
if not configs or not use_deterministic_mode():
|
| 86 |
+
return configs
|
| 87 |
+
if TRITON_HAS_CACHE_RESULTS and os.environ.get("TRITON_CACHE_AUTOTUNING") == "1":
|
| 88 |
+
return configs
|
| 89 |
+
global _autotune_warning_issued
|
| 90 |
+
if not _autotune_warning_issued:
|
| 91 |
+
_autotune_warning_issued = True
|
| 92 |
+
msg = (
|
| 93 |
+
"Deterministic mode: set TRITON_CACHE_AUTOTUNING=1 for cached autotuning."
|
| 94 |
+
if TRITON_HAS_CACHE_RESULTS
|
| 95 |
+
else "Deterministic mode: upgrade to Triton >= 3.4.0 for cached autotuning."
|
| 96 |
+
)
|
| 97 |
+
warnings.warn(msg)
|
| 98 |
+
filtered = _filter_configs_by_block_sizes(configs)
|
| 99 |
+
if filtered:
|
| 100 |
+
return filtered
|
| 101 |
+
return [min(configs, key=_estimate_config_cost)]
|
torch-rocm/_ops.py
ADDED
|
@@ -0,0 +1,38 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import torch
|
| 2 |
+
|
| 3 |
+
def get_backend() -> str:
|
| 4 |
+
"""Detect the backend by inspecting torch."""
|
| 5 |
+
import torch
|
| 6 |
+
|
| 7 |
+
if hasattr(torch, "neuron"):
|
| 8 |
+
# Needs to be sorted before specific Torch builds, since Neuron
|
| 9 |
+
# extension can be loaded into e.g. CUDA Torch builds.
|
| 10 |
+
return "neuron"
|
| 11 |
+
elif torch.version.cuda is not None:
|
| 12 |
+
return "cuda"
|
| 13 |
+
elif torch.version.hip is not None:
|
| 14 |
+
return "rocm"
|
| 15 |
+
elif torch.backends.mps.is_available():
|
| 16 |
+
return "metal"
|
| 17 |
+
elif hasattr(torch.version, "xpu") and torch.version.xpu is not None:
|
| 18 |
+
return "xpu"
|
| 19 |
+
else:
|
| 20 |
+
return "cpu"
|
| 21 |
+
|
| 22 |
+
|
| 23 |
+
def _find_ops_name() -> str:
|
| 24 |
+
kernel_name = "selective_update"
|
| 25 |
+
unique_id = "8104c19_dirty"
|
| 26 |
+
backend = get_backend()
|
| 27 |
+
return f"_{kernel_name}_{backend}_{unique_id}"
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
_OPS_NAME = _find_ops_name()
|
| 31 |
+
|
| 32 |
+
ops = getattr(torch.ops, _OPS_NAME)
|
| 33 |
+
|
| 34 |
+
def add_op_namespace_prefix(op_name: str) -> str:
|
| 35 |
+
"""
|
| 36 |
+
Prefix op by namespace.
|
| 37 |
+
"""
|
| 38 |
+
return f"{_OPS_NAME}::{op_name}"
|
torch-rocm/functional.py
ADDED
|
@@ -0,0 +1,165 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Python autograd wrapper for selective-update Triton kernel.
|
| 2 |
+
|
| 3 |
+
Matches upstream ``mamba_ssm.ops.triton.selective_state_update`` ABI exactly:
|
| 4 |
+
state: (B, D, N) or (B, H, D, N)
|
| 5 |
+
x: (B, D) or (B, H, D)
|
| 6 |
+
dt: (B, D) or (B, H, D)
|
| 7 |
+
A: (D, N) or (H, D, N)
|
| 8 |
+
B: (B, N) or (B, G, N)
|
| 9 |
+
C: (B, N) or (B, G, N)
|
| 10 |
+
D: (D,) or (H, D) or None
|
| 11 |
+
z: (B, D) or (B, H, D) or None
|
| 12 |
+
dt_bias: (D,) or (H, D) or None
|
| 13 |
+
state_batch_indices: (B,) or None
|
| 14 |
+
"""
|
| 15 |
+
|
| 16 |
+
from __future__ import annotations
|
| 17 |
+
|
| 18 |
+
import importlib.util
|
| 19 |
+
from pathlib import Path
|
| 20 |
+
|
| 21 |
+
import torch
|
| 22 |
+
import triton
|
| 23 |
+
|
| 24 |
+
_TRITON_KERNELS_PATH = (
|
| 25 |
+
Path(__file__).resolve().parent / "triton_kernels.py"
|
| 26 |
+
)
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
def _load_triton_kernel():
|
| 30 |
+
spec = importlib.util.spec_from_file_location(
|
| 31 |
+
"selective_update.triton_kernels",
|
| 32 |
+
_TRITON_KERNELS_PATH,
|
| 33 |
+
)
|
| 34 |
+
module = importlib.util.module_from_spec(spec)
|
| 35 |
+
spec.loader.exec_module(module)
|
| 36 |
+
return module.selective_state_update_kernel
|
| 37 |
+
|
| 38 |
+
|
| 39 |
+
selective_state_update_kernel = None # populated on first call
|
| 40 |
+
|
| 41 |
+
|
| 42 |
+
def _get_kernel():
|
| 43 |
+
global selective_state_update_kernel
|
| 44 |
+
if selective_state_update_kernel is None:
|
| 45 |
+
selective_state_update_kernel = _load_triton_kernel()
|
| 46 |
+
return selective_state_update_kernel
|
| 47 |
+
|
| 48 |
+
|
| 49 |
+
def _resolve_tie_hdim(A, dt, dt_bias):
|
| 50 |
+
if A.dim() == 2:
|
| 51 |
+
a0 = A[0]
|
| 52 |
+
return a0.stride(0) == 0 and a0.stride(1) == 0 and dt.stride(-1) == 0 and (dt_bias.stride(-1) == 0 if dt_bias is not None else True)
|
| 53 |
+
return A.stride(-1) == 0 and A.stride(-2) == 0 and dt.stride(-1) == 0 and (dt_bias.stride(-1) == 0 if dt_bias is not None else True)
|
| 54 |
+
|
| 55 |
+
|
| 56 |
+
class SelectiveUpdateFunction(torch.autograd.Function):
|
| 57 |
+
"""Differentiable single-token selective state update.
|
| 58 |
+
|
| 59 |
+
Mirrors upstream ``selective_state_update`` shape contract exactly.
|
| 60 |
+
Backward raises NotImplementedError — training uses the patched
|
| 61 |
+
selective_scan reverse path instead.
|
| 62 |
+
"""
|
| 63 |
+
|
| 64 |
+
@staticmethod
|
| 65 |
+
def forward(ctx, state, x, dt, A, B, C, D=None, z=None, dt_bias=None, dt_softplus=False, state_batch_indices=None):
|
| 66 |
+
orig_state = state
|
| 67 |
+
has_heads = state.dim() > 3
|
| 68 |
+
if state.dim() == 3:
|
| 69 |
+
state = state.unsqueeze(1)
|
| 70 |
+
if x.dim() == 2:
|
| 71 |
+
x = x.unsqueeze(1)
|
| 72 |
+
if dt.dim() == 2:
|
| 73 |
+
dt = dt.unsqueeze(1)
|
| 74 |
+
if A.dim() == 2:
|
| 75 |
+
A = A.unsqueeze(0)
|
| 76 |
+
if B.dim() == 2:
|
| 77 |
+
B = B.unsqueeze(1)
|
| 78 |
+
if C.dim() == 2:
|
| 79 |
+
C = C.unsqueeze(1)
|
| 80 |
+
if D is not None and D.dim() == 1:
|
| 81 |
+
D = D.unsqueeze(0)
|
| 82 |
+
if z is not None and z.dim() == 2:
|
| 83 |
+
z = z.unsqueeze(1)
|
| 84 |
+
if dt_bias is not None and dt_bias.dim() == 1:
|
| 85 |
+
dt_bias = dt_bias.unsqueeze(0)
|
| 86 |
+
|
| 87 |
+
B_sz, H, D_sz, N_sz = state.shape
|
| 88 |
+
x_sz = x.shape[0]
|
| 89 |
+
if x.shape != (x_sz, H, D_sz):
|
| 90 |
+
raise ValueError(f"x shape {x.shape} does not match state batch/heads/dim")
|
| 91 |
+
if dt.shape != (x_sz, H, D_sz):
|
| 92 |
+
raise ValueError(f"dt shape {dt.shape} does not match x shape")
|
| 93 |
+
if A.shape != (H, D_sz, N_sz):
|
| 94 |
+
raise ValueError(f"A shape {A.shape} does not match (H, D, N)")
|
| 95 |
+
ngroups = B.shape[1]
|
| 96 |
+
if H % ngroups != 0:
|
| 97 |
+
raise ValueError(f"nheads {H} must be divisible by ngroups {ngroups}")
|
| 98 |
+
if B.shape[0] != x_sz or B.shape[2] != N_sz or C.shape[0] != x_sz or C.shape[2] != N_sz:
|
| 99 |
+
raise ValueError(f"B/C batch or dstate mismatch")
|
| 100 |
+
if D is not None and D.shape not in {(H, D_sz), (H,)}:
|
| 101 |
+
raise ValueError(f"D shape {D.shape} does not match (H, D) or (H,)")
|
| 102 |
+
if z is not None and z.shape != (x_sz, H, D_sz):
|
| 103 |
+
raise ValueError(f"z shape {z.shape} does not match x shape")
|
| 104 |
+
if dt_bias is not None and dt_bias.shape not in {(H, D_sz), (H,)}:
|
| 105 |
+
raise ValueError(f"dt_bias shape {dt_bias.shape} does not match (H, D) or (H,)")
|
| 106 |
+
if state_batch_indices is not None and state_batch_indices.shape != (x_sz,):
|
| 107 |
+
raise ValueError(f"state_batch_indices shape {state_batch_indices.shape} does not match (B,)")
|
| 108 |
+
|
| 109 |
+
tie_hdim = _resolve_tie_hdim(A, dt, dt_bias)
|
| 110 |
+
nheads_ratio = H // ngroups
|
| 111 |
+
|
| 112 |
+
out = torch.empty_like(x)
|
| 113 |
+
grid = lambda META: (triton.cdiv(D_sz, META["BLOCK_SIZE_M"]), x_sz, H)
|
| 114 |
+
|
| 115 |
+
BLOCK_M = 32 if N_sz <= 16 else (16 if N_sz <= 32 else (8 if N_sz <= 64 else (4 if N_sz <= 128 else 4)))
|
| 116 |
+
num_warps = 4 if N_sz <= 64 else 8
|
| 117 |
+
|
| 118 |
+
z_strides = ((z.stride(0), z.stride(1), z.stride(2)) if z is not None else (0, 0, 0))
|
| 119 |
+
|
| 120 |
+
_get_kernel()[grid](
|
| 121 |
+
state, x, dt, dt_bias, A, B, C, D, z, out, state_batch_indices,
|
| 122 |
+
x_sz, H, D_sz, N_sz, nheads_ratio,
|
| 123 |
+
state.stride(0), state.stride(1), state.stride(2), state.stride(3),
|
| 124 |
+
x.stride(0), x.stride(1), x.stride(2),
|
| 125 |
+
dt.stride(0), dt.stride(1), dt.stride(2),
|
| 126 |
+
dt_bias.stride(0) if dt_bias is not None else 0, dt_bias.stride(1) if dt_bias is not None else 0,
|
| 127 |
+
A.stride(0), A.stride(1), A.stride(2),
|
| 128 |
+
B.stride(0), B.stride(1), B.stride(2),
|
| 129 |
+
C.stride(0), C.stride(1), C.stride(2),
|
| 130 |
+
D.stride(0) if D is not None else 0, D.stride(1) if D is not None else 0,
|
| 131 |
+
z_strides[0], z_strides[1], z_strides[2],
|
| 132 |
+
out.stride(0), out.stride(1), out.stride(2),
|
| 133 |
+
dt_softplus,
|
| 134 |
+
tie_hdim,
|
| 135 |
+
BLOCK_M,
|
| 136 |
+
num_warps=num_warps,
|
| 137 |
+
num_stages=2,
|
| 138 |
+
)
|
| 139 |
+
|
| 140 |
+
if not has_heads:
|
| 141 |
+
out = out.squeeze(1)
|
| 142 |
+
state = state.squeeze(1)
|
| 143 |
+
ctx.save_for_backward(orig_state, x, dt, A, B, C, D, z, dt_bias, state, state_batch_indices)
|
| 144 |
+
ctx.tie_hdim = tie_hdim
|
| 145 |
+
ctx.dt_softplus = dt_softplus
|
| 146 |
+
ctx.has_heads = has_heads
|
| 147 |
+
ctx.nheads_ratio = nheads_ratio
|
| 148 |
+
return out
|
| 149 |
+
|
| 150 |
+
@staticmethod
|
| 151 |
+
def backward(ctx, grad_out):
|
| 152 |
+
raise NotImplementedError(
|
| 153 |
+
"SelectiveUpdateFunction backward is not implemented. "
|
| 154 |
+
"Use autograd through the patched selective_scan path for training."
|
| 155 |
+
)
|
| 156 |
+
|
| 157 |
+
|
| 158 |
+
def selective_state_update(state, x, dt, A, B, C, D=None, z=None, dt_bias=None, dt_softplus=False, state_batch_indices=None, **kwargs):
|
| 159 |
+
"""Dispatches single-token state updates to the custom Triton backend.
|
| 160 |
+
|
| 161 |
+
Shapes mirror upstream ``mamba_ssm.ops.triton.selective_state_update`` exactly.
|
| 162 |
+
"""
|
| 163 |
+
return SelectiveUpdateFunction.apply(
|
| 164 |
+
state, x, dt, A, B, C, D, z, dt_bias, dt_softplus, state_batch_indices
|
| 165 |
+
)
|
torch-rocm/metadata.json
ADDED
|
@@ -0,0 +1,32 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"name": "selective-update",
|
| 3 |
+
"id": "_selective_update_rocm_8104c19_dirty",
|
| 4 |
+
"version": 1,
|
| 5 |
+
"license": "Apache-2.0",
|
| 6 |
+
"python-depends": [],
|
| 7 |
+
"backend": {
|
| 8 |
+
"type": "rocm"
|
| 9 |
+
},
|
| 10 |
+
"digest": {
|
| 11 |
+
"algorithm": "sha256",
|
| 12 |
+
"files": {
|
| 13 |
+
"__init__.py": "rfNR8lwmy5vE+tOcY3z9WlRB6oKnHDWFTKGl2a1da8Y=",
|
| 14 |
+
"_autotune_configs.py": "Fa+a4MPxNOJEh8C9Y5SiYmASML4kjwbqey31khACuHU=",
|
| 15 |
+
"_ops.py": "EZezVTAP8qYzjOXulaFH/WywDDnJD6HKCaxR5MyvM+w=",
|
| 16 |
+
"functional.py": "lCB9nsg0h4srtTQewSRtmSih351l92cxFQF75WLkGeA=",
|
| 17 |
+
"selective_update/__init__.py": "DFYPlrhXwYjEqCl/8n0SmWGZV8NFml5DPhMjKfv98GY=",
|
| 18 |
+
"triton_kernels.py": "l8cpfdk/bM2bmL8etGzhr4KZdAp9o0/ypRn2IyBkmOA="
|
| 19 |
+
}
|
| 20 |
+
},
|
| 21 |
+
"provenance": {
|
| 22 |
+
"kernel-builder": {
|
| 23 |
+
"version": "0.17.0-dev0",
|
| 24 |
+
"sha": "570dcf4a810e6bd9d2433716c2bc6962c6ae46bb",
|
| 25 |
+
"dirty": false
|
| 26 |
+
},
|
| 27 |
+
"kernel": {
|
| 28 |
+
"sha": "8104c195102619f0d0f1a8087307547318b78d57",
|
| 29 |
+
"dirty": true
|
| 30 |
+
}
|
| 31 |
+
}
|
| 32 |
+
}
|
torch-rocm/selective_update/__init__.py
ADDED
|
@@ -0,0 +1,26 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import ctypes
|
| 2 |
+
import importlib.util
|
| 3 |
+
import sys
|
| 4 |
+
from pathlib import Path
|
| 5 |
+
from types import ModuleType
|
| 6 |
+
|
| 7 |
+
|
| 8 |
+
def _import_from_path(file_path: Path) -> ModuleType:
|
| 9 |
+
# We cannot use the module name as-is, after adding it to `sys.modules`,
|
| 10 |
+
# it would also be used for other imports. So, we make a module name that
|
| 11 |
+
# depends on the path for it to be unique using the hex-encoded hash of
|
| 12 |
+
# the path.
|
| 13 |
+
path_hash = "{:x}".format(ctypes.c_size_t(hash(file_path.absolute())).value)
|
| 14 |
+
module_name = path_hash
|
| 15 |
+
spec = importlib.util.spec_from_file_location(module_name, file_path)
|
| 16 |
+
if spec is None:
|
| 17 |
+
raise ImportError(f"Cannot load spec for {module_name} from {file_path}")
|
| 18 |
+
module = importlib.util.module_from_spec(spec)
|
| 19 |
+
if module is None:
|
| 20 |
+
raise ImportError(f"Cannot load module {module_name} from spec")
|
| 21 |
+
sys.modules[module_name] = module
|
| 22 |
+
spec.loader.exec_module(module) # type: ignore
|
| 23 |
+
return module
|
| 24 |
+
|
| 25 |
+
|
| 26 |
+
globals().update(vars(_import_from_path(Path(__file__).parent.parent / "__init__.py")))
|
torch-rocm/triton_kernels.py
ADDED
|
@@ -0,0 +1,127 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Triton kernels for Mamba-2 single-token selective state updates.
|
| 2 |
+
|
| 3 |
+
Matches upstream ``mamba_ssm.ops.triton.selective_state_update`` ABI exactly:
|
| 4 |
+
state: (B, H, D, N) x/dt: (B, H, D) A: (H, D, N) B/C: (B, G, N)
|
| 5 |
+
"""
|
| 6 |
+
|
| 7 |
+
import triton
|
| 8 |
+
import triton.language as tl
|
| 9 |
+
|
| 10 |
+
|
| 11 |
+
@triton.jit
|
| 12 |
+
def _softplus(dt):
|
| 13 |
+
return tl.math.log(tl.math.exp(dt) + 1)
|
| 14 |
+
|
| 15 |
+
|
| 16 |
+
@triton.heuristics({"HAS_DT_BIAS": lambda args: args["dt_bias_ptr"] is not None})
|
| 17 |
+
@triton.heuristics({"HAS_D": lambda args: args["D_ptr"] is not None})
|
| 18 |
+
@triton.heuristics({"HAS_Z": lambda args: args["z_ptr"] is not None})
|
| 19 |
+
@triton.heuristics({"HAS_STATE_BATCH_INDICES": lambda args: args["state_batch_indices_ptr"] is not None})
|
| 20 |
+
@triton.heuristics({"BLOCK_SIZE_DSTATE": lambda args: triton.next_power_of_2(args["dstate"])})
|
| 21 |
+
@triton.jit
|
| 22 |
+
def selective_state_update_kernel(
|
| 23 |
+
state_ptr, x_ptr, dt_ptr, dt_bias_ptr, A_ptr, B_ptr, C_ptr, D_ptr, z_ptr, out_ptr, state_batch_indices_ptr,
|
| 24 |
+
batch, nheads, dim, dstate, nheads_ngroups_ratio,
|
| 25 |
+
s_state_batch, s_state_head, s_state_dim, s_state_dstate,
|
| 26 |
+
s_x_batch, s_x_head, s_x_dim,
|
| 27 |
+
s_dt_batch, s_dt_head, s_dt_dim,
|
| 28 |
+
s_dt_bias_head, s_dt_bias_dim,
|
| 29 |
+
s_A_head, s_A_dim, s_A_dstate,
|
| 30 |
+
s_B_batch, s_B_group, s_B_dstate,
|
| 31 |
+
s_C_batch, s_C_group, s_C_dstate,
|
| 32 |
+
s_D_head, s_D_dim,
|
| 33 |
+
s_z_batch, s_z_head, s_z_dim,
|
| 34 |
+
s_out_batch, s_out_head, s_out_dim,
|
| 35 |
+
DT_SOFTPLUS: tl.constexpr,
|
| 36 |
+
TIE_HDIM: tl.constexpr,
|
| 37 |
+
BLOCK_SIZE_M: tl.constexpr,
|
| 38 |
+
HAS_DT_BIAS: tl.constexpr,
|
| 39 |
+
HAS_D: tl.constexpr,
|
| 40 |
+
HAS_Z: tl.constexpr,
|
| 41 |
+
HAS_STATE_BATCH_INDICES: tl.constexpr,
|
| 42 |
+
BLOCK_SIZE_DSTATE: tl.constexpr,
|
| 43 |
+
):
|
| 44 |
+
pid_m = tl.program_id(axis=0)
|
| 45 |
+
pid_b = tl.program_id(axis=1)
|
| 46 |
+
pid_h = tl.program_id(axis=2)
|
| 47 |
+
|
| 48 |
+
offs_m = pid_m * BLOCK_SIZE_M + tl.arange(0, BLOCK_SIZE_M)
|
| 49 |
+
out_ptr += pid_b * s_out_batch + pid_h * s_out_head
|
| 50 |
+
out_ptrs = out_ptr + offs_m * s_out_dim
|
| 51 |
+
|
| 52 |
+
if HAS_STATE_BATCH_INDICES:
|
| 53 |
+
state_batch_indices_ptr += pid_b
|
| 54 |
+
state_batch_idx = tl.load(state_batch_indices_ptr)
|
| 55 |
+
if state_batch_idx < 0:
|
| 56 |
+
tl.store(out_ptrs, 0.0, mask=offs_m < dim)
|
| 57 |
+
return
|
| 58 |
+
state_ptr += state_batch_idx * s_state_batch + pid_h * s_state_head
|
| 59 |
+
else:
|
| 60 |
+
state_ptr += pid_b * s_state_batch + pid_h * s_state_head
|
| 61 |
+
|
| 62 |
+
x_ptr += pid_b * s_x_batch + pid_h * s_x_head
|
| 63 |
+
dt_ptr += pid_b * s_dt_batch + pid_h * s_dt_head
|
| 64 |
+
if HAS_DT_BIAS:
|
| 65 |
+
dt_bias_ptr += pid_h * s_dt_bias_head
|
| 66 |
+
A_ptr += pid_h * s_A_head
|
| 67 |
+
B_ptr += pid_b * s_B_batch + (pid_h // nheads_ngroups_ratio) * s_B_group
|
| 68 |
+
C_ptr += pid_b * s_C_batch + (pid_h // nheads_ngroups_ratio) * s_C_group
|
| 69 |
+
if HAS_Z:
|
| 70 |
+
z_ptr += pid_b * s_z_batch + pid_h * s_z_head
|
| 71 |
+
|
| 72 |
+
offs_n = tl.arange(0, BLOCK_SIZE_DSTATE)
|
| 73 |
+
state_ptrs = state_ptr + (offs_m[:, None] * s_state_dim + offs_n[None, :] * s_state_dstate)
|
| 74 |
+
x_ptrs = x_ptr + offs_m * s_x_dim
|
| 75 |
+
dt_ptrs = dt_ptr + offs_m * s_dt_dim
|
| 76 |
+
if HAS_DT_BIAS:
|
| 77 |
+
dt_bias_ptrs = dt_bias_ptr + offs_m * s_dt_bias_dim
|
| 78 |
+
if HAS_D:
|
| 79 |
+
D_ptr += pid_h * s_D_head
|
| 80 |
+
A_ptrs = A_ptr + (offs_m[:, None] * s_A_dim + offs_n[None, :] * s_A_dstate)
|
| 81 |
+
B_ptrs = B_ptr + offs_n * s_B_dstate
|
| 82 |
+
C_ptrs = C_ptr + offs_n * s_C_dstate
|
| 83 |
+
if HAS_D:
|
| 84 |
+
D_ptrs = D_ptr + offs_m * s_D_dim
|
| 85 |
+
if HAS_Z:
|
| 86 |
+
z_ptrs = z_ptr + offs_m * s_z_dim
|
| 87 |
+
|
| 88 |
+
state = tl.load(state_ptrs, mask=(offs_m[:, None] < dim) & (offs_n[None, :] < dstate), other=0.0)
|
| 89 |
+
x = tl.load(x_ptrs, mask=offs_m < dim, other=0.0).to(tl.float32)
|
| 90 |
+
if not TIE_HDIM:
|
| 91 |
+
dt = tl.load(dt_ptrs, mask=offs_m < dim, other=0.0).to(tl.float32)
|
| 92 |
+
if HAS_DT_BIAS:
|
| 93 |
+
dt += tl.load(dt_bias_ptrs, mask=offs_m < dim, other=0.0).to(tl.float32)
|
| 94 |
+
if DT_SOFTPLUS:
|
| 95 |
+
dt = tl.where(dt <= 20.0, _softplus(dt), dt)
|
| 96 |
+
A = tl.load(A_ptrs, mask=(offs_m[:, None] < dim) & (offs_n[None, :] < dstate), other=0.0).to(tl.float32)
|
| 97 |
+
dA = tl.exp(A * dt[:, None])
|
| 98 |
+
else:
|
| 99 |
+
dt = tl.load(dt_ptr).to(tl.float32)
|
| 100 |
+
if HAS_DT_BIAS:
|
| 101 |
+
dt += tl.load(dt_bias_ptr).to(tl.float32)
|
| 102 |
+
if DT_SOFTPLUS:
|
| 103 |
+
dt = tl.where(dt <= 20.0, _softplus(dt), dt)
|
| 104 |
+
A = tl.load(A_ptr).to(tl.float32)
|
| 105 |
+
dA = tl.exp(A * dt)
|
| 106 |
+
|
| 107 |
+
B = tl.load(B_ptrs, mask=offs_n < dstate, other=0.0).to(tl.float32)
|
| 108 |
+
C = tl.load(C_ptrs, mask=offs_n < dstate, other=0.0).to(tl.float32)
|
| 109 |
+
if HAS_D:
|
| 110 |
+
D = tl.load(D_ptrs, mask=offs_m < dim, other=0.0).to(tl.float32)
|
| 111 |
+
if HAS_Z:
|
| 112 |
+
z = tl.load(z_ptrs, mask=offs_m < dim, other=0.0).to(tl.float32)
|
| 113 |
+
|
| 114 |
+
if not TIE_HDIM:
|
| 115 |
+
dB = B[None, :] * dt[:, None]
|
| 116 |
+
else:
|
| 117 |
+
dB = B * dt
|
| 118 |
+
|
| 119 |
+
state = state * dA + dB * x[:, None]
|
| 120 |
+
tl.store(state_ptrs, state, mask=(offs_m[:, None] < dim) & (offs_n[None, :] < dstate))
|
| 121 |
+
out = tl.sum(state * C[None, :], axis=1)
|
| 122 |
+
if HAS_D:
|
| 123 |
+
out += x * D
|
| 124 |
+
if HAS_Z:
|
| 125 |
+
out *= z * tl.sigmoid(z)
|
| 126 |
+
|
| 127 |
+
tl.store(out_ptrs, out, mask=offs_m < dim)
|