AntonV HF Staff commited on Jul 2

Commit

2858087

verified ·

1 Parent(s): 93e66b1

Uploaded using `kernel-builder`.

Browse files

This view is limited to 50 files because it contains too many changes. See raw diff

Files changed (50) hide show

build/torch-cuda/__init__.py +10 -0
build/torch-cuda/_ops.py +38 -0
build/torch-cuda/fla/__init__.py +26 -0
build/torch-cuda/layers.py +21 -0
build/torch-cuda/metadata.json +307 -0
build/torch-cuda/modules/__init__.py +52 -0
build/torch-cuda/modules/activations.py +1205 -0
build/torch-cuda/modules/backends/__init__.py +17 -0
build/torch-cuda/modules/backends/triton_ascend/__init__.py +422 -0
build/torch-cuda/modules/backends/triton_ascend/activations.py +931 -0
build/torch-cuda/modules/backends/triton_ascend/causal_conv1d.py +1175 -0
build/torch-cuda/modules/backends/triton_ascend/fused_cross_entropy.py +469 -0
build/torch-cuda/modules/backends/triton_ascend/fused_kl_div.py +188 -0
build/torch-cuda/modules/backends/triton_ascend/fused_linear_cross_entropy.py +347 -0
build/torch-cuda/modules/backends/triton_ascend/grpo.py +266 -0
build/torch-cuda/modules/backends/triton_ascend/layernorm.py +355 -0
build/torch-cuda/modules/backends/triton_ascend/rotary.py +211 -0
build/torch-cuda/modules/conv/__init__.py +19 -0
build/torch-cuda/modules/conv/causal_conv1d.py +129 -0
build/torch-cuda/modules/conv/cp/__init__.py +13 -0
build/torch-cuda/modules/conv/cp/ops.py +258 -0
build/torch-cuda/modules/conv/cuda/__init__.py +14 -0
build/torch-cuda/modules/conv/cuda/ops.py +233 -0
build/torch-cuda/modules/conv/long_conv.py +172 -0
build/torch-cuda/modules/conv/short_conv.py +250 -0
build/torch-cuda/modules/conv/triton/__init__.py +24 -0
build/torch-cuda/modules/conv/triton/kernels.py +683 -0
build/torch-cuda/modules/conv/triton/ops.py +424 -0
build/torch-cuda/modules/convolution.py +42 -0
build/torch-cuda/modules/feature_map.py +315 -0
build/torch-cuda/modules/fused_bitlinear.py +638 -0
build/torch-cuda/modules/fused_cross_entropy.py +459 -0
build/torch-cuda/modules/fused_kl_div.py +372 -0
build/torch-cuda/modules/fused_linear_cross_entropy.py +767 -0
build/torch-cuda/modules/fused_norm_gate.py +1245 -0
build/torch-cuda/modules/grpo.py +421 -0
build/torch-cuda/modules/l2norm.py +288 -0
build/torch-cuda/modules/l2warp.py +51 -0
build/torch-cuda/modules/layernorm.py +1472 -0
build/torch-cuda/modules/layernorm_gated.py +535 -0
build/torch-cuda/modules/mlp.py +141 -0
build/torch-cuda/modules/parallel.py +44 -0
build/torch-cuda/modules/rotary.py +519 -0
build/torch-cuda/modules/token_shift.py +573 -0
build/torch-cuda/modules/token_shift_cp.py +229 -0
build/torch-cuda/ops/__init__.py +91 -0
build/torch-cuda/ops/abc/__init__.py +12 -0
build/torch-cuda/ops/abc/chunk.py +1119 -0
build/torch-cuda/ops/abc/naive.py +99 -0
build/torch-cuda/ops/attn/__init__.py +14 -0

build/torch-cuda/__init__.py ADDED Viewed

	@@ -0,0 +1,10 @@

+from . import layers
+from .ops.gated_delta_rule import chunk_gated_delta_rule, fused_recurrent_gated_delta_rule
+from .ops.kda import chunk_kda, fused_recurrent_kda
+__all__ = [
+    "layers",
+    "chunk_gated_delta_rule", "fused_recurrent_gated_delta_rule",
+    "chunk_kda", "fused_recurrent_kda",
+]

build/torch-cuda/_ops.py ADDED Viewed

	@@ -0,0 +1,38 @@

+import torch
+def get_backend() -> str:
+    """Detect the backend by inspecting torch."""
+    import torch
+    if hasattr(torch, "neuron"):
+        # Needs to be sorted before specific Torch builds, since Neuron
+        # extension can be loaded into e.g. CUDA Torch builds.
+        return "neuron"
+    elif torch.version.cuda is not None:
+        return "cuda"
+    elif torch.version.hip is not None:
+        return "rocm"
+    elif torch.backends.mps.is_available():
+        return "metal"
+    elif hasattr(torch.version, "xpu") and torch.version.xpu is not None:
+        return "xpu"
+    else:
+        return "cpu"
+def _find_ops_name() -> str:
+    kernel_name = "fla"
+    unique_id = "3fe4aab"
+    backend = get_backend()
+    return f"_{kernel_name}_{backend}_{unique_id}"
+_OPS_NAME = _find_ops_name()
+ops = getattr(torch.ops, _OPS_NAME)
+def add_op_namespace_prefix(op_name: str) -> str:
+    """
+    Prefix op by namespace.
+    """
+    return f"{_OPS_NAME}::{op_name}"

build/torch-cuda/fla/__init__.py ADDED Viewed

	@@ -0,0 +1,26 @@

+import ctypes
+import importlib.util
+import sys
+from pathlib import Path
+from types import ModuleType
+def _import_from_path(file_path: Path) -> ModuleType:
+    # We cannot use the module name as-is, after adding it to `sys.modules`,
+    # it would also be used for other imports. So, we make a module name that
+    # depends on the path for it to be unique using the hex-encoded hash of
+    # the path.
+    path_hash = "{:x}".format(ctypes.c_size_t(hash(file_path.absolute())).value)
+    module_name = path_hash
+    spec = importlib.util.spec_from_file_location(module_name, file_path)
+    if spec is None:
+        raise ImportError(f"Cannot load spec for {module_name} from {file_path}")
+    module = importlib.util.module_from_spec(spec)
+    if module is None:
+        raise ImportError(f"Cannot load module {module_name} from spec")
+    sys.modules[module_name] = module
+    spec.loader.exec_module(module)  # type: ignore
+    return module
+globals().update(vars(_import_from_path(Path(__file__).parent.parent / "__init__.py")))

build/torch-cuda/layers.py ADDED Viewed

	@@ -0,0 +1,21 @@

+import torch.nn as nn
+from .modules.fused_norm_gate import rms_norm_gated
+class FusedRMSNormGated(nn.Module):
+    def forward(self, hidden_states, gate=None):
+        return rms_norm_gated(
+            hidden_states,
+            gate,
+            self.weight,
+            None,  # bias
+            self.activation,
+            residual=None,
+            eps=self.eps,
+            prenorm=False,
+            residual_in_fp32=False,
+        )
+__all__ = ["FusedRMSNormGated"]

build/torch-cuda/metadata.json ADDED Viewed

	@@ -0,0 +1,307 @@

+{
+  "name": "fla",
+  "id": "_fla_cuda_3fe4aab",
+  "version": 1,
+  "license": "MIT",
+  "python-depends": [
+    "einops"
+  ],
+  "backend": {
+    "type": "cuda"
+  },
+  "digest": {
+    "algorithm": "sha256",
+    "files": {
+      "__init__.py": "+xNtg61+cXuUnGoNyG/jVOEEG4VhKgccGXktjfEkykA=",
+      "_ops.py": "5604/IYHp3cW4mhYvBRXPvsI/hs5cPmXQVE+hUc1m/4=",
+      "fla/__init__.py": "DFYPlrhXwYjEqCl/8n0SmWGZV8NFml5DPhMjKfv98GY=",
+      "layers.py": "UUyLiUiIY5vj9tmJwcOZN/zqjs1zi/w8I6+72KKnFP8=",
+      "modules/__init__.py": "6ZjjRMt6Rf+2OegXDC8w58RD1IW3SBK+dxBoZa4Uohc=",
+      "modules/activations.py": "Lp7xLUf7r785nt93NFhqoSQM+G7AK2e6kviE2/0TRJs=",
+      "modules/backends/__init__.py": "l0tLqWyPFwwDRTdnHZYBzoux6J/LOFveJ53KREAKKDk=",
+      "modules/backends/triton_ascend/__init__.py": "vbU1UZwQDjVfF75INGYuWlIhuUTzx2TuZHIvme/QSMw=",
+      "modules/backends/triton_ascend/activations.py": "eqE/iXLZZEan7XHSa9ziz7mWNYwxTC5kIcJiwDwRr/Y=",
+      "modules/backends/triton_ascend/causal_conv1d.py": "VRxcTWPBB221dgd0+eNHEzCi9Echi6B2XOINtNOjHsw=",
+      "modules/backends/triton_ascend/fused_cross_entropy.py": "RITRIWoSOKhG8Mz7odkko27gESEd8M48ICc1IXRSVPg=",
+      "modules/backends/triton_ascend/fused_kl_div.py": "jb7XzKT6AKmXl8NxjZ4/SNofjdD3ioi4Df/l4KsIAtI=",
+      "modules/backends/triton_ascend/fused_linear_cross_entropy.py": "Z8/wQfPzJTg9ZICxdsABldRPsMa+mUZUTHhCJS2OeQc=",
+      "modules/backends/triton_ascend/grpo.py": "Jy+RkxbJ4RgJsEbk0xNgnw/D3Gj/P+6xDaMfx3I7DPE=",
+      "modules/backends/triton_ascend/layernorm.py": "7uKqeIraFdvcAVR0g6CLw+NJDCX8cLRMjNGHoFx1AnM=",
+      "modules/backends/triton_ascend/rotary.py": "+CmvapJJ3AOlwHbRxxSEIeVDjmlFPRno+5aqdfAHznI=",
+      "modules/conv/__init__.py": "WZKA4OJWHEN0bg2Yl2i2dvfkSp9GZKaD65abY8pSKbw=",
+      "modules/conv/causal_conv1d.py": "sHN5nlzDzSQwgo40OPt9lzcLRzZZnPCFvS/pmfK8L9k=",
+      "modules/conv/cp/__init__.py": "BKKfEjTjwyEAwDu66avCFolqnmrv9qHHNvxjfOHi3fY=",
+      "modules/conv/cp/ops.py": "pv7PGhmkfBsFt4tY8Hd0UAHzUPqp6DhgjRJmUujWitU=",
+      "modules/conv/cuda/__init__.py": "q35Kd12ITYtHp5JtaAhQJ6/YCEY95RWWveKoFPJ/VCU=",
+      "modules/conv/cuda/ops.py": "YE2httgrnIB1iSE5Uw9h1wtnq0UtybPuouF3z2reUb0=",
+      "modules/conv/long_conv.py": "h+wI2DdlBSQIZC335mwgBsjxGbByGaklb6Nv4WcQMMg=",
+      "modules/conv/short_conv.py": "DdAbuCpegRP32xEECOQQ4Ty3gWAOiFUzJLCD2mxcU1w=",
+      "modules/conv/triton/__init__.py": "fDLG2FdP68jqqsBxNdKifAIEjmvGzXNrLm/xDMrfbvg=",
+      "modules/conv/triton/kernels.py": "EK+wcpRMAGXKGmFCrZzxEeuZJmWvzUqxdTBYLwrpFT8=",
+      "modules/conv/triton/ops.py": "jIxuam5Ch5U3ErbT8D6Zn/5FqBoTq15kcJ9BGMD8exU=",
+      "modules/convolution.py": "WREsoV3gCce7rMEdpuEsyBgvmfbhk2IFdpUrr4TMtBw=",
+      "modules/feature_map.py": "sfoRvsdgvhtWkKuMYE/f1c7eyINwEaZ9TfItktWD+RE=",
+      "modules/fused_bitlinear.py": "lWTe4Ym60A+rHZTdQ6cZqa1Cje5vG/xMWvzLSAzLpQc=",
+      "modules/fused_cross_entropy.py": "si+ZNK/vu/TERnt/v2/Tm+ZBfn3poUoPDDTuJmrEuYM=",
+      "modules/fused_kl_div.py": "IDl78EMB9KXWmbV2cflnbXmcQpzdoqkwlEj4tvAPsbA=",
+      "modules/fused_linear_cross_entropy.py": "SruIxSe0Hhlf7n0upOldvn9HwOfL6XQ42N6qGZC6aOA=",
+      "modules/fused_norm_gate.py": "LAOhNgUSEiIU3Y5Ya5EHRjMAxHT0StODm04AUcL4Nys=",
+      "modules/grpo.py": "JPZnSGESIMmHDVkfYDSUzbm1qb2vY6dmDmFvSMI53RU=",
+      "modules/l2norm.py": "LchUl4mEXyx2cgHPuNgUhkorbNeuN+P7SfJyGXX0OyI=",
+      "modules/l2warp.py": "5oJlj6JRXBGe5ZANNnUA8GLIjlzXCnddtz2lwgKn6WA=",
+      "modules/layernorm.py": "1QhCZYF7pFRmZTxCWC55oG1uB6Lak5RXYca8MF5q5SA=",
+      "modules/layernorm_gated.py": "s/c/gRsZ3O6jXwY3b9owPaN1QAoOvsSevWuW2EXuJjE=",
+      "modules/mlp.py": "q09Zw5Bttqq27hnlybbQ9ncr0jSw1UCrJH2Ewh9roV4=",
+      "modules/parallel.py": "y3ZBLQp3FeDfBlHYmSerllqbfE3RgzX4k26lA15DPeg=",
+      "modules/rotary.py": "qVsb20GcFf9RolZ3dChWzEi5S9rc5UaMdC3dlrhNUj4=",
+      "modules/token_shift.py": "IWlvUPZ/Zm4IKW0dWSsx4p4r+ovbpPHQhk5Dp09P5Wg=",
+      "modules/token_shift_cp.py": "Zvaa6d/GiuljKqNgLdkjX0vysktpIVV8KMSTOjdG3Hw=",
+      "ops/__init__.py": "PsKzHYHoje01DfrVdltffiZFdS5CTVps5xl9dXr2RHo=",
+      "ops/abc/__init__.py": "WwLiIkYi3xJ5RiG/bOCNNRaeeqmMuIPT6n1g23qfpDg=",
+      "ops/abc/chunk.py": "sqvLGP2r+frjpL2qWK8GHCoWWPU5Df8IT5y20oaTZTE=",
+      "ops/abc/naive.py": "AZ3uz9eDvycW3gTwZuv9J4k86KndfQMZiLBFyPmzsLw=",
+      "ops/attn/__init__.py": "gU/67mJonbeXpzfxgi8K7HIG2a8Xpjd0gOuCk/Pu9EI=",
+      "ops/attn/backends/__init__.py": "3xcuWH+iLBPRb3g4ZkyXBXm8wJFxQK4GHdUpmK8Nh88=",
+      "ops/attn/decoding.py": "ScKiTKcbl4kcnYDNz6jXd2a1qB5ymDcz8YVkotRnO9Y=",
+      "ops/attn/naive.py": "nBsWqmBWOMQdaQYvdFejDLoX0Pei9W8oMPrHZoCCin0=",
+      "ops/attn/parallel.py": "+Vkkjy4OIwR+7SXBf6fCJqrPg740qc6a1+21RSNt9Cs=",
+      "ops/attnres/__init__.py": "/ljBk1VNFjDa9XABPXqIiQ05vUdzRV6Dbx/KkQihrWs=",
+      "ops/attnres/fused.py": "Q8RKiwlUtFZkHELCLV3wtQGf1cVDPoDNMK1waEwbxFQ=",
+      "ops/attnres/naive.py": "nV4WPz/vfRwfRsIzF7pKV95fEx2dQP4BrrPoQCh0a0M=",
+      "ops/backends/__init__.py": "3DU4e6t7dmPuZLlHjPnt4q/MAtDXtaMmAcSY4ycyytY=",
+      "ops/based/__init__.py": "qdBLUigpgjQaIN3HTtq57uxoqAaBnQkZbFiEIf18Rvk=",
+      "ops/based/fused_chunk.py": "Evgk49vwK6VxWWmw3KdwhiUV5N8RF/PcxZP8J1dlYR8=",
+      "ops/based/naive.py": "YWIFSLeslp2ruyQDz/uCr5V4ao2cyRyeQH1wiVLMB8U=",
+      "ops/based/parallel.py": "7hNqqI2UIeJNebDTF7RwW+Q9ST15DwMxJugiaVO9s/k=",
+      "ops/comba/__init__.py": "FBKonbvPPFHY7an2Auh1sXVJLbcgwX8aAD5fxi2oecY=",
+      "ops/comba/chunk.py": "o+3soCUQ27uZXe+YOZpYF1/aiLFEyTdbb+bmetY6O2c=",
+      "ops/comba/fused_recurrent.py": "ujg1r9D2Jx+/sgli1msfkXoiCZb2gX9dL/OpOca3kGs=",
+      "ops/comba/naive.py": "QH967c5rxlSSDs6J6jRtLjZ3iLiwgAYgY9+0m9rgD18=",
+      "ops/comba/utils.py": "rcW3h8gjoEzSoLgY1HkrIwsn9XKEYtz274CGqr8wWzU=",
+      "ops/comba/wy_fast.py": "EsJfyG3HDGY9rgJ97cn2SAq1DeqnXdgj5mzdOlWtz2I=",
+      "ops/common/__init__.py": "47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=",
+      "ops/common/backends/__init__.py": "aEhY8S2W0I6qbjedaCYjqLt4VaNumWM+bjKtPEJSAN0=",
+      "ops/common/backends/intracard.py": "MrZzCaTWZbmwp7zOF8HAaFHaF/kAFYXTEiyFwHnEgNU=",
+      "ops/common/backends/tilelang/__init__.py": "fG3v0hLT2R2HkFAGZ3Z/KrA5uuCIBcILI4QWui69gqM=",
+      "ops/common/backends/tilelang/chunk_bwd.py": "5zXyBzqIXRgoZdZ8zjLNEIz3jBksHpaVxnqUk66ckjQ=",
+      "ops/common/backends/tilelang/parallel_attn_bwd.py": "4OlC0Vj9N1auoz9ZCUeOcBt8QSg4eTByDUnc9BPpZ7k=",
+      "ops/common/backends/tilelang/parallel_attn_fwd.py": "7INMyGs1d3H8BKcjRnMvu/NHMm9VeNFTETR5z7ZO15c=",
+      "ops/common/chunk_delta_h.py": "RvwbMKgbr6O8HVWzPRSJ0B90gjpJKkxTqoRuxOKa/fU=",
+      "ops/common/chunk_h.py": "imyz71p9AD6nOtStpN+zzgE0yVbGnq4DO3FitrO3hvc=",
+      "ops/common/chunk_h_parallel.py": "YGNR5a+eC7UShlVAvQF3fSLK+HXoP2Nc06H3p/PyzVg=",
+      "ops/common/chunk_h_split.py": "zYoWf1Li3Q7MFPusp5BRf1F+OuGzxxTeiZ3hdunYnCY=",
+      "ops/common/chunk_o.py": "14g/dwOOarecHQQbiQ0msu7e+hL4NdqSd2HPPrgGHoM=",
+      "ops/common/chunk_scaled_dot_kkt.py": "MyoKTPKUUZlOzdC4FdP4etL6pPBGzaNL5DJQyH3603E=",
+      "ops/common/fused_chunk.py": "nKIxBC8UD00KuY7WGFysC8P4ELNLEJhHmT60VcqyHbc=",
+      "ops/common/fused_recurrent.py": "RnQdVD59iOcRa8sSK4XcQT59RhZJr42YplDaQrDt97o=",
+      "ops/common/gate.py": "ExEV4w/h/y44Qb6L1cwOFEsMutGjs/t338KimPVjeHM=",
+      "ops/common/intracard_cp.py": "R+wNrYC2m0gUn0jr5CpsxpCIVRCRkRnFXjBus9XbDE8=",
+      "ops/cp/__init__.py": "uud2WkodJvH5U5DjBa5oSLJtZ1G8O4D4XEuspXyC9lA=",
+      "ops/cp/chunk_delta_h.py": "+pPn7+hNImlTrCZLHNpaJKabA+OaoNXofYbX2F4fTD0=",
+      "ops/cp/comm.py": "lqyQDwmK19vowTtGJGO+6SEn/neFFE0G8xFvlxLOmb0=",
+      "ops/cp/context.py": "N/p87Th0z6m2cGhP6icCN7rEg3QPVua8ymP8oDD4qy0=",
+      "ops/delta_rule/__init__.py": "1gIkBajhtbVwuJOgPQ+FYYdq4tYgNgib+veOnont+N4=",
+      "ops/delta_rule/chunk.py": "r1b6g3/4OO1oDZ8n/MMyzagYHs0c5FPKU8//7KskU8w=",
+      "ops/delta_rule/fused_chunk.py": "2NiUBRAl98YgRVGr2uzYJV22pQW3WNOe6euTX1Mdqyw=",
+      "ops/delta_rule/fused_recurrent.py": "aerY1sPJ0y6IknruBVOwbu3v/pLz2PjP3aNEzsArp5M=",
+      "ops/delta_rule/naive.py": "IK4DL04IAZtCmMyfzyw5fYcfqsD9HxgCQ9n2makZKWg=",
+      "ops/delta_rule/parallel.py": "GMhpAZoEDuzyIjK0Qcpu+ku528WnLgDUujcsoRnXU6o=",
+      "ops/delta_rule/wy_fast.py": "O0UuKiOw3mImzRQHFLKHifSewFAoqRJ9i9r0puiX9fw=",
+      "ops/deltaformer/__init__.py": "51ZmNFoHy3P0lVn/2EHKu1QS2BMC6h+hnAwSIr9szYg=",
+      "ops/deltaformer/invcum.py": "ren181AxlHBv2MoZIHmSGMmJ0sW6HDb3t8fqG7gDELU=",
+      "ops/deltaformer/naive.py": "8b88rviKkDZpW+F9GwgEkVkwuSioECwgRJX//CVksnI=",
+      "ops/deltaformer/parallel.py": "0ryf3d3zmD53IorXkXXJnXQIPEg8+3PXsXMbjvzWTbA=",
+      "ops/dsa/__init__.py": "ZNyws889LPBiY8gwB5zTs2vQwJYV8bX5jg6Cldkkq9s=",
+      "ops/dsa/naive.py": "lKgrdKOoE/qCdF4DtrI0uEkRdB28kUYbtn8hO63fzyo=",
+      "ops/forgetting_attn/__init__.py": "A6ozg9zdEsz4lgUAXqEIUY8CeNp+5IZqYKOpChpj/A4=",
+      "ops/forgetting_attn/naive.py": "RDPXw01ZnhTMV125OFvEVeDDMht3BYj5gQI4oaT+T1M=",
+      "ops/forgetting_attn/parallel.py": "fcKF7Q84XXmeeKI2Tpv5XIfmxFovuXlmBov7Ie8OjKg=",
+      "ops/gated_delta_product/__init__.py": "0MAzYdxP0ODZjN4WNQ+3HLwigokyfMv15rXUpTX+qnw=",
+      "ops/gated_delta_product/chunk.py": "QNHYG+1YKzot3cQ2oWvzuv7nDoZjllEXEomCk11WUJM=",
+      "ops/gated_delta_product/chunk_deltaproduct_h.py": "5IvD8ZZxaN216Wj8dop1cwJE9P5rvZ+Si98qB6kC5DQ=",
+      "ops/gated_delta_product/chunk_deltaproduct_o.py": "nvMmsV+E9AG4Z9HA6UV69DdGMUYH2XcL8NAAubUtIHc=",
+      "ops/gated_delta_product/chunk_ref.py": "R1dHdPy4rU+lTQQKodAn9h2xPLxmwI2z0T6pRvh8Llo=",
+      "ops/gated_delta_product/naive.py": "Q907vtcavFpalFlttThkWSb/SaTa4vqAG+I/nWvKDno=",
+      "ops/gated_delta_rule/__init__.py": "Xt0hhHr3CECyZDmBIZakdufk3bvvIa/V/iBmpkYwpnw=",
+      "ops/gated_delta_rule/chunk.py": "DFEZWmp016p+65pnDynAMVOY+NsUTGOfVcXsAhQr1i4=",
+      "ops/gated_delta_rule/chunk_fwd.py": "DisrNyeiIB6SxcYZn5jsUnrbmkLzVbcqQubFI14qLsY=",
+      "ops/gated_delta_rule/fused_recurrent.py": "M7x3/U1Xnf5mndUc+lvGpzxu4KDNc2Sfevv+RVZZFNk=",
+      "ops/gated_delta_rule/gate.py": "H9JLPKjgZmorhyV+1IJFSMIa6x9admDn+EXXlwg167g=",
+      "ops/gated_delta_rule/naive.py": "0c8XmSNJ/T6Ur5mbIuPTqBvkotGIHOW3CjRXJXFm4Ms=",
+      "ops/gated_delta_rule/wy_fast.py": "ufSeCLwEQxwRS4pYiiNsIeXN3A6sLKgLdu1Fy/gkXoM=",
+      "ops/gated_oja_rule/__init__.py": "oShC/cL0jMy+xWR7YBV9QxkAdxt1xasXDCypGAU6Hgo=",
+      "ops/gated_oja_rule/chunk.py": "lq+5BXEaJLfL4dqqcJb4TmtyF9OrP9Wh+JpRfpdzd2s=",
+      "ops/gated_oja_rule/chunk_h.py": "kr6KBcPbFRrIQFBoQjmt1wsyvO1pyLeGW5v/k+NI87Q=",
+      "ops/gated_oja_rule/chunk_kkt.py": "N/sU3E2i5+jQBXnFuWyaUAiQkAgWUxNjO9lxdlpEkDA=",
+      "ops/gated_oja_rule/chunk_o.py": "4tPVh8KWj+D2onfXbIhnhaGWq35az5uJUwxTUbbRll0=",
+      "ops/gated_oja_rule/fused_recurrent.py": "yPfAcIXWBlr50lvK59bbF2qkhKWt40eT+EXx6exojK0=",
+      "ops/gated_oja_rule/wy_fast.py": "mS9GSN2RVUbEMkz2o1Hfi6j0nOhXtjLLSIccXkwmW3c=",
+      "ops/gdn2/__init__.py": "xiEsgvVEfHhKcurQqL7ZKGGzEBW2BtGAylE9vxoNmAk=",
+      "ops/gdn2/chunk.py": "zgJK/Mwhn1fA9PpvagLdy6yFuxo2c1Z8DmUY5FJp2kw=",
+      "ops/gdn2/chunk_bwd.py": "Ay8kHSsnyD1wxGE1YcEfV3d4L3yae/8bDBMReKPbVBw=",
+      "ops/gdn2/chunk_fwd.py": "7Z0d9SnKSby4yCAeXx5kKx6HIdYfaXl0ZhYDDAniClU=",
+      "ops/gdn2/chunk_intra.py": "UjnmvfEg9EB41jGA9YHcZT9R2oHpBQI4PCq1Dwj1ZiQ=",
+      "ops/gdn2/chunk_intra_token_parallel.py": "U1lf9FZ6mfW2iIm06YJ5qPX8IQeT4U/G+J7RlZoWutw=",
+      "ops/gdn2/fused_recurrent.py": "3QsJl80MRQYrQXVN9lmgw4cVD1hLHrQXg6xcIgU5Bz8=",
+      "ops/gdn2/naive.py": "amWAVAHrMA34W2KXYRoN0VjdGDQjzyvNXg8Ozd4+nno=",
+      "ops/gdn2/wy_fast.py": "DNH9FhRaZl4nqZ/YrxH2SqCgZdC77bkJ24sFIemnkRM=",
+      "ops/generalized_delta_rule/__init__.py": "cn6rKWEDrxWw428Dk6EEy4Y7nyWHpc8rtR+EbRO9ndA=",
+      "ops/generalized_delta_rule/dplr/__init__.py": "F8TbaiZeRKjbKhlI4XYopNd4bfzV10feNO06i32h2Vo=",
+      "ops/generalized_delta_rule/dplr/chunk.py": "qeR4srXoGT8NjNx93IQEyPGq3sqsK78CdOovbRwbntU=",
+      "ops/generalized_delta_rule/dplr/chunk_A_bwd.py": "MDoA0HOWDti16rAh8Q4D4RJonJkmIN04dCwUsHOM+sk=",
+      "ops/generalized_delta_rule/dplr/chunk_A_fwd.py": "wQSOE/PoqvjP6ugh3wSmbsst0ECSclxJEl5x/4NQ0KI=",
+      "ops/generalized_delta_rule/dplr/chunk_h_bwd.py": "hAISrlVF9QETkcoDAgEKXRY0w9DSDKdohqTXfkg421E=",
+      "ops/generalized_delta_rule/dplr/chunk_h_fwd.py": "vWtEs8L4DXX52UF5qt6Vuu/pOsHzKOFOIFtyezheWPY=",
+      "ops/generalized_delta_rule/dplr/chunk_o_bwd.py": "CuHt6fYI2g/6m0k2mj7JSLhCQ+zcKrMUYc2LOmKuUUs=",
+      "ops/generalized_delta_rule/dplr/chunk_o_fwd.py": "KOF8wWIEMIyhDfSL44bucPPsCp6u3AdYPlRDKK10KyQ=",
+      "ops/generalized_delta_rule/dplr/fused_recurrent.py": "ABaz+l4n77U9FrE3bdsGx/5jexAnvJ1cJcI/XJkeIac=",
+      "ops/generalized_delta_rule/dplr/naive.py": "4flLSZUtwaq5S/PH+fu04rzOM2+USdRX6Ofte4lanE8=",
+      "ops/generalized_delta_rule/dplr/wy_fast_bwd.py": "HiMBn/U+a0vSCOVmn29UYLylI5x26qqualdl4E6ipd0=",
+      "ops/generalized_delta_rule/dplr/wy_fast_fwd.py": "R+CamNa7b49LfvrNZQs4roi+TReWcLGvBcoT3DbhoLM=",
+      "ops/generalized_delta_rule/iplr/__init__.py": "6Vg8MTc+Byrnmx3HJz2yySi9nk2kyKCSg7aL4CMJ+GA=",
+      "ops/generalized_delta_rule/iplr/chunk.py": "BsCf/fiYn3wpS0eDPrR+mCrqsAihXD5v/QD2QkA5ATc=",
+      "ops/generalized_delta_rule/iplr/fused_recurrent.py": "nNo+ww/QmydOqBLnpmxXHAbANGDPTeWGjFI7nA9Vrw0=",
+      "ops/generalized_delta_rule/iplr/naive.py": "YotccLOueXwBcQys4ICmFAysSrX8fj9eRasXIpmpRMo=",
+      "ops/generalized_delta_rule/iplr/wy_fast.py": "SoTcF2asZEIAnZS1qR9PXHGkHwnBspjuUsz6RpuqNiU=",
+      "ops/gla/__init__.py": "PTUqqQlC4cXrhRhG/KRx+7X3m9WJKxGyzrIB/WJEXSo=",
+      "ops/gla/chunk.py": "B7reIP0UzTvYgK8Ds50r0uABlaucJWm70JVKKdYe1L0=",
+      "ops/gla/fused_chunk.py": "OjzKcjSQujVQ+Q+d1Lmg4tsTzZaK6xWs8AGuplRn5zo=",
+      "ops/gla/fused_recurrent.py": "ZIzMstFj8WAXagDzTlZo3hFZy0k+egHNg8Qr61sb4vw=",
+      "ops/gla/naive.py": "rEuH5bEAieya1CTvtZbebVFo2J5LdOX1ztL1WiDkZB0=",
+      "ops/gsa/__init__.py": "9uygbV6orNtaTkZl9Hz+aAp7EvT6RuffawfeFtuUNEE=",
+      "ops/gsa/chunk.py": "SsfwmTOpKRG2Fja/lArJkw1teaomk/Vgczjr2/Oa+l0=",
+      "ops/gsa/fused_recurrent.py": "K7qBWOmh0uL9CO7D3pSlrxKFHy1fobTUyTFV5p7okXg=",
+      "ops/gsa/naive.py": "rezNszsU38KD1iHBW+ErV/Z01T5LIPadEjJnWiXz9LI=",
+      "ops/hgrn/__init__.py": "69DMv7FQYvobQTXqOFtu7muAt1k1k9nvsFXcMAmjQmg=",
+      "ops/hgrn/chunk.py": "Br3GeHkw1EcOF/4X2BOvUTclhfLDZwkthIshoHTMMqg=",
+      "ops/hgrn/fused_recurrent.py": "KvO/5l/3wnJiT0jT0UA9IYxQhGHJy8jC7lYaOOyLM1w=",
+      "ops/hgrn/naive.py": "XKo+5zJ+G4elkgK5ARsoXm5WYkG1QidTKNIuPT4k588=",
+      "ops/kda/__init__.py": "JFZPAQH4eiYFbpBh7Xccrw2PbZoAtNve1wHqdSW0Wss=",
+      "ops/kda/backends/__init__.py": "pk1xLku49AblKggFmQjmGlr1oNqUgdzxggzzgx8yeZk=",
+      "ops/kda/backends/flashkda.py": "j2FmWHLh09/B1EE0SFrNw+OR7JFOJuD5O4x49bNpKRg=",
+      "ops/kda/backends/tilelang/__init__.py": "w+bbm7//9PdpfGGOTCOxsTwrC95ERRuYmxPROcEXHLc=",
+      "ops/kda/backends/tilelang/chunk_bwd_dqkg.py": "Fk1eT3M/KPFvfVRrJQ6C0158h5GaR72YK3tgSxkKoUo=",
+      "ops/kda/chunk.py": "LnoIH4MI8gHR8613fOKLqJGVNQkbtzjEJ0R7qX8/kQE=",
+      "ops/kda/chunk_bwd.py": "AahkP3AepxWsoyGCZHlwUGc7rGIHJh68Im5r/Zs3MK0=",
+      "ops/kda/chunk_fwd.py": "wUTWWQVWLRGeDj9e7dO03+0IosaRuWAfYqsl2KBy0ZA=",
+      "ops/kda/chunk_intra.py": "Ao7C9D/HzYJkiFxIL5s/LIlVE2YNgpL9yASJl5giPOs=",
+      "ops/kda/chunk_intra_token_parallel.py": "ENjjhDAR0cK2mqoU4q7QpBMPHmICgoLO06e4dEnd5hY=",
+      "ops/kda/fused_recurrent.py": "r6ZJMiLVmCRLpbYXS25mIX58KU2yKtDJqGwwHTXY0MY=",
+      "ops/kda/gate.py": "5056ezuomW2CNvPE04MraGJtf0gHvGqicxin1H0DICg=",
+      "ops/kda/naive.py": "YKMihdS2cGj/YztIu+irMQKAZtJPANJ+EhmaiPxz8BY=",
+      "ops/kda/wy_fast.py": "9pMMqS5E/l0mFV/Tyap1QKY39CB5Iy94/Yn4s9WzjUw=",
+      "ops/lightning_attn/__init__.py": "FPVVYNLjfO3GJQYIp+4HCQ4MdJtF8UZx5yktWDIatFA=",
+      "ops/lightning_attn/chunk.py": "kFjfxpUOVj61TnH0vXWZhDnw318suRLhtT2Qo50pkKM=",
+      "ops/lightning_attn/fused_recurrent.py": "SsJ+HC5opio7H9LsPWyrbnPOoK2EvOcxLuaq8Vp69GY=",
+      "ops/linear_attn/__init__.py": "3OeyduLmyi1manyo7FQz1ZLSoLgEXAKbHKn1uqDcK90=",
+      "ops/linear_attn/chunk.py": "5xqgOnPPA7nGPt/Rda4wOxSljFTD3Izw63VpuDLJNd8=",
+      "ops/linear_attn/fused_chunk.py": "MD6FycgZSdn8jV88UDvxpioPAw6hCxvsSq49qNIKpmo=",
+      "ops/linear_attn/fused_recurrent.py": "jUZ5gPuN2VngqbebHbN0lrIAOiZOYkxIku+C3Eh/9Vw=",
+      "ops/linear_attn/naive.py": "46BX2exbemO1c8XXVhmkz2m6H1ywTmrvT4/odStEA8g=",
+      "ops/linear_attn/utils.py": "7X94oxSkHAGSpk7SQ5izs5Q0rYEDZZzUV/RorFdzY1o=",
+      "ops/log_linear_attn/__init__.py": "qsAhfXSfoQBDmP6Li6+CjNd9ysMDGjcbYLonvcZcrjU=",
+      "ops/log_linear_attn/chunk.py": "NTaU1ErT6E0PzSt9JL9KayX8dQgLG++85PFwj3sRm1Q=",
+      "ops/log_linear_attn/naive.py": "xmXXuoIdEkTucVYJ/s4dTCvMWuJuO8KP0299J4jdtEU=",
+      "ops/mesa_net/__init__.py": "32w1RDjxggXJ9JobAR01dDnN2UZSG2Fui7qePyQT2J4=",
+      "ops/mesa_net/chunk.py": "GTupTJ0BI3Ix2giVi7sMA2ocs71yvZ4uuKvf8rkUqKQ=",
+      "ops/mesa_net/chunk_cg_solver_bwd.py": "MvXMHU8MSOTNG62zURV+iMpvkwEh4eCo7epde0DIvBc=",
+      "ops/mesa_net/chunk_cg_solver_fwd.py": "KG4SXzD6dZomzRmYmqaWtmBoYVgSk6Fxda8uXjJc5go=",
+      "ops/mesa_net/chunk_h_fwd.py": "F3AUgFAYjqUAxslqpjsv+zW1PjdxGnwaU/Qjh5tT0iw=",
+      "ops/mesa_net/chunk_h_kk_intra_bwd.py": "17Le7w0rK0MrLUROFQqcZ9yCQfxpRYMhe5o3dRHGIfo=",
+      "ops/mesa_net/chunk_h_kv_intra_bwd.py": "mQnvHOYVu3HMX6IA4rD2zl+9w5BOi2yzFsr0GSo5+Nw=",
+      "ops/mesa_net/chunk_h_kv_intra_bwd_separate.py": "FOrwfCVxiMtLYaR9JIBqzxExwDWMWPVuOkaxgHD1AgY=",
+      "ops/mesa_net/decoding_one_step.py": "ZFuKB5H06F+uDagQLPGiu+Qk/qpZxjRbKcnuqWtC6/0=",
+      "ops/mesa_net/naive.py": "zv3jT+71UegNb0JyIKnPv8OddyzHu4weEETtt0nJvzA=",
+      "ops/moba/__init__.py": "P7ARNbLLMsmTtxcxPk09NPmjcvHGwWCmooXPYvT6ANk=",
+      "ops/moba/parallel.py": "8qUN2zo88Pldh809UyDowBVj8kCNplsS00fIxLRdSyQ=",
+      "ops/nsa/__init__.py": "6Mw2zDl4Z74QXVjME1d88PMSKTSO/AYUa00lW/IK+eI=",
+      "ops/nsa/compression.py": "XjFqvEXvuIEeCBrJU+YzhWRH5UAkPGcUX4Tz/xx2M6Y=",
+      "ops/nsa/naive.py": "5A7lyoN39oK0Wc+Yzm/dQbNEtlaGmZfJsMDv8II+m7c=",
+      "ops/nsa/parallel.py": "BLaPXXsO8tcAJ79ltV3xwX/1qTganF2Nh+Lto/3zvyQ=",
+      "ops/nsa/utils.py": "lbhybLC0CqXTn6liY8d1sQfTdZ2Gl+++Y5Go53OX4xE=",
+      "ops/parallax/__init__.py": "g2pTOybF7cyoyaMAku1gpjV5MBY6s/Ll1X2NT6/w/YM=",
+      "ops/parallax/decode.py": "qHX0Le4hA66S2LLTcMa+p0VMB0uWvhYGI9urSPLrB4I=",
+      "ops/parallax/naive.py": "XEvAU0cSJtunA3ElSy4GFs6idarjK7Vb+xkrDKSp2OQ=",
+      "ops/parallax/parallel.py": "ZrLKoopSj60DSxmqbaS3MIf3AUyEpYh7evJPlTUN+AQ=",
+      "ops/path_attn/__init__.py": "eyvdV5LkkxztzDTDYAl862vhXITUOcMtF6D8n6n0O/E=",
+      "ops/path_attn/cumprod_householder_bwd.py": "EZ0t+c2hG8Z/0IvrVxMBLDgO7zLgJsI9Bc7L0WYgvh4=",
+      "ops/path_attn/cumprod_householder_fwd.py": "UF1wvSIMRHeJmPf5qbFuxbC+qCm3ipAfeAVf3IEOA6A=",
+      "ops/path_attn/intra_chunk_preprocess_bwd.py": "HMQYfpKCHEMmzomrzb60u3gJK419yOAg2kx9XolFf6U=",
+      "ops/path_attn/intra_chunk_preprocess_bwd_prepare.py": "6XIZztbrFYFR/7XJSIy10RxKid3+6dcZ5c+A6XL2fBs=",
+      "ops/path_attn/intra_chunk_preprocess_fwd.py": "U8P46fLq//0vDN0SiE3cwI4n6FbAO6/UGNhfOiXegRo=",
+      "ops/path_attn/naive.py": "4QsKiZUcF0q4IJbrku4Xu/IR3YBg+n6Epio7FAOW9lk=",
+      "ops/path_attn/parallel.py": "V1c/T2VKbs8MgNJ0AT2JA5ULpx49aLb5maoxizAbVjk=",
+      "ops/path_attn/parallel_path_bwd_inter_dkv.py": "6/EOBq9NQdKaCrEGwPwbi69vKbOoNCsKU1q2vRv7YFc=",
+      "ops/path_attn/parallel_path_bwd_inter_dqh.py": "fp1NEXRSVAqhkmGcvoNfQn2pt5dtD0plx6W0TGR2xDw=",
+      "ops/path_attn/parallel_path_bwd_intra.py": "+3QE/gzK3fAYqgo/WNHNC5LV8NJrL4+UHjDN6wOd15w=",
+      "ops/path_attn/parallel_path_fwd.py": "cx6fGpfZmEuxHhl81gsLOBAYa61ZnkI9usqTakOIV7A=",
+      "ops/path_attn/prepare_k_cache.py": "LqEoXyDlVfIaJQhrmpoS3pmetIaavAm+SmIm1NKFohQ=",
+      "ops/path_attn/transform_q.py": "uZ9u6JDzXFKkWiim4KUkXWoAHYKlSDK5L1fMBNFrEGY=",
+      "ops/rebased/__init__.py": "pawYkzOJ/mQY5Qj0Ibuq81P3KNgO9Lv6swOxCjlDlGs=",
+      "ops/rebased/naive.py": "XOrMn2Ohd7MDJ7QvP93dPfjv/6prnZW/ozxHtU0KqWQ=",
+      "ops/rebased/parallel.py": "+3WRG3oxV0T78e8HWkN32Q8KBe0V2GMQ/OpxowWfXFI=",
+      "ops/retention/__init__.py": "i6uHA1ndTR6T/fVfooceKo12wyBFGYLe1EAm0Mo/sg0=",
+      "ops/retention/chunk.py": "x0fX3U8KTjr9WLQK9+2C6Mn7cYNtACQ8RhFuUoCGpBw=",
+      "ops/retention/fused_chunk.py": "sAmIqFiH4IFM4yiM1G/pNvlsZxLtkORemYy5WW4i1pw=",
+      "ops/retention/fused_recurrent.py": "beES/zDbxCUWWWL9HSCc72a1+DWVvDKpBIsjKjkUet0=",
+      "ops/retention/naive.py": "MG107SmOiRKLTZ8ziqaWN85nbgIPj/GqNJJ7KMt5Tag=",
+      "ops/retention/parallel.py": "gTnLcV5w17fZAOJyJttw52LGAe9XlkuS++lMGTZ3vRs=",
+      "ops/rwkv4/__init__.py": "GnWakdtZiQJ0+fTYxqqIyosoPwjH6Yp8P1AbpHIDnts=",
+      "ops/rwkv4/fused_recurrent.py": "0fvq/4+NZMq6tzq7oHMUCaPg10qOzteSKLj5cmQKhvk=",
+      "ops/rwkv6/__init__.py": "wPYgC6wG+xc4I2vessGhv1EaZ4tL5yuVQVSwi1rEEUk=",
+      "ops/rwkv6/chunk.py": "2VMAk6p0tKEudhieXLIg2bYg+89JbsGUiuhFpfvx6C8=",
+      "ops/rwkv6/chunk_naive.py": "zyj0Og15jGxNMzmlD9FoEdf/odzu3tqio/jiceVMNk4=",
+      "ops/rwkv6/fused_recurrent.py": "wMTP/Y17ayzQXxXw+fN8P/FB5KGefLB4bB0wJ9kDg6Q=",
+      "ops/rwkv6/recurrent_naive.py": "/lCgua1vSaJDqA4tOWWUaVxol5Axi3cOcjI+mJj7OAM=",
+      "ops/rwkv7/__init__.py": "dPKBISLNnwlKb+xp/8ejsc2peC9gKQ3NedtMT5wQ1sE=",
+      "ops/rwkv7/channel_mixing.py": "+XrwuE2RjLxjsiKrdhssskBTvW/zL9RpyFp85uv5J5A=",
+      "ops/rwkv7/chunk.py": "9+rVTjACj0FP2Mnwo30DHWIHkpZG6YfYN4gE6891Oa0=",
+      "ops/rwkv7/fused_addcmul.py": "uUzh8wc75jBLJhCagQfeE/yor25Jw34d9kQQNsQWg04=",
+      "ops/rwkv7/fused_k_update.py": "CETn2jde9U8K8kfVihcyjARgpcDT9nN09qw9wM1QQac=",
+      "ops/rwkv7/fused_recurrent.py": "nnNrjgLS5nv9plvU/MuMjt4/vDcGD0Wax9Dfigvwmp8=",
+      "ops/rwkv7/gate_output_correction.py": "Ah6eqJrlTgFGaGI711+pMmWlxCOpvHrExx8H9U+nO0Q=",
+      "ops/simple_gla/__init__.py": "sSyIh8oSfhyZI+VnYUM/uhIp+HL+gDcQBReLn/nJ5OQ=",
+      "ops/simple_gla/chunk.py": "eA/L+T+42GiIhYu1oG0fM66QFGnzW9Uz8oc2SnLTKig=",
+      "ops/simple_gla/fused_chunk.py": "eCALWbi61vaG6zPewlPmATvjlidM9lso9xQC8H4xAlM=",
+      "ops/simple_gla/fused_recurrent.py": "Ab98sk1dnnFQmBmueO3D95vUs0zK8PC30ZRJYEGaIOM=",
+      "ops/simple_gla/naive.py": "g3fAN++vd0aWvZcdOW7Y0boMYcs2mOld8kePWRbl2R0=",
+      "ops/simple_gla/parallel.py": "hB6Zq+allqAYg/H8/9kpk0l7bsZCK9IsITkfLIhqZQ8=",
+      "ops/titans/__init__.py": "4t9OK1aYdW03I+sUlfUjj3ALTez1M/XFD/H6KzeZtGc=",
+      "ops/titans/log_impl.py": "Y5tMWJgtVE+GX394DdxQHd6Yirrg3sLx6sinMdQvkAE=",
+      "ops/titans/naive.py": "NUmchQQpsxFwNaEpSQbwZ/638zSr0tuy2xGa3DD07+A=",
+      "ops/ttt/__init__.py": "cFaLRxAWRsKjcmXh01bJbh5U59ZaFaM7DwYgm/KBAbo=",
+      "ops/ttt/chunk.py": "xk4Cw5LLMUKTauseXa+oybDIOp6A0e+McU/vqzp3KhM=",
+      "ops/ttt/fused_chunk.py": "KDdTpBvHj3jcIm2EkkNZTxWkhGbf0qvSQYEFkTwf+d4=",
+      "ops/ttt/naive.py": "IaJu/Zd1/PgFA5Bnbo9WQkpdz6BPZD0bl8iGKkWQlCo=",
+      "ops/utils/__init__.py": "eUiV7OAFYzGDyI/WtrSNltVrfU3crATGfKFWo9U87Hc=",
+      "ops/utils/backends/__init__.py": "Bfq8p7M9iRlUzfIfPTlvsKd/LMutWWtEzp2wfwOOK54=",
+      "ops/utils/backends/triton_ascend/__init__.py": "PEm11g9OLX2mdI66BW5JI+xL9vxIyNhZFzJxxGRaEkY=",
+      "ops/utils/backends/triton_ascend/cumsum.py": "AOxEn57vvwiNwOaDcVK3KbP+n5IUGrsbUWkG+kkukBg=",
+      "ops/utils/cache.py": "pksJ/9P1Gu1UetclWd7pTLmur9Kl8CxsjmCm4CAUepw=",
+      "ops/utils/constant.py": "uVK9zwOd7kM/DPSQk8cllx5nkYeK0K/KZVyCu0ZFdPg=",
+      "ops/utils/csr.py": "comXJRo/nULFkkZk556qHNxBfWeXk0EofLnQ69EIEjw=",
+      "ops/utils/cumsum.py": "6oQ1YDSIgMFZOZRn2oX8VKy/O+B5mbVWXl6146frCmY=",
+      "ops/utils/index.py": "BdattFUI0BkVExltE0Zxl/BNO9Ldp6hSUfU1zbM3710=",
+      "ops/utils/logcumsumexp.py": "7uEgeqUuLxi06jN/hOcTHvqJEITDGXECjNzk5SKPce0=",
+      "ops/utils/logsumexp.py": "7g4hdIjRKkO8CylqAs80XR02A5PyIWYxrKcc79qRcy4=",
+      "ops/utils/matmul.py": "TOLhNlPSm2JJeFmVsgh5D3t9LPUjw30w+mq5TIHef5Y=",
+      "ops/utils/op.py": "nFuDfGU88dXF1Pe8xrcxoda8QWZA5QTojiFsse/53fQ=",
+      "ops/utils/pack.py": "XT3/YxB71p6VO9oXPkpD0te42kbwU0dFaG3aZawdBHQ=",
+      "ops/utils/pooling.py": "LB2lncX1ThcjDIEWvMwiQpW2T8KE/AQ5dTqcJC/hNhs=",
+      "ops/utils/softmax.py": "uq/rygFZaAImpnHgOmAN9D+jrq/m3WHTyu6QGQR/EoU=",
+      "ops/utils/softplus.py": "tdlxnzZRHWEZWhVl9uFVl/EGQ6phOBMEEwlbCtu8jz0=",
+      "ops/utils/solve_tril.py": "oy0zat1/r4IjvqQ0MkJbrn1MYoayluYORr3nvX7z9NE=",
+      "ops/wall_attn/__init__.py": "r/FbDGjGO8WK1DaMbl9axWHpENUa3pzSc0v9qviHJ6I=",
+      "ops/wall_attn/decode.py": "JphI1kWj0Wwu024gnwgUWrWmhkAnKloXi1ik6cDpRrY=",
+      "ops/wall_attn/naive.py": "9YKwBSvQ9imy229T5KMYKpuVNb+xEzhXsXdMemRWqjQ=",
+      "ops/wall_attn/parallel.py": "oa2UJtSi8ouSG8g6bUWHwOehbwLH74SSHIU8LUiwVrg=",
+      "utils/__init__.py": "bdtrr5b2e7dzcO88Il6bH72LypiM103tHkbY2kPoaeE=",
+      "utils/_compat.py": "kYAtqK4JdTczM1FpqT44T/mILN7DcnPb8C4uPnO4Ho0=",
+      "utils/_config.py": "l1PBnf2C6SABltOAhs9GB0GTr0olQ1fA7d/bhzTb6IA=",
+      "utils/_decorators.py": "8OxDii2b+D+deTCQLPK1sdFx7McS/Wi9+AO67mlLrYg=",
+      "utils/_device.py": "yXcM5ZkBLCOvHl/Dh25SveTgbSemR6PkFAl/UabLYWE=",
+      "utils/_testing.py": "eFchc/74QJDXWv5T1YmIZFzRRpFWh+3gaYpQ1doztzs=",
+      "utils/ascend_ub_manager.py": "TcjVL6L7YiNwsa7YZ4+9KrIPQ2RmG7PJCfmXYc24uEc="
+    }
+  }
+}

build/torch-cuda/modules/__init__.py ADDED Viewed

	@@ -0,0 +1,52 @@

+# Copyright (c) 2023-2026, Songlin Yang, Yu Zhang, Zhiyuan Li
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+# For a list of all contributors, visit:
+#   https://github.com/fla-org/flash-linear-attention/graphs/contributors
+from ..modules.convolution import ImplicitLongConvolution, LongConvolution, ShortConvolution
+from ..modules.fused_bitlinear import BitLinear, FusedBitLinear
+from ..modules.fused_cross_entropy import FusedCrossEntropyLoss
+from ..modules.fused_kl_div import FusedKLDivLoss
+from ..modules.fused_linear_cross_entropy import FusedLinearCrossEntropyLoss
+from ..modules.fused_norm_gate import (
+    FusedLayerNormGated,
+    FusedLayerNormSwishGate,
+    FusedLayerNormSwishGateLinear,
+    FusedRMSNormGated,
+    FusedRMSNormSwishGate,
+    FusedRMSNormSwishGateLinear,
+)
+from ..modules.l2norm import L2Norm
+from ..modules.layernorm import GroupNorm, GroupNormLinear, LayerNorm, LayerNormLinear, RMSNorm, RMSNormLinear
+from ..modules.mlp import GatedMLP
+from ..modules.rotary import RotaryEmbedding
+from ..modules.token_shift import TokenShift
+__all__ = [
+    'BitLinear',
+    'FusedBitLinear',
+    'FusedCrossEntropyLoss',
+    'FusedKLDivLoss',
+    'FusedLayerNormGated',
+    'FusedLayerNormSwishGate',
+    'FusedLayerNormSwishGateLinear',
+    'FusedLinearCrossEntropyLoss',
+    'FusedRMSNormGated',
+    'FusedRMSNormSwishGate',
+    'FusedRMSNormSwishGateLinear',
+    'GatedMLP',
+    'GroupNorm',
+    'GroupNormLinear',
+    'ImplicitLongConvolution',
+    'L2Norm',
+    'LayerNorm',
+    'LayerNormLinear',
+    'LongConvolution',
+    'RMSNorm',
+    'RMSNormLinear',
+    'RotaryEmbedding',
+    'ShortConvolution',
+    'TokenShift',
+]

build/torch-cuda/modules/activations.py ADDED Viewed

	@@ -0,0 +1,1205 @@

+# Copyright (c) 2023-2026, Songlin Yang, Yu Zhang, Zhiyuan Li
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+# For a list of all contributors, visit:
+#   https://github.com/fla-org/flash-linear-attention/graphs/contributors
+"""Fused activation kernels.
+The kernels address their inputs through the row stride instead of assuming a fully contiguous buffer.
+An inner-contiguous input — such as one half of ``x.chunk(2, dim=-1)`` — is therefore read in place, sparing the extra
+``.contiguous()`` copy (and its memory traffic) that a plain flat element-wise kernel would force on every call.
+"""
+import torch
+import torch.nn.functional as F
+import triton
+import triton.language as tl
+from ..modules.backends import dispatch
+from ..ops.utils.op import exp, log
+from ..utils import IS_AMD, autocast_custom_bwd, autocast_custom_fwd, autotune_cache_kwargs, input_guard
+NUM_WARPS_AUTOTUNE = [1, 2, 4, 8, 16] if IS_AMD else [1, 2, 4, 8, 16, 32]
+def _get_stride(x: torch.Tensor) -> int:
+    """Get the row stride for viewing a tensor as 2D (num_rows, D) where D = shape[-1].
+    Returns stride(-2) if the tensor is at least 2D, or 0 for 1D tensors.
+    The caller must ensure the tensor is "inner-contiguous" (stride(-1) == 1 and
+    higher dims are contiguous relative to dim -2) before using this value.
+    """
+    if x.ndim < 2:
+        return 0
+    return x.stride(-2)
+def _is_inner_contiguous(x: torch.Tensor) -> bool:
+    """Check if a tensor can be safely viewed as 2D (num_rows, D) with row stride = stride(-2).
+    This holds when stride(-1) == 1 and all dimensions above -2 are contiguous
+    with respect to the dimension below them.
+    """
+    ndim = x.ndim
+    if ndim < 2:
+        return True
+    if x.stride(-1) != 1:
+        return False
+    if ndim == 2:
+        # 2D: any layout with stride(-1)==1 is valid (can view as (T, D))
+        return True
+    if ndim == 3:
+        # 3D (B, T, D): stride should be (T*D, D, 1)
+        return x.stride(0) == x.stride(-2) * x.shape[-2]
+    if ndim == 4:
+        # 4D (B, H, T, D): stride should be (H*T*D, T*D, D, 1)
+        if x.stride(1) != x.stride(-2) * x.shape[-2]:
+            return False
+        return x.stride(0) == x.stride(1) * x.shape[1]
+    # 5D+ fallback to loop
+    expected = x.stride(-2) * x.shape[-2]
+    for d in range(ndim - 3, -1, -1):
+        if x.stride(d) != expected:
+            return False
+        expected *= x.shape[d]
+    return True
+def _ensure_inner_contiguous(x: torch.Tensor) -> torch.Tensor:
+    """Make the tensor inner-contiguous if it isn't already."""
+    if _is_inner_contiguous(x):
+        return x
+    return x.contiguous()
+def _alloc_output(x: torch.Tensor, contiguous: bool = False) -> torch.Tensor:
+    """Allocate the output: a fresh contiguous buffer, or ``empty_like`` otherwise.
+    ``empty_like`` keeps the input's memory format only when it is dense; a non-dense
+    strided view (e.g. a ``chunk`` slice) falls back to contiguous, not the input stride.
+    """
+    if contiguous:
+        return x.new_empty(x.shape)
+    return torch.empty_like(x)
+@triton.autotune(
+    configs=[
+        triton.Config({'B': bs}, num_warps=num_warps)
+        for bs in [512, 1024, 2048, 4096, 8192]
+        for num_warps in NUM_WARPS_AUTOTUNE
+    ],
+    key=['D'],
+    **autotune_cache_kwargs,
+)
+@triton.jit(do_not_specialize=['T'])
+def sigmoid_fwd_kernel(
+    x, y,
+    stride_x_row,
+    stride_y_row,
+    T,
+    D: tl.constexpr,
+    B: tl.constexpr,
+):
+    i_n = tl.program_id(0).to(tl.int64)
+    offs = i_n * B + tl.arange(0, B)
+    mask = offs < T
+    row = offs // D
+    col = offs % D
+    b_x = tl.load(x + row * stride_x_row + col, mask=mask, other=0.).to(tl.float32)
+    b_y = tl.sigmoid(b_x)
+    tl.store(y + row * stride_y_row + col, b_y.to(y.dtype.element_ty), mask=mask)
+@triton.autotune(
+    configs=[
+        triton.Config({'B': bs}, num_warps=num_warps)
+        for bs in [512, 1024, 2048, 4096, 8192]
+        for num_warps in NUM_WARPS_AUTOTUNE
+    ],
+    key=['D'],
+    **autotune_cache_kwargs,
+)
+@triton.jit(do_not_specialize=['T'])
+def sigmoid_bwd_kernel(
+    x, dy, dx,
+    stride_x_row,
+    stride_dy_row,
+    stride_dx_row,
+    T,
+    D: tl.constexpr,
+    B: tl.constexpr,
+):
+    i_n = tl.program_id(0).to(tl.int64)
+    offs = i_n * B + tl.arange(0, B)
+    mask = offs < T
+    row = offs // D
+    col = offs % D
+    b_x = tl.load(x + row * stride_x_row + col, mask=mask, other=0.).to(tl.float32)
+    b_dy = tl.load(dy + row * stride_dy_row + col, mask=mask, other=0.).to(tl.float32)
+    b_s = tl.sigmoid(b_x)
+    b_dx = b_dy * b_s * (1.0 - b_s)
+    tl.store(dx + row * stride_dx_row + col, b_dx.to(dx.dtype.element_ty), mask=mask)
+@dispatch('modules')
+def sigmoid_fwd(x: torch.Tensor, output_contiguous: bool = False) -> torch.Tensor:
+    x = _ensure_inner_contiguous(x)
+    T, D = x.numel(), x.shape[-1]
+    y = _alloc_output(x, output_contiguous)
+    sigmoid_fwd_kernel[lambda meta: (triton.cdiv(T, meta['B']),)](
+        x=x,
+        y=y,
+        stride_x_row=_get_stride(x),
+        stride_y_row=_get_stride(y),
+        T=T,
+        D=D,
+    )
+    return y
+@dispatch('modules')
+def sigmoid_bwd(x: torch.Tensor, dy: torch.Tensor, output_contiguous: bool = False) -> torch.Tensor:
+    x = _ensure_inner_contiguous(x)
+    dy = _ensure_inner_contiguous(dy)
+    T, D = x.numel(), x.shape[-1]
+    dx = _alloc_output(x, output_contiguous)
+    sigmoid_bwd_kernel[lambda meta: (triton.cdiv(T, meta['B']),)](
+        x=x,
+        dy=dy,
+        dx=dx,
+        stride_x_row=_get_stride(x),
+        stride_dy_row=_get_stride(dy),
+        stride_dx_row=_get_stride(dx),
+        T=T,
+        D=D,
+    )
+    return dx
+class SigmoidFunction(torch.autograd.Function):
+    @staticmethod
+    @input_guard(no_guard_contiguous=True)
+    def forward(ctx, x):
+        ctx.save_for_backward(x)
+        return sigmoid_fwd(x)
+    @staticmethod
+    @input_guard(no_guard_contiguous=True)
+    def backward(ctx, dout):
+        x, = ctx.saved_tensors
+        return sigmoid_bwd(x, dout)
+sigmoid = SigmoidFunction.apply
+@triton.autotune(
+    configs=[
+        triton.Config({'B': bs}, num_warps=num_warps)
+        for bs in [512, 1024, 2048, 4096, 8192]
+        for num_warps in NUM_WARPS_AUTOTUNE
+    ],
+    key=['D'],
+    **autotune_cache_kwargs,
+)
+@triton.jit(do_not_specialize=['T'])
+def logsigmoid_fwd_kernel(
+    x,
+    y,
+    stride_x_row,
+    stride_y_row,
+    temperature,
+    T,
+    D: tl.constexpr,
+    B: tl.constexpr,
+):
+    i_n = tl.program_id(0).to(tl.int64)
+    offs = i_n * B + tl.arange(0, B)
+    mask = offs < T
+    row = offs // D
+    col = offs % D
+    b_x = tl.load(x + row * stride_x_row + col, mask=mask, other=0.).to(tl.float32)
+    b_m = tl.minimum(0., b_x)
+    b_z = 1. + exp(-tl.abs(b_x))
+    b_y = (b_m - log(b_z)) / temperature
+    tl.store(y + row * stride_y_row + col, b_y.to(y.dtype.element_ty), mask=mask)
+@triton.autotune(
+    configs=[
+        triton.Config({'B': bs}, num_warps=num_warps)
+        for bs in [512, 1024, 2048, 4096, 8192]
+        for num_warps in NUM_WARPS_AUTOTUNE
+    ],
+    key=['D'],
+    **autotune_cache_kwargs,
+)
+@triton.jit(do_not_specialize=['T'])
+def logsigmoid_bwd_kernel(
+    x,
+    dy,
+    dx,
+    stride_x_row,
+    stride_dy_row,
+    stride_dx_row,
+    temperature,
+    T,
+    D: tl.constexpr,
+    B: tl.constexpr,
+):
+    i_n = tl.program_id(0).to(tl.int64)
+    offs = i_n * B + tl.arange(0, B)
+    mask = offs < T
+    row = offs // D
+    col = offs % D
+    b_x = tl.load(x + row * stride_x_row + col, mask=mask, other=0.).to(tl.float32)
+    b_dy = tl.load(dy + row * stride_dy_row + col, mask=mask, other=0.).to(tl.float32)
+    b_dx = b_dy * ((1. - tl.sigmoid(b_x)) / temperature)
+    tl.store(dx + row * stride_dx_row + col, b_dx.to(dx.dtype.element_ty), mask=mask)
+@dispatch('modules')
+def logsigmoid_fwd(x: torch.Tensor, temperature: float = 1., output_contiguous: bool = False) -> torch.Tensor:
+    x = _ensure_inner_contiguous(x)
+    T, D = x.numel(), x.shape[-1]
+    y = _alloc_output(x, output_contiguous)
+    logsigmoid_fwd_kernel[lambda meta: (triton.cdiv(T, meta['B']),)](
+        x=x,
+        y=y,
+        stride_x_row=_get_stride(x),
+        stride_y_row=_get_stride(y),
+        temperature=temperature,
+        T=T,
+        D=D,
+    )
+    return y
+@dispatch('modules')
+def logsigmoid_bwd(
+    x: torch.Tensor,
+    dy: torch.Tensor,
+    temperature: float = 1.,
+    output_contiguous: bool = False,
+) -> torch.Tensor:
+    x = _ensure_inner_contiguous(x)
+    dy = _ensure_inner_contiguous(dy)
+    T, D = x.numel(), x.shape[-1]
+    dx = _alloc_output(x, output_contiguous)
+    logsigmoid_bwd_kernel[lambda meta: (triton.cdiv(T, meta['B']),)](
+        x=x,
+        dy=dy,
+        dx=dx,
+        stride_x_row=_get_stride(x),
+        stride_dy_row=_get_stride(dy),
+        stride_dx_row=_get_stride(dx),
+        temperature=temperature,
+        T=T,
+        D=D,
+    )
+    return dx
+class LogSigmoidFunction(torch.autograd.Function):
+    @staticmethod
+    @input_guard(no_guard_contiguous=True)
+    def forward(ctx, x, temperature):
+        ctx.save_for_backward(x)
+        ctx.temperature = temperature
+        return logsigmoid_fwd(x, temperature)
+    @staticmethod
+    @input_guard(no_guard_contiguous=True)
+    def backward(ctx, dy):
+        x, = ctx.saved_tensors
+        return logsigmoid_bwd(x, dy, ctx.temperature), None
+def logsigmoid(x: torch.Tensor, temperature: float = 1.) -> torch.Tensor:
+    return LogSigmoidFunction.apply(x, temperature)
+@triton.autotune(
+    configs=[
+        triton.Config({'B': bs}, num_warps=num_warps)
+        for bs in [512, 1024, 2048, 4096, 8192]
+        for num_warps in NUM_WARPS_AUTOTUNE
+    ],
+    key=['D'],
+    **autotune_cache_kwargs,
+)
+@triton.jit(do_not_specialize=['T'])
+def swish_fwd_kernel(
+    x, y,
+    stride_x_row,
+    stride_y_row,
+    T,
+    D: tl.constexpr,
+    B: tl.constexpr,
+):
+    i_n = tl.program_id(0).to(tl.int64)
+    offs = i_n * B + tl.arange(0, B)
+    mask = offs < T
+    row = offs // D
+    col = offs % D
+    b_x = tl.load(x + row * stride_x_row + col, mask=mask, other=0.).to(tl.float32)
+    b_y = b_x * tl.sigmoid(b_x)
+    tl.store(y + row * stride_y_row + col, b_y.to(y.dtype.element_ty), mask=mask)
+@triton.autotune(
+    configs=[
+        triton.Config({'B': bs}, num_warps=num_warps)
+        for bs in [512, 1024, 2048, 4096, 8192]
+        for num_warps in NUM_WARPS_AUTOTUNE
+    ],
+    key=['D'],
+    **autotune_cache_kwargs,
+)
+@triton.jit(do_not_specialize=['T'])
+def swish_bwd_kernel(
+    x, dy, dx,
+    stride_x_row,
+    stride_dy_row,
+    stride_dx_row,
+    T,
+    D: tl.constexpr,
+    B: tl.constexpr,
+):
+    i_n = tl.program_id(0).to(tl.int64)
+    offs = i_n * B + tl.arange(0, B)
+    mask = offs < T
+    row = offs // D
+    col = offs % D
+    b_x = tl.load(x + row * stride_x_row + col, mask=mask, other=0.).to(tl.float32)
+    b_dy = tl.load(dy + row * stride_dy_row + col, mask=mask, other=0.).to(tl.float32)
+    b_s = tl.sigmoid(b_x)
+    b_dx = b_dy * b_s * (1.0 + b_x * (1.0 - b_s))
+    tl.store(dx + row * stride_dx_row + col, b_dx.to(dx.dtype.element_ty), mask=mask)
+@dispatch('modules')
+def swish_fwd(x: torch.Tensor, output_contiguous: bool = False) -> torch.Tensor:
+    x = _ensure_inner_contiguous(x)
+    T, D = x.numel(), x.shape[-1]
+    y = _alloc_output(x, output_contiguous)
+    swish_fwd_kernel[lambda meta: (triton.cdiv(T, meta['B']),)](
+        x=x,
+        y=y,
+        stride_x_row=_get_stride(x),
+        stride_y_row=_get_stride(y),
+        T=T,
+        D=D,
+    )
+    return y
+@dispatch('modules')
+def swish_bwd(x: torch.Tensor, dy: torch.Tensor, output_contiguous: bool = False) -> torch.Tensor:
+    x = _ensure_inner_contiguous(x)
+    dy = _ensure_inner_contiguous(dy)
+    T, D = x.numel(), x.shape[-1]
+    dx = _alloc_output(x, output_contiguous)
+    swish_bwd_kernel[lambda meta: (triton.cdiv(T, meta['B']),)](
+        x=x,
+        dy=dy,
+        dx=dx,
+        stride_x_row=_get_stride(x),
+        stride_dy_row=_get_stride(dy),
+        stride_dx_row=_get_stride(dx),
+        T=T,
+        D=D,
+    )
+    return dx
+class SwishFunction(torch.autograd.Function):
+    @staticmethod
+    @input_guard(no_guard_contiguous=True)
+    def forward(ctx, x):
+        ctx.save_for_backward(x)
+        return swish_fwd(x)
+    @staticmethod
+    @input_guard(no_guard_contiguous=True)
+    def backward(ctx, dout):
+        x, = ctx.saved_tensors
+        return swish_bwd(x, dout)
+swish = SwishFunction.apply
+# 1/sqrt(2*pi)-> 0.3989423
+# 1/sqrt(2)   -> 0.70710678
+# sqrt(2/pi)  -> 0.79788456
+# this function is tanh approximation of gelu
+# actual gelu is:
+# x * 0.5 * (1.0 + torch.erf(x * 0.70710678))
+@torch.compile
+def bias_gelu(y, bias):
+    x = bias + y
+    return (x * 0.5 * (1.0 + torch.tanh(0.79788456 * x * (1 + 0.044715 * x * x)))).to(dtype=y.dtype)
+# gradient of tanh approximation of gelu
+# gradient of actual gelu is:
+# 0.5 * (1. + torch.erf(x * 0.70710678)) + 0.3989423 * x * torch.exp(-0.5 * x * x)
+@torch.compile
+def bias_gelu_bwd(g, y, bias):
+    """Assume that y has shape (B, D=D) and bias has shape (D)"""
+    x = bias + y
+    tanh_out = torch.tanh(0.79788456 * x * (1 + 0.044715 * x * x))
+    # sqrt(2/pi) * 3 * 0.044715 -> 0.1070322243
+    ff = 0.5 * x * ((1 - tanh_out * tanh_out) * (0.79788456 + 0.1070322243 * x * x)) + 0.5 * (
+        1 + tanh_out
+    )
+    grad_y = ff * g
+    return grad_y.to(dtype=y.dtype), grad_y.sum(dim=(0), dtype=bias.dtype)
+class GeLUFunction(torch.autograd.Function):
+    @staticmethod
+    # bias is an optional argument
+    def forward(ctx, input, bias):
+        ctx.save_for_backward(input, bias)
+        return bias_gelu(input, bias)
+    @staticmethod
+    def backward(ctx, grad_output):
+        input, bias = ctx.saved_tensors
+        return bias_gelu_bwd(grad_output, input, bias)
+bias_gelu_impl = GeLUFunction.apply
+# this function is tanh approximation of gelu
+# actual gelu is:
+# x * 0.5 * (1.0 + torch.erf(x * 0.70710678))
+@dispatch('modules')
+@torch.compile
+def gelu_fwd(x):
+    return (x * 0.5 * (1.0 + torch.tanh(0.79788456 * x * (1 + 0.044715 * x * x)))).to(dtype=x.dtype)
+# gradient of tanh approximation of gelu
+# gradient of actual gelu is:
+# 0.5 * (1. + torch.erf(x * 0.70710678)) + 0.3989423 * x * torch.exp(-0.5 * x * x)
+@dispatch('modules')
+@torch.compile
+def gelu_bwd(g, x):
+    tanh_out = torch.tanh(0.79788456 * x * (1 + 0.044715 * x * x))
+    # sqrt(2/pi) * 3 * 0.044715 -> 0.1070322243
+    ff = 0.5 * x * ((1 - tanh_out * tanh_out) * (0.79788456 + 0.1070322243 * x * x)) + 0.5 * (
+        1 + tanh_out
+    )
+    return (ff * g).to(dtype=x.dtype)
+class FastGeLUFunction(torch.autograd.Function):
+    @staticmethod
+    # bias is an optional argument
+    def forward(ctx, input):
+        ctx.save_for_backward(input)
+        return gelu_fwd(input)
+    @staticmethod
+    def backward(ctx, grad_output):
+        (input,) = ctx.saved_tensors
+        tmp = gelu_bwd(grad_output, input)
+        return tmp
+fast_gelu_impl = FastGeLUFunction.apply
+@torch.compile
+def relu_bwd(g, x):
+    return torch.where(x >= 0, g, 0.0).to(dtype=x.dtype)
+@dispatch('modules')
+@torch.compile
+def sqrelu_fwd(x):
+    r = F.relu(x.float())
+    return (r * r).to(dtype=x.dtype)
+@dispatch('modules')
+@torch.compile
+def sqrelu_bwd(g, x):
+    return (2.0 * g * F.relu(x.float())).to(dtype=x.dtype)
+class SquaredReLUFunction(torch.autograd.Function):
+    @staticmethod
+    def forward(ctx, input):
+        ctx.save_for_backward(input)
+        return sqrelu_fwd(input)
+    @staticmethod
+    def backward(ctx, grad_output):
+        input, = ctx.saved_tensors
+        return sqrelu_bwd(grad_output, input)
+sqrelu = SquaredReLUFunction.apply
+@triton.autotune(
+    configs=[
+        triton.Config({'B': bs}, num_warps=num_warps)
+        for bs in [512, 1024, 2048, 4096, 8192]
+        for num_warps in NUM_WARPS_AUTOTUNE
+    ],
+    key=['D'],
+    **autotune_cache_kwargs,
+)
+@triton.jit(do_not_specialize=['T'])
+def swiglu_fwd_kernel(
+    x, y, z,
+    stride_x_row,
+    stride_y_row,
+    stride_z_row,
+    T,
+    D: tl.constexpr,
+    B: tl.constexpr,
+):
+    i_n = tl.program_id(0).to(tl.int64)
+    offs = i_n * B + tl.arange(0, B)
+    mask = offs < T
+    row = offs // D
+    col = offs % D
+    b_x = tl.load(x + row * stride_x_row + col, mask=mask, other=0.).to(tl.float32)
+    b_y = tl.load(y + row * stride_y_row + col, mask=mask, other=0.).to(tl.float32)
+    b_z = b_x * tl.sigmoid(b_x) * b_y
+    tl.store(z + row * stride_z_row + col, b_z.to(z.dtype.element_ty), mask=mask)
+@triton.heuristics({
+    'HAS_WEIGHT': lambda args: args['z'] is not None,
+})
+@triton.autotune(
+    configs=[
+        triton.Config({'B': bs}, num_warps=num_warps)
+        for bs in [512, 1024, 2048, 4096, 8192]
+        for num_warps in NUM_WARPS_AUTOTUNE
+    ],
+    key=['D'],
+    **autotune_cache_kwargs,
+)
+@triton.jit(do_not_specialize=['T'])
+def swiglu_fwdbwd_kernel(
+    x, y, g, dx, dy, z,
+    stride_x_row,
+    stride_y_row,
+    stride_g_row,
+    stride_dx_row,
+    stride_dy_row,
+    stride_z_row,
+    T,
+    D: tl.constexpr,
+    B: tl.constexpr,
+    HAS_WEIGHT: tl.constexpr,
+):
+    i_n = tl.program_id(0).to(tl.int64)
+    offs = i_n * B + tl.arange(0, B)
+    mask = offs < T
+    row = offs // D
+    col = offs % D
+    b_x = tl.load(x + row * stride_x_row + col, mask=mask, other=0.).to(tl.float32)
+    b_y = tl.load(y + row * stride_y_row + col, mask=mask, other=0.).to(tl.float32)
+    b_g = tl.load(g + row * stride_g_row + col, mask=mask, other=0.).to(tl.float32)
+    b_s = tl.sigmoid(b_x)
+    b_xs = b_x * b_s
+    b_dx = b_g * b_s * (1.0 + b_x * (1.0 - b_s)) * b_y
+    b_dy = b_g * b_xs
+    tl.store(dx + row * stride_dx_row + col, b_dx.to(dx.dtype.element_ty), mask=mask)
+    tl.store(dy + row * stride_dy_row + col, b_dy.to(dy.dtype.element_ty), mask=mask)
+    if HAS_WEIGHT:
+        b_z = b_xs * b_y
+        tl.store(z + row * stride_z_row + col, b_z.to(z.dtype.element_ty), mask=mask)
+@dispatch('modules')
+def swiglu_fwd(x: torch.Tensor, y: torch.Tensor, output_contiguous: bool = False) -> torch.Tensor:
+    assert x.shape == y.shape, f"swiglu_fwd: shape mismatch x={x.shape} y={y.shape}"
+    x = _ensure_inner_contiguous(x)
+    y = _ensure_inner_contiguous(y)
+    T, D = x.numel(), x.shape[-1]
+    z = _alloc_output(x, output_contiguous)
+    swiglu_fwd_kernel[lambda meta: (triton.cdiv(T, meta['B']),)](
+        x=x,
+        y=y,
+        z=z,
+        stride_x_row=_get_stride(x),
+        stride_y_row=_get_stride(y),
+        stride_z_row=_get_stride(z),
+        T=T,
+        D=D,
+    )
+    return z
+@dispatch('modules')
+def swiglu_fwdbwd(
+    x: torch.Tensor,
+    y: torch.Tensor,
+    g: torch.Tensor,
+    use_weight: bool = False,
+    output_contiguous: bool = False,
+):
+    assert x.shape == y.shape == g.shape, f"swiglu_fwdbwd: shape mismatch x={x.shape} y={y.shape} g={g.shape}"
+    x = _ensure_inner_contiguous(x)
+    y = _ensure_inner_contiguous(y)
+    g = _ensure_inner_contiguous(g)
+    T, D = x.numel(), x.shape[-1]
+    dx = _alloc_output(x, output_contiguous)
+    dy = _alloc_output(y, output_contiguous)
+    if use_weight:
+        z = _alloc_output(x, output_contiguous)
+    else:
+        z = None
+    swiglu_fwdbwd_kernel[lambda meta: (triton.cdiv(T, meta['B']),)](
+        x=x,
+        y=y,
+        g=g,
+        dx=dx,
+        dy=dy,
+        z=z,
+        stride_x_row=_get_stride(x),
+        stride_y_row=_get_stride(y),
+        stride_g_row=_get_stride(g),
+        stride_dx_row=_get_stride(dx),
+        stride_dy_row=_get_stride(dy),
+        stride_z_row=_get_stride(z) if z is not None else 0,
+        T=T,
+        D=D,
+    )
+    if use_weight:
+        return dx, dy, z
+    return dx, dy
+class SwiGLUFunction(torch.autograd.Function):
+    r"""
+    Swish-Gated Linear Unit (SwiGLU) function.
+    .. math::
+        \text{SwiGLU}(x, y) = swish(x) * y = \frac{x}{1 + \exp(-x)} * y
+    """
+    @staticmethod
+    @input_guard(no_guard_contiguous=True)
+    def forward(ctx, x, y):
+        ctx.save_for_backward(x, y)
+        return swiglu_fwd(x, y)
+    @staticmethod
+    @input_guard(no_guard_contiguous=True)
+    def backward(ctx, dout):
+        x, y = ctx.saved_tensors
+        return swiglu_fwdbwd(x, y, dout)
+class SwiGLULinearFunction(torch.autograd.Function):
+    r"""
+    Swish-Gated Linear Unit (SwiGLU) function followed by a linear transformation.
+    .. math::
+        \text{SwiGLULinear}(x, y, W, b) = (swish(x) * y) W + b
+    This simple wrap discards the intermediate results of SwiGLU(x, y) to save memory.
+    """
+    @staticmethod
+    @input_guard(no_guard_contiguous=True)
+    @autocast_custom_fwd
+    def forward(ctx, x, y, weight, bias):
+        z = swiglu_fwd(x, y, output_contiguous=True)
+        out = F.linear(z, weight, bias)
+        ctx.save_for_backward(x, y, weight)
+        ctx.linear_bias_is_none = bias is None
+        return out
+    @staticmethod
+    @input_guard(no_guard_contiguous=True)
+    @autocast_custom_bwd
+    def backward(ctx, dout, *args):
+        x, y, weight = ctx.saved_tensors
+        dout = dout.reshape(-1, dout.shape[-1])
+        dz = F.linear(dout, weight.t()).view_as(x)
+        dx, dy, z = swiglu_fwdbwd(x, y, dz, use_weight=True, output_contiguous=True)
+        dlinear_weight = torch.einsum("bo,bi->oi", dout, z.reshape(-1, z.shape[-1]))
+        dlinear_bias = None if ctx.linear_bias_is_none else dout.sum(0)
+        return dx, dy, dlinear_weight, dlinear_bias
+swiglu = SwiGLUFunction.apply
+@dispatch('modules')
+def swiglu_linear(x, y, weight, bias):
+    return SwiGLULinearFunction.apply(x, y, weight, bias)
+@triton.autotune(
+    configs=[
+        triton.Config({'B': bs}, num_warps=num_warps)
+        for bs in [512, 1024, 2048, 4096, 8192]
+        for num_warps in NUM_WARPS_AUTOTUNE
+    ],
+    key=['D'],
+    **autotune_cache_kwargs,
+)
+@triton.jit(do_not_specialize=['T'])
+def sigmoidglu_fwd_kernel(
+    x, y, z,
+    stride_x_row,
+    stride_y_row,
+    stride_z_row,
+    T,
+    D: tl.constexpr,
+    B: tl.constexpr,
+):
+    i_n = tl.program_id(0).to(tl.int64)
+    offs = i_n * B + tl.arange(0, B)
+    mask = offs < T
+    row = offs // D
+    col = offs % D
+    b_x = tl.load(x + row * stride_x_row + col, mask=mask, other=0.).to(tl.float32)
+    b_y = tl.load(y + row * stride_y_row + col, mask=mask, other=0.).to(tl.float32)
+    b_z = tl.sigmoid(b_x) * b_y
+    tl.store(z + row * stride_z_row + col, b_z.to(z.dtype.element_ty), mask=mask)
+@triton.heuristics({
+    'HAS_WEIGHT': lambda args: args['z'] is not None,
+})
+@triton.autotune(
+    configs=[
+        triton.Config({'B': bs}, num_warps=num_warps)
+        for bs in [512, 1024, 2048, 4096, 8192]
+        for num_warps in NUM_WARPS_AUTOTUNE
+    ],
+    key=['D'],
+    **autotune_cache_kwargs,
+)
+@triton.jit(do_not_specialize=['T'])
+def sigmoidglu_fwdbwd_kernel(
+    x, y, g, dx, dy, z,
+    stride_x_row,
+    stride_y_row,
+    stride_g_row,
+    stride_dx_row,
+    stride_dy_row,
+    stride_z_row,
+    T,
+    D: tl.constexpr,
+    B: tl.constexpr,
+    HAS_WEIGHT: tl.constexpr,
+):
+    i_n = tl.program_id(0).to(tl.int64)
+    offs = i_n * B + tl.arange(0, B)
+    mask = offs < T
+    row = offs // D
+    col = offs % D
+    b_x = tl.load(x + row * stride_x_row + col, mask=mask, other=0.).to(tl.float32)
+    b_y = tl.load(y + row * stride_y_row + col, mask=mask, other=0.).to(tl.float32)
+    b_g = tl.load(g + row * stride_g_row + col, mask=mask, other=0.).to(tl.float32)
+    b_s = tl.sigmoid(b_x)
+    b_dx = b_g * b_s * (1.0 - b_s) * b_y
+    b_dy = b_g * b_s
+    tl.store(dx + row * stride_dx_row + col, b_dx.to(dx.dtype.element_ty), mask=mask)
+    tl.store(dy + row * stride_dy_row + col, b_dy.to(dy.dtype.element_ty), mask=mask)
+    if HAS_WEIGHT:
+        b_z = b_s * b_y
+        tl.store(z + row * stride_z_row + col, b_z.to(z.dtype.element_ty), mask=mask)
+@torch.compiler.disable
+def sigmoidglu_fwd(x: torch.Tensor, y: torch.Tensor, output_contiguous: bool = False) -> torch.Tensor:
+    assert x.shape == y.shape, f"sigmoidglu_fwd: shape mismatch x={x.shape} y={y.shape}"
+    x = _ensure_inner_contiguous(x)
+    y = _ensure_inner_contiguous(y)
+    T, D = x.numel(), x.shape[-1]
+    z = _alloc_output(x, output_contiguous)
+    sigmoidglu_fwd_kernel[lambda meta: (triton.cdiv(T, meta['B']),)](
+        x=x,
+        y=y,
+        z=z,
+        stride_x_row=_get_stride(x),
+        stride_y_row=_get_stride(y),
+        stride_z_row=_get_stride(z),
+        T=T,
+        D=D,
+    )
+    return z
+@torch.compiler.disable
+def sigmoidglu_fwdbwd(
+    x: torch.Tensor,
+    y: torch.Tensor,
+    g: torch.Tensor,
+    use_weight: bool = False,
+    output_contiguous: bool = False,
+):
+    assert x.shape == y.shape == g.shape, f"sigmoidglu_fwdbwd: shape mismatch x={x.shape} y={y.shape} g={g.shape}"
+    x = _ensure_inner_contiguous(x)
+    y = _ensure_inner_contiguous(y)
+    g = _ensure_inner_contiguous(g)
+    T, D = x.numel(), x.shape[-1]
+    dx = _alloc_output(x, output_contiguous)
+    dy = _alloc_output(y, output_contiguous)
+    if use_weight:
+        z = _alloc_output(x, output_contiguous)
+    else:
+        z = None
+    sigmoidglu_fwdbwd_kernel[lambda meta: (triton.cdiv(T, meta['B']),)](
+        x=x,
+        y=y,
+        g=g,
+        dx=dx,
+        dy=dy,
+        z=z,
+        stride_x_row=_get_stride(x),
+        stride_y_row=_get_stride(y),
+        stride_g_row=_get_stride(g),
+        stride_dx_row=_get_stride(dx),
+        stride_dy_row=_get_stride(dy),
+        stride_z_row=_get_stride(z) if z is not None else 0,
+        T=T,
+        D=D,
+    )
+    if use_weight:
+        return dx, dy, z
+    return dx, dy
+class SigmoidGLUFunction(torch.autograd.Function):
+    r"""
+    Sigmoid-Gated Linear Unit (SigmoidGLU) function.
+    .. math::
+        \text{SigmoidGLU}(x, y) = sigmoid(x) * y = \frac{1}{1 + \exp(-x)} * y
+    """
+    @staticmethod
+    @input_guard(no_guard_contiguous=True)
+    def forward(ctx, x, y):
+        ctx.save_for_backward(x, y)
+        return sigmoidglu_fwd(x, y)
+    @staticmethod
+    @input_guard(no_guard_contiguous=True)
+    def backward(ctx, dout):
+        x, y = ctx.saved_tensors
+        return sigmoidglu_fwdbwd(x, y, dout)
+class SigmoidGLULinearFunction(torch.autograd.Function):
+    r"""
+    Sigmoid-Gated Linear Unit (SigmoidGLU) function followed by a linear transformation.
+    .. math::
+        \text{SigmoidGLULinear}(x, y, W, b) = (sigmoid(x) * y) W + b
+    This simple wrap discards the intermediate results of SigmoidGLU(x, y) to save memory.
+    """
+    @staticmethod
+    @input_guard(no_guard_contiguous=True)
+    @autocast_custom_fwd
+    def forward(ctx, x, y, weight, bias):
+        z = sigmoidglu_fwd(x, y, output_contiguous=True)
+        out = F.linear(z, weight, bias)
+        ctx.save_for_backward(x, y, weight)
+        ctx.linear_bias_is_none = bias is None
+        return out
+    @staticmethod
+    @input_guard(no_guard_contiguous=True)
+    @autocast_custom_bwd
+    def backward(ctx, dout, *args):
+        x, y, weight = ctx.saved_tensors
+        dout = dout.reshape(-1, dout.shape[-1])
+        dz = F.linear(dout, weight.t()).view_as(x)
+        dx, dy, z = sigmoidglu_fwdbwd(x, y, dz, use_weight=True, output_contiguous=True)
+        dlinear_weight = torch.einsum("bo,bi->oi", dout, z.reshape(-1, z.shape[-1]))
+        dlinear_bias = None if ctx.linear_bias_is_none else dout.sum(0)
+        return dx, dy, dlinear_weight, dlinear_bias
+sigmoidglu = SigmoidGLUFunction.apply
+sigmoidglu_linear = SigmoidGLULinearFunction.apply
+@triton.autotune(
+    configs=[
+        triton.Config({'B': bs}, num_warps=num_warps)
+        for bs in [512, 1024, 2048, 4096, 8192]
+        for num_warps in NUM_WARPS_AUTOTUNE
+    ],
+    key=['D'],
+    **autotune_cache_kwargs,
+)
+@triton.jit(do_not_specialize=['T'])
+def powglu_fwd_kernel(
+    x, y, z,
+    stride_x_row,
+    stride_y_row,
+    stride_z_row,
+    m,
+    T,
+    D: tl.constexpr,
+    B: tl.constexpr,
+):
+    i_n = tl.program_id(0).to(tl.int64)
+    offs = i_n * B + tl.arange(0, B)
+    mask = offs < T
+    row = offs // D
+    col = offs % D
+    b_x = tl.load(x + row * stride_x_row + col, mask=mask, other=0.).to(tl.float32)
+    b_y = tl.load(y + row * stride_y_row + col, mask=mask, other=0.).to(tl.float32)
+    b_s = tl.sigmoid(b_x)
+    b_pos = b_x > 0
+    # feed only positive lanes to log/sqrt; masked lanes give x**p = 1 and are dropped by the where
+    b_xp = tl.where(b_pos, b_x, 1.0)
+    b_sqrt = tl.sqrt(b_xp)
+    b_p = m / (b_sqrt + 1.0)
+    b_pow = exp(b_p * log(b_xp))
+    b_g = tl.where(b_pos, b_pow * b_s, b_x * b_s)
+    b_z = b_g * b_y
+    tl.store(z + row * stride_z_row + col, b_z.to(z.dtype.element_ty), mask=mask)
+@triton.heuristics({
+    'HAS_WEIGHT': lambda args: args['z'] is not None,
+})
+@triton.autotune(
+    configs=[
+        triton.Config({'B': bs}, num_warps=num_warps)
+        for bs in [512, 1024, 2048, 4096, 8192]
+        for num_warps in NUM_WARPS_AUTOTUNE
+    ],
+    key=['D'],
+    **autotune_cache_kwargs,
+)
+@triton.jit(do_not_specialize=['T'])
+def powglu_fwdbwd_kernel(
+    x, y, g, dx, dy, z,
+    stride_x_row,
+    stride_y_row,
+    stride_g_row,
+    stride_dx_row,
+    stride_dy_row,
+    stride_z_row,
+    m,
+    T,
+    D: tl.constexpr,
+    B: tl.constexpr,
+    HAS_WEIGHT: tl.constexpr,
+):
+    i_n = tl.program_id(0).to(tl.int64)
+    offs = i_n * B + tl.arange(0, B)
+    mask = offs < T
+    row = offs // D
+    col = offs % D
+    b_x = tl.load(x + row * stride_x_row + col, mask=mask, other=0.).to(tl.float32)
+    b_y = tl.load(y + row * stride_y_row + col, mask=mask, other=0.).to(tl.float32)
+    b_g = tl.load(g + row * stride_g_row + col, mask=mask, other=0.).to(tl.float32)
+    b_s = tl.sigmoid(b_x)
+    b_pos = b_x > 0
+    b_xp = tl.where(b_pos, b_x, 1.0)
+    b_sqrt = tl.sqrt(b_xp)
+    b_ln = log(b_xp)
+    b_p = m / (b_sqrt + 1.0)
+    b_pow = exp(b_p * b_ln)
+    b_gate_pos = b_pow * b_s
+    # d/dx of the exponent term: p' = -m / (2*sqrt(x)*(sqrt(x)+1)**2)
+    b_pprime = -m / (2.0 * b_sqrt * (b_sqrt + 1.0) * (b_sqrt + 1.0))
+    b_dgate_pos = b_gate_pos * (b_pprime * b_ln + b_p / b_xp + 1.0 - b_s)
+    b_gate_neg = b_x * b_s
+    b_dgate_neg = b_s * (1.0 + b_x * (1.0 - b_s))
+    b_gate = tl.where(b_pos, b_gate_pos, b_gate_neg)
+    b_dgate = tl.where(b_pos, b_dgate_pos, b_dgate_neg)
+    b_dx = b_g * b_y * b_dgate
+    b_dy = b_g * b_gate
+    tl.store(dx + row * stride_dx_row + col, b_dx.to(dx.dtype.element_ty), mask=mask)
+    tl.store(dy + row * stride_dy_row + col, b_dy.to(dy.dtype.element_ty), mask=mask)
+    if HAS_WEIGHT:
+        b_z = b_gate * b_y
+        tl.store(z + row * stride_z_row + col, b_z.to(z.dtype.element_ty), mask=mask)
+@dispatch('modules')
+def powglu_fwd(x: torch.Tensor, y: torch.Tensor, power: float = 3.0, output_contiguous: bool = False) -> torch.Tensor:
+    assert x.shape == y.shape, f"powglu_fwd: shape mismatch x={x.shape} y={y.shape}"
+    x = _ensure_inner_contiguous(x)
+    y = _ensure_inner_contiguous(y)
+    T, D = x.numel(), x.shape[-1]
+    z = _alloc_output(x, output_contiguous)
+    powglu_fwd_kernel[lambda meta: (triton.cdiv(T, meta['B']),)](
+        x=x,
+        y=y,
+        z=z,
+        stride_x_row=_get_stride(x),
+        stride_y_row=_get_stride(y),
+        stride_z_row=_get_stride(z),
+        m=power,
+        T=T,
+        D=D,
+    )
+    return z
+@dispatch('modules')
+def powglu_fwdbwd(
+    x: torch.Tensor,
+    y: torch.Tensor,
+    g: torch.Tensor,
+    power: float = 3.0,
+    use_weight: bool = False,
+    output_contiguous: bool = False,
+):
+    assert x.shape == y.shape == g.shape, f"powglu_fwdbwd: shape mismatch x={x.shape} y={y.shape} g={g.shape}"
+    x = _ensure_inner_contiguous(x)
+    y = _ensure_inner_contiguous(y)
+    g = _ensure_inner_contiguous(g)
+    T, D = x.numel(), x.shape[-1]
+    dx = _alloc_output(x, output_contiguous)
+    dy = _alloc_output(y, output_contiguous)
+    if use_weight:
+        z = _alloc_output(x, output_contiguous)
+    else:
+        z = None
+    powglu_fwdbwd_kernel[lambda meta: (triton.cdiv(T, meta['B']),)](
+        x=x,
+        y=y,
+        g=g,
+        dx=dx,
+        dy=dy,
+        z=z,
+        stride_x_row=_get_stride(x),
+        stride_y_row=_get_stride(y),
+        stride_g_row=_get_stride(g),
+        stride_dx_row=_get_stride(dx),
+        stride_dy_row=_get_stride(dy),
+        stride_z_row=_get_stride(z) if z is not None else 0,
+        m=power,
+        T=T,
+        D=D,
+    )
+    if use_weight:
+        return dx, dy, z
+    return dx, dy
+class PowGLUFunction(torch.autograd.Function):
+    r"""
+    Power-Gated Linear Unit (PowGLU) function.
+    .. math::
+        \text{PowGLU}(x, y) = g(x) * y,\quad
+        g(x) = \begin{cases} x^{power/(\sqrt{x}+1)}\,\sigma(x) & x > 0 \\ x\,\sigma(x) & x \le 0 \end{cases}
+    For ``x <= 0`` the gate reduces to swish, matching SwiGLU; for large ``x > 0`` it saturates instead of
+    growing, replacing SwiGLU's quadratic amplification with bounded growth (Power Linear Unit, arXiv:2605.25704).
+    """
+    @staticmethod
+    @input_guard(no_guard_contiguous=True)
+    def forward(ctx, x, y, power):
+        ctx.save_for_backward(x, y)
+        ctx.power = power
+        return powglu_fwd(x, y, power)
+    @staticmethod
+    @input_guard(no_guard_contiguous=True)
+    def backward(ctx, dout):
+        x, y = ctx.saved_tensors
+        dx, dy = powglu_fwdbwd(x, y, dout, ctx.power)
+        return dx, dy, None
+class PowGLULinearFunction(torch.autograd.Function):
+    r"""
+    Power-Gated Linear Unit (PowGLU) function followed by a linear transformation.
+    .. math::
+        \text{PowGLULinear}(x, y, W, b) = (g(x) * y) W + b
+    This simple wrap discards the intermediate results of PowGLU(x, y) to save memory.
+    """
+    @staticmethod
+    @input_guard(no_guard_contiguous=True)
+    @autocast_custom_fwd
+    def forward(ctx, x, y, weight, bias, power):
+        z = powglu_fwd(x, y, power, output_contiguous=True)
+        out = F.linear(z, weight, bias)
+        ctx.save_for_backward(x, y, weight)
+        ctx.linear_bias_is_none = bias is None
+        ctx.power = power
+        return out
+    @staticmethod
+    @input_guard(no_guard_contiguous=True)
+    @autocast_custom_bwd
+    def backward(ctx, dout, *args):
+        x, y, weight = ctx.saved_tensors
+        dout = dout.reshape(-1, dout.shape[-1])
+        dz = F.linear(dout, weight.t()).view_as(x)
+        dx, dy, z = powglu_fwdbwd(x, y, dz, ctx.power, use_weight=True, output_contiguous=True)
+        dlinear_weight = torch.einsum("bo,bi->oi", dout, z.reshape(-1, z.shape[-1]))
+        dlinear_bias = None if ctx.linear_bias_is_none else dout.sum(0)
+        return dx, dy, dlinear_weight, dlinear_bias, None
+def powglu(x: torch.Tensor, y: torch.Tensor, power: float = 3.0) -> torch.Tensor:
+    return PowGLUFunction.apply(x, y, power)
+@dispatch('modules')
+def powglu_linear(
+    x: torch.Tensor,
+    y: torch.Tensor,
+    weight: torch.Tensor,
+    bias: torch.Tensor,
+    power: float = 3.0,
+) -> torch.Tensor:
+    return PowGLULinearFunction.apply(x, y, weight, bias, power)
+ACT2FN = {
+    'relu': F.relu,
+    'sigmoid': sigmoid,
+    'logsigmoid': logsigmoid,
+    'silu': swish,
+    'swish': swish,
+    'sqrelu': sqrelu,
+    'gelu': fast_gelu_impl,
+    'bias_gelu': bias_gelu_impl,
+}

build/torch-cuda/modules/backends/__init__.py ADDED Viewed

	@@ -0,0 +1,17 @@

+# Copyright (c) 2023-2026, Songlin Yang, Yu Zhang, Zhiyuan Li
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+# For a list of all contributors, visit:
+#   https://github.com/fla-org/flash-linear-attention/graphs/contributors
+"""Module-level backends for FLA components such as rotary and cross-entropy."""
+from ...modules.backends.triton_ascend import TritonAscendBackend
+from ...ops.backends import BackendRegistry, dispatch
+modules_registry = BackendRegistry("modules")
+modules_registry.register(TritonAscendBackend())
+__all__ = ['dispatch', 'modules_registry']

build/torch-cuda/modules/backends/triton_ascend/__init__.py ADDED Viewed

	@@ -0,0 +1,422 @@

+# Copyright (c) 2023-2026, Songlin Yang, Yu Zhang, Zhiyuan Li
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+# For a list of all contributors, visit:
+#   https://github.com/fla-org/flash-linear-attention/graphs/contributors
+"""Triton-Ascend (Huawei NPU) backend for FLA modules."""
+from __future__ import annotations
+from ....ops.backends import BaseBackend
+class TritonAscendBackend(BaseBackend):
+    """Ascend NPU backend using triton-ascend kernels."""
+    backend_type = "triton_ascend"
+    package_name = None
+    env_var = None
+    priority = 0
+    @classmethod
+    def is_available(cls) -> bool:
+        from ....utils import IS_NPU
+        return IS_NPU
+    def rotary_embedding_fwdbwd(
+        self,
+        x,
+        cos,
+        sin,
+        seqlen_offsets=0,
+        cu_seqlens=None,
+        interleaved=False,
+        inplace=False,
+        conjugate=False,
+        chunk_indices=None,
+    ):
+        from ....modules.backends.triton_ascend.rotary import rotary_embedding_fwdbwd_npu
+        return rotary_embedding_fwdbwd_npu(
+            x,
+            cos,
+            sin,
+            seqlen_offsets=seqlen_offsets,
+            cu_seqlens=cu_seqlens,
+            interleaved=interleaved,
+            inplace=inplace,
+            conjugate=conjugate,
+            chunk_indices=chunk_indices,
+        )
+    def cross_entropy_loss(
+        self,
+        logits,
+        target,
+        label_smoothing=0.0,
+        logit_scale=1.0,
+        lse_square_scale=0.0,
+        logit_softcapping=None,
+        ignore_index=-100,
+        inplace_backward=False,
+        process_group=None,
+    ):
+        from ....modules.backends.triton_ascend.fused_cross_entropy import (
+            cross_entropy_loss_npu,
+        )
+        return cross_entropy_loss_npu(
+            logits,
+            target,
+            label_smoothing,
+            logit_scale,
+            lse_square_scale,
+            logit_softcapping,
+            ignore_index,
+            inplace_backward,
+            process_group,
+        )
+    def logsumexp_fwd(
+        self,
+        x,
+        scale=None,
+        softcapping=None,
+        dtype=None,
+    ):
+        from ....modules.backends.triton_ascend.fused_linear_cross_entropy import (
+            logsumexp_fwd_npu,
+        )
+        return logsumexp_fwd_npu(x, scale=scale, softcapping=softcapping, dtype=dtype)
+    def fused_linear_cross_entropy_forward(
+        self,
+        x,
+        target,
+        weight,
+        bias=None,
+        ignore_index=-100,
+        label_smoothing=0.0,
+        logit_scale=1.0,
+        logit_softcapping=None,
+        num_chunks=8,
+        reduction="mean",
+        use_l2warp=False,
+        l2_penalty_factor=1e-4,
+        accumulate_grad_in_fp32=True,
+    ):
+        from ....modules.backends.triton_ascend.fused_linear_cross_entropy import (
+            fused_linear_cross_entropy_forward_npu,
+        )
+        return fused_linear_cross_entropy_forward_npu(
+            x,
+            target,
+            weight,
+            bias,
+            ignore_index,
+            label_smoothing,
+            logit_scale,
+            logit_softcapping,
+            num_chunks,
+            reduction,
+            use_l2warp,
+            l2_penalty_factor,
+            accumulate_grad_in_fp32,
+        )
+    def fused_linear_cross_entropy_backward(
+        self,
+        do,
+        dx,
+        dw,
+        db,
+    ):
+        from ....modules.backends.triton_ascend.fused_linear_cross_entropy import (
+            fused_linear_cross_entropy_backward_npu,
+        )
+        return fused_linear_cross_entropy_backward_npu(do, dx, dw, db)
+    def sigmoid_fwd(self, x, output_contiguous=False):
+        from ....modules.backends.triton_ascend.activations import sigmoid_fwd_npu
+        return sigmoid_fwd_npu(x, output_contiguous=output_contiguous)
+    def sigmoid_bwd(self, x, dy, output_contiguous=False):
+        from ....modules.backends.triton_ascend.activations import sigmoid_bwd_npu
+        return sigmoid_bwd_npu(x, dy, output_contiguous=output_contiguous)
+    def logsigmoid_fwd(self, x, temperature=1., output_contiguous=False):
+        from ....modules.backends.triton_ascend.activations import logsigmoid_fwd_npu
+        return logsigmoid_fwd_npu(x, temperature=temperature, output_contiguous=output_contiguous)
+    def logsigmoid_bwd(self, x, dy, temperature=1., output_contiguous=False):
+        from ....modules.backends.triton_ascend.activations import logsigmoid_bwd_npu
+        return logsigmoid_bwd_npu(x, dy, temperature=temperature, output_contiguous=output_contiguous)
+    def swish_fwd(self, x, output_contiguous=False):
+        from ....modules.backends.triton_ascend.activations import swish_fwd_npu
+        return swish_fwd_npu(x, output_contiguous=output_contiguous)
+    def swish_bwd(self, x, dy, output_contiguous=False):
+        from ....modules.backends.triton_ascend.activations import swish_bwd_npu
+        return swish_bwd_npu(x, dy, output_contiguous=output_contiguous)
+    def swiglu_fwd(self, x, y, output_contiguous=False):
+        from ....modules.backends.triton_ascend.activations import swiglu_fwd_npu
+        return swiglu_fwd_npu(x, y, output_contiguous=output_contiguous)
+    def swiglu_fwdbwd(self, x, y, g, use_weight=False, output_contiguous=False):
+        from ....modules.backends.triton_ascend.activations import swiglu_fwdbwd_npu
+        return swiglu_fwdbwd_npu(x, y, g, use_weight=use_weight, output_contiguous=output_contiguous)
+    def swiglu_linear(self, x, y, weight, bias):
+        from ....modules.backends.triton_ascend.activations import swiglu_linear_npu
+        return swiglu_linear_npu(x, y, weight, bias)
+    def gelu_fwd(self, x):
+        from ....modules.backends.triton_ascend.activations import gelu_fwd_npu
+        return gelu_fwd_npu(x)
+    def gelu_bwd(self, g, x):
+        from ....modules.backends.triton_ascend.activations import gelu_bwd_npu
+        return gelu_bwd_npu(g, x)
+    def sqrelu_fwd(self, x):
+        from ....modules.backends.triton_ascend.activations import sqrelu_fwd_npu
+        return sqrelu_fwd_npu(x)
+    def sqrelu_bwd(self, g, x):
+        from ....modules.backends.triton_ascend.activations import sqrelu_bwd_npu
+        return sqrelu_bwd_npu(g, x)
+    def powglu_fwd(self, x, y, power=3.0, output_contiguous=False):
+        from ....modules.backends.triton_ascend.activations import powglu_fwd_npu
+        return powglu_fwd_npu(x, y, power=power, output_contiguous=output_contiguous)
+    def powglu_fwdbwd(self, x, y, g, power=3.0, use_weight=False, output_contiguous=False):
+        from ....modules.backends.triton_ascend.activations import powglu_fwdbwd_npu
+        return powglu_fwdbwd_npu(
+            x, y, g, power=power, use_weight=use_weight, output_contiguous=output_contiguous,
+        )
+    def powglu_linear(self, x, y, weight, bias, power=3.0):
+        from ....modules.backends.triton_ascend.activations import powglu_linear_npu
+        return powglu_linear_npu(x, y, weight, bias, power)
+    def fused_kl_div_forward(
+        self,
+        x,
+        target_x,
+        weight,
+        target_weight,
+        reduction='batchmean',
+        accumulate_grad_in_fp32=True,
+    ):
+        from ....modules.backends.triton_ascend.fused_kl_div import fused_kl_div_forward_npu
+        return fused_kl_div_forward_npu(
+            x,
+            target_x,
+            weight,
+            target_weight,
+            reduction,
+            accumulate_grad_in_fp32,
+        )
+    def fused_kl_div_backward(self, do, dx, dw):
+        from ....modules.backends.triton_ascend.fused_kl_div import fused_kl_div_backward_npu
+        return fused_kl_div_backward_npu(do, dx, dw)
+    def layer_norm_fwd(
+        self,
+        x,
+        weight,
+        bias,
+        eps=1e-5,
+        residual=None,
+        out_dtype=None,
+        residual_dtype=None,
+        is_rms_norm=False,
+        num_groups=1,
+    ):
+        from ....modules.backends.triton_ascend.layernorm import layer_norm_fwd_npu
+        return layer_norm_fwd_npu(
+            x,
+            weight,
+            bias,
+            eps,
+            residual,
+            out_dtype,
+            residual_dtype,
+            is_rms_norm,
+            num_groups,
+        )
+    def layer_norm_bwd(
+        self,
+        dy,
+        x,
+        weight,
+        bias,
+        mean=None,
+        rstd=None,
+        dres=None,
+        has_residual=False,
+        is_rms_norm=False,
+        x_dtype=None,
+        recompute_output=False,
+        num_groups=1,
+    ):
+        from ....modules.backends.triton_ascend.layernorm import layer_norm_bwd_npu
+        return layer_norm_bwd_npu(
+            dy,
+            x,
+            weight,
+            bias,
+            mean,
+            rstd,
+            dres,
+            has_residual,
+            is_rms_norm,
+            x_dtype,
+            recompute_output,
+            num_groups,
+        )
+    def fused_grpo_loss(
+        self,
+        logits,
+        ref_logp,
+        input_ids,
+        advantages,
+        beta=0.1,
+        completion_mask=None,
+        save_kl=False,
+        inplace=False,
+    ):
+        from ....modules.backends.triton_ascend.grpo import fused_grpo_loss_npu
+        return fused_grpo_loss_npu(
+            logits,
+            ref_logp,
+            input_ids,
+            advantages,
+            beta,
+            completion_mask,
+            save_kl,
+            inplace,
+        )
+    def causal_conv1d_fwd(
+        self,
+        x,
+        weight,
+        bias,
+        residual,
+        initial_state=None,
+        output_final_state=False,
+        activation=None,
+        cu_seqlens=None,
+        cu_seqlens_cpu=None,
+        chunk_indices=None,
+        BT=64,
+        layout_fallback=False,
+    ):
+        from ....modules.backends.triton_ascend.causal_conv1d import causal_conv1d_fwd_npu
+        return causal_conv1d_fwd_npu(
+            x,
+            weight,
+            bias,
+            residual,
+            initial_state,
+            output_final_state,
+            activation,
+            cu_seqlens,
+            cu_seqlens_cpu,
+            chunk_indices,
+            BT,
+            layout_fallback,
+        )
+    def causal_conv1d_bwd(
+        self,
+        x,
+        dy,
+        dht,
+        weight=None,
+        bias=None,
+        residual=None,
+        initial_state=None,
+        activation=None,
+        cu_seqlens=None,
+        cu_seqlens_cpu=None,
+        chunk_indices=None,
+        BT=64,
+        layout_fallback=False,
+    ):
+        from ....modules.backends.triton_ascend.causal_conv1d import causal_conv1d_bwd_npu
+        return causal_conv1d_bwd_npu(
+            x,
+            dy,
+            dht,
+            weight,
+            bias,
+            residual,
+            initial_state,
+            activation,
+            cu_seqlens,
+            cu_seqlens_cpu,
+            chunk_indices,
+            BT,
+            layout_fallback,
+        )
+    def compute_dh0_triton(
+        self,
+        dy,
+        y,
+        weight,
+        initial_state,
+        activation,
+        cu_seqlens,
+    ):
+        from ....modules.backends.triton_ascend.causal_conv1d import compute_dh0_npu
+        return compute_dh0_npu(
+            dy,
+            y,
+            weight,
+            initial_state,
+            activation,
+            cu_seqlens,
+        )
+    def causal_conv1d_update_states(
+        self,
+        x,
+        state_len,
+        initial_state=None,
+        cu_seqlens=None,
+    ):
+        from ....modules.backends.triton_ascend.causal_conv1d import causal_conv1d_update_states_npu
+        return causal_conv1d_update_states_npu(
+            x,
+            state_len,
+            initial_state,
+            cu_seqlens,
+        )
+    def causal_conv1d_update(
+        self,
+        x,
+        cache,
+        residual=None,
+        weight=None,
+        bias=None,
+        activation=None,
+    ):
+        from ....modules.backends.triton_ascend.causal_conv1d import causal_conv1d_update_npu
+        return causal_conv1d_update_npu(
+            x,
+            cache,
+            residual,
+            weight,
+            bias,
+            activation,
+        )

build/torch-cuda/modules/backends/triton_ascend/activations.py ADDED Viewed

	@@ -0,0 +1,931 @@

+# Copyright (c) 2023-2026, Songlin Yang, Yu Zhang, Zhiyuan Li
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+# For a list of all contributors, visit:
+#   https://github.com/fla-org/flash-linear-attention/graphs/contributors
+"""Activation kernels adapted for triton-ascend on Huawei NPU."""
+import torch
+import torch.nn.functional as F
+import triton
+import triton.language as tl
+from ....ops.utils.op import exp, log
+from ....utils import autocast_custom_bwd, autocast_custom_fwd, input_guard
+from ....utils.ascend_ub_manager import ASCEND_MAX_GRID_DIM, compute_activation_block_size
+# Ascend launch limits: grid dim and per-core vector width.
+_MAX_CORE_DIM = 65535
+def _activation_launch_config(
+    T: int,
+    is_backward: bool = False,
+    *,
+    memory_multiplier: float | None = None,
+) -> tuple[tuple[int], int]:
+    """Pick block size under Ascend launch and UB limits."""
+    B = compute_activation_block_size(
+        T,
+        is_backward,
+        max_grid=ASCEND_MAX_GRID_DIM,
+        max_core_dim=_MAX_CORE_DIM,
+        memory_multiplier=memory_multiplier,
+    )
+    return (triton.cdiv(T, B),), B
+@triton.jit
+def _flat_offset(
+    offs,
+    D: tl.constexpr,
+    stride,
+    IS_LINEAR: tl.constexpr,
+):
+    if IS_LINEAR:
+        return offs
+    row = offs // D
+    col = offs % D
+    return row * stride + col
+def _get_stride(x: torch.Tensor) -> int:
+    if x.ndim < 2:
+        return 0
+    return x.stride(-2)
+def _is_linear_stride(stride: int, D: int) -> bool:
+    return stride == D
+_LINEAR_HEURISTICS_XY = {
+    'X_LINEAR': lambda args: _is_linear_stride(args['stride_x_row'], args['D']),
+    'Y_LINEAR': lambda args: _is_linear_stride(args['stride_y_row'], args['D']),
+}
+_LINEAR_HEURISTICS_XYZ = {
+    **_LINEAR_HEURISTICS_XY,
+    'Z_LINEAR': lambda args: _is_linear_stride(args['stride_z_row'], args['D']),
+}
+_LINEAR_HEURISTICS_BWD = {
+    'X_LINEAR': lambda args: _is_linear_stride(args['stride_x_row'], args['D']),
+    'DY_LINEAR': lambda args: _is_linear_stride(args['stride_dy_row'], args['D']),
+    'DX_LINEAR': lambda args: _is_linear_stride(args['stride_dx_row'], args['D']),
+}
+_LINEAR_HEURISTICS_FWDBWD = {
+    **_LINEAR_HEURISTICS_XYZ,
+    'G_LINEAR': lambda args: _is_linear_stride(args['stride_g_row'], args['D']),
+    'DX_LINEAR': lambda args: _is_linear_stride(args['stride_dx_row'], args['D']),
+    'DY_LINEAR': lambda args: _is_linear_stride(args['stride_dy_row'], args['D']),
+}
+def _is_inner_contiguous(x: torch.Tensor) -> bool:
+    ndim = x.ndim
+    if ndim < 2:
+        return True
+    if x.stride(-1) != 1:
+        return False
+    if ndim == 2:
+        return True
+    if ndim == 3:
+        return x.stride(0) == x.stride(-2) * x.shape[-2]
+    if ndim == 4:
+        if x.stride(1) != x.stride(-2) * x.shape[-2]:
+            return False
+        return x.stride(0) == x.stride(1) * x.shape[1]
+    expected = x.stride(-2) * x.shape[-2]
+    for d in range(ndim - 3, -1, -1):
+        if x.stride(d) != expected:
+            return False
+        expected *= x.shape[d]
+    return True
+def _ensure_inner_contiguous(x: torch.Tensor) -> torch.Tensor:
+    if _is_inner_contiguous(x):
+        return x
+    return x.contiguous()
+def _alloc_output(x: torch.Tensor, contiguous: bool = False) -> torch.Tensor:
+    if contiguous:
+        return x.new_empty(x.shape)
+    return torch.empty_like(x)
+@triton.heuristics(_LINEAR_HEURISTICS_XY)
+@triton.jit(do_not_specialize=['T'])
+def sigmoid_fwd_kernel(
+    x, y,
+    T,
+    D: tl.constexpr,
+    stride_x_row,
+    stride_y_row,
+    B: tl.constexpr,
+    X_LINEAR: tl.constexpr,
+    Y_LINEAR: tl.constexpr,
+):
+    pid = tl.program_id(0)
+    offs = pid * B + tl.arange(0, B)
+    mask = offs < T
+    x_off = _flat_offset(offs, D, stride_x_row, X_LINEAR)
+    y_off = _flat_offset(offs, D, stride_y_row, Y_LINEAR)
+    x_val = tl.load(x + x_off, mask=mask, other=0.).to(tl.float32)
+    y_val = tl.sigmoid(x_val)
+    tl.store(y + y_off, y_val.to(y.dtype.element_ty), mask=mask)
+@triton.heuristics(_LINEAR_HEURISTICS_BWD)
+@triton.jit(do_not_specialize=['T'])
+def sigmoid_bwd_kernel(
+    x, dy, dx,
+    T,
+    D: tl.constexpr,
+    stride_x_row,
+    stride_dy_row,
+    stride_dx_row,
+    B: tl.constexpr,
+    X_LINEAR: tl.constexpr,
+    DY_LINEAR: tl.constexpr,
+    DX_LINEAR: tl.constexpr,
+):
+    pid = tl.program_id(0)
+    offs = pid * B + tl.arange(0, B)
+    mask = offs < T
+    x_off = _flat_offset(offs, D, stride_x_row, X_LINEAR)
+    dy_off = _flat_offset(offs, D, stride_dy_row, DY_LINEAR)
+    dx_off = _flat_offset(offs, D, stride_dx_row, DX_LINEAR)
+    x_val = tl.load(x + x_off, mask=mask, other=0.).to(tl.float32)
+    g_val = tl.load(dy + dy_off, mask=mask, other=0.).to(tl.float32)
+    s = tl.sigmoid(x_val)
+    dx_val = g_val * s * (1.0 - s)
+    tl.store(dx + dx_off, dx_val.to(dx.dtype.element_ty), mask=mask)
+@triton.heuristics(_LINEAR_HEURISTICS_XY)
+@triton.jit(do_not_specialize=['T'])
+def logsigmoid_fwd_kernel(
+    x,
+    y,
+    temperature,
+    T,
+    D: tl.constexpr,
+    stride_x_row,
+    stride_y_row,
+    B: tl.constexpr,
+    X_LINEAR: tl.constexpr,
+    Y_LINEAR: tl.constexpr,
+):
+    i = tl.program_id(0)
+    offs = i * B + tl.arange(0, B)
+    mask = offs < T
+    x_off = _flat_offset(offs, D, stride_x_row, X_LINEAR)
+    y_off = _flat_offset(offs, D, stride_y_row, Y_LINEAR)
+    b_x = tl.load(x + x_off, mask=mask, other=0.).to(tl.float32)
+    b_m = tl.minimum(0., b_x)
+    b_z = 1. + exp(-tl.abs(b_x))
+    b_y = (b_m - log(b_z)) / temperature
+    tl.store(y + y_off, b_y.to(y.dtype.element_ty), mask=mask)
+@triton.heuristics(_LINEAR_HEURISTICS_BWD)
+@triton.jit(do_not_specialize=['T'])
+def logsigmoid_bwd_kernel(
+    x,
+    dx,
+    dy,
+    temperature,
+    T,
+    D: tl.constexpr,
+    stride_x_row,
+    stride_dx_row,
+    stride_dy_row,
+    B: tl.constexpr,
+    X_LINEAR: tl.constexpr,
+    DX_LINEAR: tl.constexpr,
+    DY_LINEAR: tl.constexpr,
+):
+    i = tl.program_id(0)
+    offs = i * B + tl.arange(0, B)
+    mask = offs < T
+    x_off = _flat_offset(offs, D, stride_x_row, X_LINEAR)
+    dx_off = _flat_offset(offs, D, stride_dx_row, DX_LINEAR)
+    dy_off = _flat_offset(offs, D, stride_dy_row, DY_LINEAR)
+    b_x = tl.load(x + x_off, mask=mask, other=0.).to(tl.float32)
+    b_dy = tl.load(dy + dy_off, mask=mask, other=0.).to(tl.float32)
+    b_s = tl.sigmoid(b_x)
+    b_dx = b_dy * ((1. - b_s) / temperature)
+    tl.store(dx + dx_off, b_dx.to(dx.dtype.element_ty), mask=mask)
+@triton.heuristics(_LINEAR_HEURISTICS_XY)
+@triton.jit(do_not_specialize=['T'])
+def swish_fwd_kernel(
+    x, y,
+    T,
+    D: tl.constexpr,
+    stride_x_row,
+    stride_y_row,
+    B: tl.constexpr,
+    X_LINEAR: tl.constexpr,
+    Y_LINEAR: tl.constexpr,
+):
+    pid = tl.program_id(0)
+    offs = pid * B + tl.arange(0, B)
+    mask = offs < T
+    x_off = _flat_offset(offs, D, stride_x_row, X_LINEAR)
+    y_off = _flat_offset(offs, D, stride_y_row, Y_LINEAR)
+    x_val = tl.load(x + x_off, mask=mask, other=0.).to(tl.float32)
+    s = tl.sigmoid(x_val)
+    y_val = x_val * s
+    tl.store(y + y_off, y_val.to(y.dtype.element_ty), mask=mask)
+@triton.heuristics(_LINEAR_HEURISTICS_BWD)
+@triton.jit(do_not_specialize=['T'])
+def swish_bwd_kernel(
+    x, dy, dx,
+    T,
+    D: tl.constexpr,
+    stride_x_row,
+    stride_dy_row,
+    stride_dx_row,
+    B: tl.constexpr,
+    X_LINEAR: tl.constexpr,
+    DY_LINEAR: tl.constexpr,
+    DX_LINEAR: tl.constexpr,
+):
+    pid = tl.program_id(0)
+    offs = pid * B + tl.arange(0, B)
+    mask = offs < T
+    x_off = _flat_offset(offs, D, stride_x_row, X_LINEAR)
+    dy_off = _flat_offset(offs, D, stride_dy_row, DY_LINEAR)
+    dx_off = _flat_offset(offs, D, stride_dx_row, DX_LINEAR)
+    x_val = tl.load(x + x_off, mask=mask, other=0.).to(tl.float32)
+    g_val = tl.load(dy + dy_off, mask=mask, other=0.).to(tl.float32)
+    s = tl.sigmoid(x_val)
+    dx_val = g_val * s * (1.0 + x_val * (1.0 - s))
+    tl.store(dx + dx_off, dx_val.to(dx.dtype.element_ty), mask=mask)
+@triton.heuristics(_LINEAR_HEURISTICS_XYZ)
+@triton.jit(do_not_specialize=['T'])
+def swiglu_fwd_kernel(
+    x, y, z,
+    T,
+    D: tl.constexpr,
+    stride_x_row,
+    stride_y_row,
+    stride_z_row,
+    B: tl.constexpr,
+    X_LINEAR: tl.constexpr,
+    Y_LINEAR: tl.constexpr,
+    Z_LINEAR: tl.constexpr,
+):
+    pid = tl.program_id(0)
+    offs = pid * B + tl.arange(0, B)
+    mask = offs < T
+    x_off = _flat_offset(offs, D, stride_x_row, X_LINEAR)
+    y_off = _flat_offset(offs, D, stride_y_row, Y_LINEAR)
+    z_off = _flat_offset(offs, D, stride_z_row, Z_LINEAR)
+    x_val = tl.load(x + x_off, mask=mask, other=0.).to(tl.float32)
+    y_val = tl.load(y + y_off, mask=mask, other=0.).to(tl.float32)
+    s = tl.sigmoid(x_val)
+    z_val = x_val * s * y_val
+    tl.store(z + z_off, z_val.to(z.dtype.element_ty), mask=mask)
+@triton.heuristics({
+    'HAS_WEIGHT': lambda args: args['z'] is not None,
+    **_LINEAR_HEURISTICS_FWDBWD,
+})
+@triton.jit(do_not_specialize=['T'])
+def swiglu_fwdbwd_kernel(
+    x, y, g, dx, dy, z,
+    T,
+    D: tl.constexpr,
+    stride_x_row,
+    stride_y_row,
+    stride_g_row,
+    stride_dx_row,
+    stride_dy_row,
+    stride_z_row,
+    B: tl.constexpr,
+    HAS_WEIGHT: tl.constexpr,
+    X_LINEAR: tl.constexpr,
+    Y_LINEAR: tl.constexpr,
+    G_LINEAR: tl.constexpr,
+    DX_LINEAR: tl.constexpr,
+    DY_LINEAR: tl.constexpr,
+    Z_LINEAR: tl.constexpr,
+):
+    pid = tl.program_id(0)
+    offs = pid * B + tl.arange(0, B)
+    mask = offs < T
+    x_off = _flat_offset(offs, D, stride_x_row, X_LINEAR)
+    y_off = _flat_offset(offs, D, stride_y_row, Y_LINEAR)
+    g_off = _flat_offset(offs, D, stride_g_row, G_LINEAR)
+    dx_off = _flat_offset(offs, D, stride_dx_row, DX_LINEAR)
+    dy_off = _flat_offset(offs, D, stride_dy_row, DY_LINEAR)
+    x_val = tl.load(x + x_off, mask=mask, other=0.).to(tl.float32)
+    y_val = tl.load(y + y_off, mask=mask, other=0.).to(tl.float32)
+    g_val = tl.load(g + g_off, mask=mask, other=0.).to(tl.float32)
+    s = tl.sigmoid(x_val)
+    x_s = x_val * s
+    dx_val = g_val * s * (1.0 + x_val * (1.0 - s)) * y_val
+    dy_val = g_val * x_s
+    tl.store(dx + dx_off, dx_val.to(dx.dtype.element_ty), mask=mask)
+    tl.store(dy + dy_off, dy_val.to(dy.dtype.element_ty), mask=mask)
+    if HAS_WEIGHT:
+        z_off = _flat_offset(offs, D, stride_z_row, Z_LINEAR)
+        z_val = x_s * y_val
+        tl.store(z + z_off, z_val.to(z.dtype.element_ty), mask=mask)
+@triton.heuristics(_LINEAR_HEURISTICS_XY)
+@triton.jit(do_not_specialize=['T'])
+def gelu_fwd_kernel(
+    x, y,
+    T,
+    D: tl.constexpr,
+    stride_x_row,
+    stride_y_row,
+    B: tl.constexpr,
+    X_LINEAR: tl.constexpr,
+    Y_LINEAR: tl.constexpr,
+):
+    pid = tl.program_id(0)
+    offs = pid * B + tl.arange(0, B)
+    mask = offs < T
+    x_off = _flat_offset(offs, D, stride_x_row, X_LINEAR)
+    y_off = _flat_offset(offs, D, stride_y_row, Y_LINEAR)
+    x_val = tl.load(x + x_off, mask=mask, other=0.).to(tl.float32)
+    t = 0.79788456 * x_val * (1.0 + 0.044715 * x_val * x_val)
+    tanh_out = tl.tanh(t)
+    y_val = x_val * 0.5 * (1.0 + tanh_out)
+    tl.store(y + y_off, y_val.to(y.dtype.element_ty), mask=mask)
+@triton.heuristics(_LINEAR_HEURISTICS_BWD)
+@triton.jit(do_not_specialize=['T'])
+def gelu_bwd_kernel(
+    x, dy, dx,
+    T,
+    D: tl.constexpr,
+    stride_x_row,
+    stride_dy_row,
+    stride_dx_row,
+    B: tl.constexpr,
+    X_LINEAR: tl.constexpr,
+    DY_LINEAR: tl.constexpr,
+    DX_LINEAR: tl.constexpr,
+):
+    pid = tl.program_id(0)
+    offs = pid * B + tl.arange(0, B)
+    mask = offs < T
+    x_off = _flat_offset(offs, D, stride_x_row, X_LINEAR)
+    dy_off = _flat_offset(offs, D, stride_dy_row, DY_LINEAR)
+    dx_off = _flat_offset(offs, D, stride_dx_row, DX_LINEAR)
+    x_val = tl.load(x + x_off, mask=mask, other=0.).to(tl.float32)
+    g_val = tl.load(dy + dy_off, mask=mask, other=0.).to(tl.float32)
+    t = 0.79788456 * x_val * (1.0 + 0.044715 * x_val * x_val)
+    tanh_out = tl.tanh(t)
+    ff = 0.5 * x_val * (
+        (1.0 - tanh_out * tanh_out) * (0.79788456 + 0.1070322243 * x_val * x_val)
+    ) + 0.5 * (1.0 + tanh_out)
+    dx_val = ff * g_val
+    tl.store(dx + dx_off, dx_val.to(dx.dtype.element_ty), mask=mask)
+@triton.heuristics(_LINEAR_HEURISTICS_XY)
+@triton.jit(do_not_specialize=['T'])
+def sqrelu_fwd_kernel(
+    x, y,
+    T,
+    D: tl.constexpr,
+    stride_x_row,
+    stride_y_row,
+    B: tl.constexpr,
+    X_LINEAR: tl.constexpr,
+    Y_LINEAR: tl.constexpr,
+):
+    pid = tl.program_id(0)
+    offs = pid * B + tl.arange(0, B)
+    mask = offs < T
+    x_off = _flat_offset(offs, D, stride_x_row, X_LINEAR)
+    y_off = _flat_offset(offs, D, stride_y_row, Y_LINEAR)
+    x_val = tl.load(x + x_off, mask=mask, other=0.).to(tl.float32)
+    r = tl.maximum(x_val, 0.0)
+    y_val = r * r
+    tl.store(y + y_off, y_val.to(y.dtype.element_ty), mask=mask)
+@triton.heuristics(_LINEAR_HEURISTICS_BWD)
+@triton.jit(do_not_specialize=['T'])
+def sqrelu_bwd_kernel(
+    x, dy, dx,
+    T,
+    D: tl.constexpr,
+    stride_x_row,
+    stride_dy_row,
+    stride_dx_row,
+    B: tl.constexpr,
+    X_LINEAR: tl.constexpr,
+    DY_LINEAR: tl.constexpr,
+    DX_LINEAR: tl.constexpr,
+):
+    pid = tl.program_id(0)
+    offs = pid * B + tl.arange(0, B)
+    mask = offs < T
+    x_off = _flat_offset(offs, D, stride_x_row, X_LINEAR)
+    dy_off = _flat_offset(offs, D, stride_dy_row, DY_LINEAR)
+    dx_off = _flat_offset(offs, D, stride_dx_row, DX_LINEAR)
+    x_val = tl.load(x + x_off, mask=mask, other=0.).to(tl.float32)
+    g_val = tl.load(dy + dy_off, mask=mask, other=0.).to(tl.float32)
+    dx_val = 2.0 * g_val * tl.maximum(x_val, 0.0)
+    tl.store(dx + dx_off, dx_val.to(dx.dtype.element_ty), mask=mask)
+@torch.compiler.disable
+def gelu_fwd_npu(x: torch.Tensor) -> torch.Tensor:
+    x = _ensure_inner_contiguous(x)
+    T, D = x.numel(), x.shape[-1]
+    y = _alloc_output(x)
+    grid, B = _activation_launch_config(T)
+    gelu_fwd_kernel[grid](
+        x, y, T=T, D=D,
+        stride_x_row=_get_stride(x),
+        stride_y_row=_get_stride(y),
+        BLOCK_SIZE=B,
+    )
+    return y
+@torch.compiler.disable
+def gelu_bwd_npu(g: torch.Tensor, x: torch.Tensor) -> torch.Tensor:
+    x = _ensure_inner_contiguous(x)
+    g = _ensure_inner_contiguous(g)
+    T, D = x.numel(), x.shape[-1]
+    dx = _alloc_output(x)
+    grid, B = _activation_launch_config(T, is_backward=True)
+    gelu_bwd_kernel[grid](
+        x, g, dx, T=T, D=D,
+        stride_x_row=_get_stride(x),
+        stride_dy_row=_get_stride(g),
+        stride_dx_row=_get_stride(dx),
+        BLOCK_SIZE=B,
+    )
+    return dx
+@torch.compiler.disable
+def sqrelu_fwd_npu(x: torch.Tensor) -> torch.Tensor:
+    x = _ensure_inner_contiguous(x)
+    T, D = x.numel(), x.shape[-1]
+    y = _alloc_output(x)
+    grid, B = _activation_launch_config(T)
+    sqrelu_fwd_kernel[grid](
+        x, y, T=T, D=D,
+        stride_x_row=_get_stride(x),
+        stride_y_row=_get_stride(y),
+        BLOCK_SIZE=B,
+    )
+    return y
+@torch.compiler.disable
+def sqrelu_bwd_npu(g: torch.Tensor, x: torch.Tensor) -> torch.Tensor:
+    x = _ensure_inner_contiguous(x)
+    g = _ensure_inner_contiguous(g)
+    T, D = x.numel(), x.shape[-1]
+    dx = _alloc_output(x)
+    grid, B = _activation_launch_config(T, is_backward=True)
+    sqrelu_bwd_kernel[grid](
+        x, g, dx, T=T, D=D,
+        stride_x_row=_get_stride(x),
+        stride_dy_row=_get_stride(g),
+        stride_dx_row=_get_stride(dx),
+        BLOCK_SIZE=B,
+    )
+    return dx
+@torch.compiler.disable
+def sigmoid_fwd_npu(x: torch.Tensor, output_contiguous: bool = False) -> torch.Tensor:
+    x = _ensure_inner_contiguous(x)
+    T, D = x.numel(), x.shape[-1]
+    y = _alloc_output(x, output_contiguous)
+    grid, B = _activation_launch_config(T)
+    sigmoid_fwd_kernel[grid](
+        x, y, T=T, D=D,
+        stride_x_row=_get_stride(x),
+        stride_y_row=_get_stride(y),
+        B=B,
+    )
+    return y
+@torch.compiler.disable
+def sigmoid_bwd_npu(x: torch.Tensor, dy: torch.Tensor, output_contiguous: bool = False) -> torch.Tensor:
+    x = _ensure_inner_contiguous(x)
+    dy = _ensure_inner_contiguous(dy)
+    T, D = x.numel(), x.shape[-1]
+    dx = _alloc_output(x, output_contiguous)
+    grid, B = _activation_launch_config(T, is_backward=True)
+    sigmoid_bwd_kernel[grid](
+        x, dy, dx, T=T, D=D,
+        stride_x_row=_get_stride(x),
+        stride_dy_row=_get_stride(dy),
+        stride_dx_row=_get_stride(dx),
+        B=B,
+    )
+    return dx
+@torch.compiler.disable
+def logsigmoid_fwd_npu(x: torch.Tensor, temperature: float = 1., output_contiguous: bool = False) -> torch.Tensor:
+    x = _ensure_inner_contiguous(x)
+    T, D = x.numel(), x.shape[-1]
+    y = _alloc_output(x, output_contiguous)
+    grid, B = _activation_launch_config(T)
+    logsigmoid_fwd_kernel[grid](
+        x=x,
+        y=y,
+        temperature=temperature,
+        T=T,
+        D=D,
+        stride_x_row=_get_stride(x),
+        stride_y_row=_get_stride(y),
+        B=B,
+    )
+    return y
+@torch.compiler.disable
+def logsigmoid_bwd_npu(
+    x: torch.Tensor,
+    dy: torch.Tensor,
+    temperature: float = 1.,
+    output_contiguous: bool = False,
+) -> torch.Tensor:
+    x = _ensure_inner_contiguous(x)
+    dy = _ensure_inner_contiguous(dy)
+    T, D = x.numel(), x.shape[-1]
+    dx = _alloc_output(x, output_contiguous)
+    grid, B = _activation_launch_config(T, is_backward=True)
+    logsigmoid_bwd_kernel[grid](
+        x=x,
+        dx=dx,
+        dy=dy,
+        temperature=temperature,
+        T=T,
+        D=D,
+        stride_x_row=_get_stride(x),
+        stride_dx_row=_get_stride(dx),
+        stride_dy_row=_get_stride(dy),
+        B=B,
+    )
+    return dx
+@torch.compiler.disable
+def swish_fwd_npu(x: torch.Tensor, output_contiguous: bool = False) -> torch.Tensor:
+    x = _ensure_inner_contiguous(x)
+    T, D = x.numel(), x.shape[-1]
+    y = _alloc_output(x, output_contiguous)
+    grid, B = _activation_launch_config(T)
+    swish_fwd_kernel[grid](
+        x, y, T=T, D=D,
+        stride_x_row=_get_stride(x),
+        stride_y_row=_get_stride(y),
+        B=B,
+    )
+    return y
+@torch.compiler.disable
+def swish_bwd_npu(x: torch.Tensor, dy: torch.Tensor, output_contiguous: bool = False) -> torch.Tensor:
+    x = _ensure_inner_contiguous(x)
+    dy = _ensure_inner_contiguous(dy)
+    T, D = x.numel(), x.shape[-1]
+    dx = _alloc_output(x, output_contiguous)
+    grid, B = _activation_launch_config(T, is_backward=True)
+    swish_bwd_kernel[grid](
+        x, dy, dx, T=T, D=D,
+        stride_x_row=_get_stride(x),
+        stride_dy_row=_get_stride(dy),
+        stride_dx_row=_get_stride(dx),
+        B=B,
+    )
+    return dx
+@torch.compiler.disable
+def swiglu_fwd_npu(x: torch.Tensor, y: torch.Tensor, output_contiguous: bool = False) -> torch.Tensor:
+    assert x.shape == y.shape, f"swiglu_fwd: shape mismatch x={x.shape} y={y.shape}"
+    x = _ensure_inner_contiguous(x)
+    y = _ensure_inner_contiguous(y)
+    T, D = x.numel(), x.shape[-1]
+    z = _alloc_output(x, output_contiguous)
+    grid, B = _activation_launch_config(T)
+    swiglu_fwd_kernel[grid](
+        x, y, z, T=T, D=D,
+        stride_x_row=_get_stride(x),
+        stride_y_row=_get_stride(y),
+        stride_z_row=_get_stride(z),
+        B=B,
+    )
+    return z
+@torch.compiler.disable
+def swiglu_fwdbwd_npu(
+    x: torch.Tensor,
+    y: torch.Tensor,
+    g: torch.Tensor,
+    use_weight: bool = False,
+    output_contiguous: bool = False,
+):
+    assert x.shape == y.shape == g.shape, f"swiglu_fwdbwd: shape mismatch x={x.shape} y={y.shape} g={g.shape}"
+    x = _ensure_inner_contiguous(x)
+    y = _ensure_inner_contiguous(y)
+    g = _ensure_inner_contiguous(g)
+    T, D = x.numel(), x.shape[-1]
+    dx = _alloc_output(x, output_contiguous)
+    dy = _alloc_output(y, output_contiguous)
+    if use_weight:
+        z = _alloc_output(x, output_contiguous)
+    else:
+        z = None
+    grid, B = _activation_launch_config(T, is_backward=True)
+    swiglu_fwdbwd_kernel[grid](
+        x, y, g, dx, dy, z, T=T, D=D,
+        stride_x_row=_get_stride(x),
+        stride_y_row=_get_stride(y),
+        stride_g_row=_get_stride(g),
+        stride_dx_row=_get_stride(dx),
+        stride_dy_row=_get_stride(dy),
+        stride_z_row=_get_stride(z) if z is not None else 0,
+        B=B,
+    )
+    if use_weight:
+        return dx, dy, z
+    return dx, dy
+class SwiGLULinearFunctionNPU(torch.autograd.Function):
+    @staticmethod
+    @input_guard(no_guard_contiguous=True)
+    @autocast_custom_fwd
+    def forward(ctx, x, y, weight, bias):
+        z = swiglu_fwd_npu(x, y, output_contiguous=True)
+        out = F.linear(z, weight, bias)
+        ctx.save_for_backward(x, y, weight)
+        ctx.linear_bias_is_none = bias is None
+        return out
+    @staticmethod
+    @input_guard(no_guard_contiguous=True)
+    @autocast_custom_bwd
+    def backward(ctx, dout, *args):
+        x, y, weight = ctx.saved_tensors
+        dout = dout.reshape(-1, dout.shape[-1])
+        dz = F.linear(dout, weight.t()).view_as(x)
+        dx, dy, z = swiglu_fwdbwd_npu(x, y, dz, use_weight=True, output_contiguous=True)
+        z_flat = z.reshape(-1, z.shape[-1])
+        dlinear_weight = dout.t() @ z_flat
+        dlinear_bias = None if ctx.linear_bias_is_none else dout.sum(0)
+        return dx, dy, dlinear_weight, dlinear_bias
+def swiglu_linear_npu(x, y, weight, bias):
+    return SwiGLULinearFunctionNPU.apply(x, y, weight, bias)
+@triton.heuristics(_LINEAR_HEURISTICS_XYZ)
+@triton.jit(do_not_specialize=['T'])
+def powglu_fwd_kernel(
+    x, y, z,
+    stride_x_row,
+    stride_y_row,
+    stride_z_row,
+    m,
+    T,
+    D: tl.constexpr,
+    B: tl.constexpr,
+    X_LINEAR: tl.constexpr,
+    Y_LINEAR: tl.constexpr,
+    Z_LINEAR: tl.constexpr,
+):
+    i_n = tl.program_id(0)
+    offs = i_n * B + tl.arange(0, B)
+    mask = offs < T
+    x_off = _flat_offset(offs, D, stride_x_row, X_LINEAR)
+    y_off = _flat_offset(offs, D, stride_y_row, Y_LINEAR)
+    z_off = _flat_offset(offs, D, stride_z_row, Z_LINEAR)
+    b_x = tl.load(x + x_off, mask=mask, other=0.).to(tl.float32)
+    b_y = tl.load(y + y_off, mask=mask, other=0.).to(tl.float32)
+    b_s = tl.sigmoid(b_x)
+    b_pos = b_x > 0
+    # feed only positive lanes to log/sqrt; masked lanes give x**p = 1 and are dropped by the where
+    b_xp = tl.where(b_pos, b_x, 1.0)
+    b_sqrt = tl.sqrt(b_xp)
+    b_p = m / (b_sqrt + 1.0)
+    b_pow = exp(b_p * log(b_xp))
+    b_g = tl.where(b_pos, b_pow * b_s, b_x * b_s)
+    b_z = b_g * b_y
+    tl.store(z + z_off, b_z.to(z.dtype.element_ty), mask=mask)
+@triton.heuristics({
+    'HAS_WEIGHT': lambda args: args['z'] is not None,
+    **_LINEAR_HEURISTICS_FWDBWD,
+})
+@triton.jit(do_not_specialize=['T'])
+def powglu_fwdbwd_kernel(
+    x, y, g, dx, dy, z,
+    stride_x_row,
+    stride_y_row,
+    stride_g_row,
+    stride_dx_row,
+    stride_dy_row,
+    stride_z_row,
+    m,
+    T,
+    D: tl.constexpr,
+    B: tl.constexpr,
+    HAS_WEIGHT: tl.constexpr,
+    X_LINEAR: tl.constexpr,
+    Y_LINEAR: tl.constexpr,
+    G_LINEAR: tl.constexpr,
+    DX_LINEAR: tl.constexpr,
+    DY_LINEAR: tl.constexpr,
+    Z_LINEAR: tl.constexpr,
+):
+    i_n = tl.program_id(0)
+    offs = i_n * B + tl.arange(0, B)
+    mask = offs < T
+    x_off = _flat_offset(offs, D, stride_x_row, X_LINEAR)
+    y_off = _flat_offset(offs, D, stride_y_row, Y_LINEAR)
+    g_off = _flat_offset(offs, D, stride_g_row, G_LINEAR)
+    dx_off = _flat_offset(offs, D, stride_dx_row, DX_LINEAR)
+    dy_off = _flat_offset(offs, D, stride_dy_row, DY_LINEAR)
+    b_x = tl.load(x + x_off, mask=mask, other=0.).to(tl.float32)
+    b_y = tl.load(y + y_off, mask=mask, other=0.).to(tl.float32)
+    b_g = tl.load(g + g_off, mask=mask, other=0.).to(tl.float32)
+    b_s = tl.sigmoid(b_x)
+    b_pos = b_x > 0
+    b_xp = tl.where(b_pos, b_x, 1.0)
+    b_sqrt = tl.sqrt(b_xp)
+    b_ln = log(b_xp)
+    b_p = m / (b_sqrt + 1.0)
+    b_pow = exp(b_p * b_ln)
+    b_gate_pos = b_pow * b_s
+    # d/dx of the exponent term: p' = -m / (2*sqrt(x)*(sqrt(x)+1)**2)
+    b_pprime = -m / (2.0 * b_sqrt * (b_sqrt + 1.0) * (b_sqrt + 1.0))
+    b_dgate_pos = b_gate_pos * (b_pprime * b_ln + b_p / b_xp + 1.0 - b_s)
+    b_gate_neg = b_x * b_s
+    b_dgate_neg = b_s * (1.0 + b_x * (1.0 - b_s))
+    b_gate = tl.where(b_pos, b_gate_pos, b_gate_neg)
+    b_dgate = tl.where(b_pos, b_dgate_pos, b_dgate_neg)
+    b_dx = b_g * b_y * b_dgate
+    b_dy = b_g * b_gate
+    tl.store(dx + dx_off, b_dx.to(dx.dtype.element_ty), mask=mask)
+    tl.store(dy + dy_off, b_dy.to(dy.dtype.element_ty), mask=mask)
+    if HAS_WEIGHT:
+        b_z = b_gate * b_y
+        z_off = _flat_offset(offs, D, stride_z_row, Z_LINEAR)
+        tl.store(z + z_off, b_z.to(z.dtype.element_ty), mask=mask)
+# Peak fp32 temporaries: sigmoid, sqrt, log, exp, pow, gate, output.
+_POWGLU_FWD_MEM_MULT = 8.0
+_POWGLU_BWD_MEM_MULT = 10.0
+@torch.compiler.disable
+def powglu_fwd_npu(x: torch.Tensor, y: torch.Tensor, power: float = 3.0, output_contiguous: bool = False) -> torch.Tensor:
+    assert x.shape == y.shape, f"powglu_fwd: shape mismatch x={x.shape} y={y.shape}"
+    x = _ensure_inner_contiguous(x)
+    y = _ensure_inner_contiguous(y)
+    T, D = x.numel(), x.shape[-1]
+    z = _alloc_output(x, output_contiguous)
+    grid, B = _activation_launch_config(T, memory_multiplier=_POWGLU_FWD_MEM_MULT)
+    powglu_fwd_kernel[grid](
+        x=x,
+        y=y,
+        z=z,
+        stride_x_row=_get_stride(x),
+        stride_y_row=_get_stride(y),
+        stride_z_row=_get_stride(z),
+        m=power,
+        T=T,
+        D=D,
+        B=B,
+    )
+    return z
+@torch.compiler.disable
+def powglu_fwdbwd_npu(
+    x: torch.Tensor,
+    y: torch.Tensor,
+    g: torch.Tensor,
+    power: float = 3.0,
+    use_weight: bool = False,
+    output_contiguous: bool = False,
+):
+    assert x.shape == y.shape == g.shape, f"powglu_fwdbwd: shape mismatch x={x.shape} y={y.shape} g={g.shape}"
+    x = _ensure_inner_contiguous(x)
+    y = _ensure_inner_contiguous(y)
+    g = _ensure_inner_contiguous(g)
+    T, D = x.numel(), x.shape[-1]
+    dx = _alloc_output(x, output_contiguous)
+    dy = _alloc_output(y, output_contiguous)
+    if use_weight:
+        z = _alloc_output(x, output_contiguous)
+    else:
+        z = None
+    grid, B = _activation_launch_config(T, is_backward=True, memory_multiplier=_POWGLU_BWD_MEM_MULT)
+    powglu_fwdbwd_kernel[grid](
+        x=x,
+        y=y,
+        g=g,
+        dx=dx,
+        dy=dy,
+        z=z,
+        stride_x_row=_get_stride(x),
+        stride_y_row=_get_stride(y),
+        stride_g_row=_get_stride(g),
+        stride_dx_row=_get_stride(dx),
+        stride_dy_row=_get_stride(dy),
+        stride_z_row=_get_stride(z) if z is not None else 0,
+        m=power,
+        T=T,
+        D=D,
+        B=B,
+    )
+    if use_weight:
+        return dx, dy, z
+    return dx, dy
+class PowGLULinearFunctionNPU(torch.autograd.Function):
+    r"""
+    Power-Gated Linear Unit (PowGLU) function followed by a linear transformation.
+    .. math::
+        \text{PowGLULinear}(x, y, W, b) = (g(x) * y) W + b
+    This simple wrap discards the intermediate results of PowGLU(x, y) to save memory.
+    """
+    @staticmethod
+    @input_guard(no_guard_contiguous=True)
+    @autocast_custom_fwd
+    def forward(ctx, x, y, weight, bias, power):
+        z = powglu_fwd_npu(x, y, power, output_contiguous=True)
+        out = F.linear(z, weight, bias)
+        ctx.save_for_backward(x, y, weight)
+        ctx.linear_bias_is_none = bias is None
+        ctx.power = power
+        return out
+    @staticmethod
+    @input_guard(no_guard_contiguous=True)
+    @autocast_custom_bwd
+    def backward(ctx, dout, *args):
+        x, y, weight = ctx.saved_tensors
+        dout = dout.reshape(-1, dout.shape[-1])
+        dz = F.linear(dout, weight.t()).view_as(x)
+        dx, dy, z = powglu_fwdbwd_npu(x, y, dz, ctx.power, use_weight=True, output_contiguous=True)
+        z_flat = z.reshape(-1, z.shape[-1])
+        dlinear_weight = dout.t() @ z_flat
+        dlinear_bias = None if ctx.linear_bias_is_none else dout.sum(0)
+        return dx, dy, dlinear_weight, dlinear_bias, None
+def powglu_linear_npu(
+    x: torch.Tensor,
+    y: torch.Tensor,
+    weight: torch.Tensor,
+    bias: torch.Tensor,
+    power: float = 3.0,
+) -> torch.Tensor:
+    return PowGLULinearFunctionNPU.apply(x, y, weight, bias, power)

build/torch-cuda/modules/backends/triton_ascend/causal_conv1d.py ADDED Viewed

	@@ -0,0 +1,1175 @@

+# Copyright (c) 2023-2026, Songlin Yang, Yu Zhang, Zhiyuan Li
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+# For a list of all contributors, visit:
+#   https://github.com/fla-org/flash-linear-attention/graphs/contributors
+"""Causal 1D convolution kernels adapted for triton-ascend on Huawei NPU."""
+import torch
+import triton
+import triton.language as tl
+from einops import rearrange
+from ....ops.utils import prepare_chunk_indices
+from ....utils import input_guard
+STATIC_WARPS = 2
+# Ascend Triton rejects grids whose product exceeds 65535 (see fla/modules/token_shift.py).
+_NPU_MAX_TRITON_GRID = 65535
+_ELEM_BLOCK = 2048
+def _elementwise_launch_iters(numel: int):
+    n_blocks = triton.cdiv(numel, _ELEM_BLOCK)
+    for block_off in range(0, n_blocks, _NPU_MAX_TRITON_GRID):
+        yield min(_NPU_MAX_TRITON_GRID, n_blocks - block_off), block_off * _ELEM_BLOCK
+def _npu_chunk_size(T: int, BT: int) -> int:
+    BT = min(max(BT, 1), 64)
+    if BT not in (1, 2, 4, 8, 16, 32, 64):
+        BT = triton.next_power_of_2(BT)
+    # Ascend compiler requires power-of-2 BT; pad with mask when BT > T.
+    if T not in (1, 2, 4, 8, 16, 32, 64):
+        BT = min(triton.next_power_of_2(T), 64)
+    else:
+        BT = min(BT, T, 64)
+    return BT
+def _clamp_bd_for_grid(B: int, NT: int, D: int, BD: int) -> int:
+    while triton.cdiv(D, BD) * NT * B > _NPU_MAX_TRITON_GRID and BD < 64:
+        BD *= 2
+    return BD
+def _npu_max_axis_chunks(grid_dim0: int, batch: int = 1) -> int:
+    denom = grid_dim0 * batch
+    if denom > _NPU_MAX_TRITON_GRID:
+        raise RuntimeError(
+            f'Ascend Triton grid dim0*batch={denom} exceeds {_NPU_MAX_TRITON_GRID}',
+        )
+    return max(1, _NPU_MAX_TRITON_GRID // max(denom, 1))
+def _npu_tile_config(
+    T: int,
+    BT: int,
+    D: int,
+    dtype: torch.dtype,
+    initial_state: torch.Tensor | None,
+) -> tuple[int, int, int]:
+    BT = _npu_chunk_size(T, BT)
+    BD = 16
+    if D >= 8192:
+        BD = 8
+        BT = min(BT, 8)
+    elif D >= 1024:
+        # BD=4 overflows Ascend UB on large-D forward; cap BT to limit NT.
+        BD = 8
+        BT = min(BT, 32)
+    elif D >= 512:
+        BD = 8
+    if dtype == torch.float16 and initial_state is not None:
+        BD = min(BD, 8)
+    if dtype == torch.bfloat16 and T <= 16:
+        BD = 8
+    return BD, BT, STATIC_WARPS
+def _npu_bwd_tile_config(
+    T: int,
+    BT: int,
+    D: int,
+    dtype: torch.dtype,
+    initial_state: torch.Tensor | None,
+) -> tuple[int, int, int]:
+    BT = _npu_chunk_size(T, BT)
+    BD = 16
+    if initial_state is not None:
+        BD = min(BD, 8)
+        BT = min(BT, 32)
+    if D >= 2048:
+        BD = 8
+        BT = min(BT, 8)
+    elif D >= 1024:
+        BD = 8
+        BT = min(BT, 16)
+    elif D >= 512:
+        BD = 8
+        BT = min(BT, 32)
+    if dtype == torch.bfloat16 and T <= 16:
+        BD = 8
+        BT = 32
+    return BD, BT, STATIC_WARPS
+@triton.heuristics({
+    'HAS_WEIGHT': lambda args: args['weight'] is not None,
+    'HAS_BIAS': lambda args: args['bias'] is not None,
+    'USE_INITIAL_STATE': lambda args: args['initial_state'] is not None,
+    'IS_VARLEN': lambda args: args['cu_seqlens'] is not None,
+})
+@triton.jit
+def causal_conv1d_fwd_kernel(
+    x,
+    y,
+    weight,
+    bias,
+    cu_seqlens,
+    initial_state,
+    chunk_indices,
+    B,
+    T,
+    stride_x_n,
+    stride_x_t,
+    stride_x_d,
+    stride_y_n,
+    stride_y_t,
+    stride_y_d,
+    D: tl.constexpr,
+    W: tl.constexpr,
+    BT: tl.constexpr,
+    BW: tl.constexpr,
+    BD: tl.constexpr,
+    HAS_WEIGHT: tl.constexpr,
+    HAS_BIAS: tl.constexpr,
+    USE_INITIAL_STATE: tl.constexpr,
+    IS_VARLEN: tl.constexpr,
+    CHUNK_OFFSET: tl.constexpr,
+):
+    i_d, i_t, i_b = tl.program_id(0), tl.program_id(1), tl.program_id(2)
+    if IS_VARLEN:
+        i_n, i_t = tl.load(chunk_indices + i_t * 2).to(tl.int32), tl.load(chunk_indices + i_t * 2 + 1).to(tl.int32)
+        bos, eos = tl.load(cu_seqlens + i_n).to(tl.int64), tl.load(cu_seqlens + i_n + 1).to(tl.int64)
+        T = eos - bos
+        p_x = x + bos * stride_x_t
+        p_y = y + bos * stride_y_t
+    else:
+        i_n = i_b
+        i_t = i_t + CHUNK_OFFSET
+        bos = (i_b * T).to(tl.int64)
+        p_x = x + tl.cast(i_b, tl.int64) * stride_x_n
+        p_y = y + tl.cast(i_b, tl.int64) * stride_y_n
+    o_d = i_d * BD + tl.arange(0, BD)
+    o_w = tl.arange(0, BW) + W - BW
+    m_d = o_d < D
+    m_w = o_w >= 0
+    if HAS_WEIGHT:
+        b_w = tl.load(weight + o_d[:, None] * W + o_w, mask=m_d[:, None] & m_w, other=0).to(tl.float32)
+    o_t = i_t * BT + tl.arange(0, BT)
+    m_t = (o_t >= 0) & (o_t < T)
+    b_y = tl.zeros((BT, BD), dtype=tl.float32)
+    for i_w in tl.static_range(-W + 1, 1):
+        o_x = o_t + i_w
+        m_x = ((o_x >= 0) & (o_x < T))[:, None] & m_d[None, :]
+        b_yi = tl.load(
+            p_x + o_x[:, None] * stride_x_t + o_d[None, :] * stride_x_d,
+            mask=m_x,
+            other=0,
+        ).to(tl.float32)
+        if USE_INITIAL_STATE:
+            m_c = ((o_x + W >= 0) & (o_x < 0))[:, None] & m_d[None, :]
+            b_yi += tl.load(
+                initial_state + i_n * D * W + o_d[None, :] * W + (o_x + W)[:, None],
+                mask=m_c,
+                other=0,
+            ).to(tl.float32)
+        if HAS_WEIGHT:
+            b_yi = b_yi * tl.sum(b_w * (o_w == (i_w + W - 1)), 1)[None, :]
+        b_y += b_yi
+    if HAS_BIAS:
+        b_y += tl.load(bias + o_d, mask=m_d).to(tl.float32)[None, :]
+    tl.store(
+        p_y + o_t[:, None] * stride_y_t + o_d[None, :] * stride_y_d,
+        tl.cast(b_y, dtype=y.dtype.element_ty, fp_downcast_rounding='rtne'),
+        mask=m_t[:, None] & m_d[None, :],
+    )
+@triton.jit
+def _silu_kernel(
+    x_ptr,
+    y_ptr,
+    n_elements,
+    ELEM_OFFSET: tl.constexpr,
+    BLOCK: tl.constexpr,
+):
+    pid = tl.program_id(0)
+    offs = pid * BLOCK + tl.arange(0, BLOCK) + ELEM_OFFSET
+    mask = offs < n_elements
+    x = tl.load(x_ptr + offs, mask=mask, other=0.).to(tl.float32)
+    y = x * tl.sigmoid(x)
+    tl.store(y_ptr + offs, y.to(y_ptr.dtype.element_ty), mask=mask)
+@triton.jit
+def _add_kernel(
+    a_ptr,
+    b_ptr,
+    out_ptr,
+    n_elements,
+    ELEM_OFFSET: tl.constexpr,
+    BLOCK: tl.constexpr,
+):
+    pid = tl.program_id(0)
+    offs = pid * BLOCK + tl.arange(0, BLOCK) + ELEM_OFFSET
+    mask = offs < n_elements
+    a = tl.load(a_ptr + offs, mask=mask, other=0.).to(tl.float32)
+    b = tl.load(b_ptr + offs, mask=mask, other=0.).to(tl.float32)
+    tl.store(out_ptr + offs, (a + b).to(out_ptr.dtype.element_ty), mask=mask)
+def _launch_silu(y: torch.Tensor) -> torch.Tensor:
+    y = y.contiguous()
+    out = torch.zeros_like(y)
+    n = y.numel()
+    for grid, elem_off in _elementwise_launch_iters(n):
+        _silu_kernel[(grid,)](
+            y, out, n,
+            ELEM_OFFSET=elem_off,
+            BLOCK=_ELEM_BLOCK,
+            num_warps=STATIC_WARPS,
+        )
+    return out
+def _launch_add(a: torch.Tensor, b: torch.Tensor) -> torch.Tensor:
+    a = a.contiguous()
+    b = b.contiguous()
+    out = torch.zeros_like(a)
+    n = a.numel()
+    for grid, elem_off in _elementwise_launch_iters(n):
+        _add_kernel[(grid,)](
+            a, b, out, n,
+            ELEM_OFFSET=elem_off,
+            BLOCK=_ELEM_BLOCK,
+            num_warps=STATIC_WARPS,
+        )
+    return out
+@triton.jit
+def _silu_bwd_kernel(
+    y_ptr,
+    dy_ptr,
+    out_ptr,
+    stride_y_n,
+    stride_y_t,
+    stride_y_d,
+    stride_dy_n,
+    stride_dy_t,
+    stride_dy_d,
+    stride_out_n,
+    stride_out_t,
+    stride_out_d,
+    B,
+    T,
+    D,
+    ELEM_OFFSET: tl.constexpr,
+    BLOCK: tl.constexpr,
+):
+    pid = tl.program_id(0)
+    offs = pid * BLOCK + tl.arange(0, BLOCK) + ELEM_OFFSET
+    n_elements = B * T * D
+    mask = offs < n_elements
+    rem = offs % D
+    d = rem
+    rem = (offs - d) // D
+    t = rem % T
+    b = rem // T
+    y_off = b * stride_y_n + t * stride_y_t + d * stride_y_d
+    dy_off = b * stride_dy_n + t * stride_dy_t + d * stride_dy_d
+    out_off = b * stride_out_n + t * stride_out_t + d * stride_out_d
+    y = tl.load(y_ptr + y_off, mask=mask, other=0.).to(tl.float32)
+    dy = tl.load(dy_ptr + dy_off, mask=mask, other=0.).to(tl.float32)
+    s = tl.sigmoid(y)
+    out = dy * s * (1.0 + y * (1.0 - s))
+    tl.store(out_ptr + out_off, out.to(out_ptr.dtype.element_ty), mask=mask)
+def _launch_silu_bwd(y_pre: torch.Tensor, dy: torch.Tensor) -> torch.Tensor:
+    out = torch.zeros_like(dy, memory_format=torch.contiguous_format)
+    B, T, D = dy.shape
+    n = B * T * D
+    sy_n, sy_t, sy_d = y_pre.stride()
+    sdy_n, sdy_t, sdy_d = dy.stride()
+    so_n, so_t, so_d = out.stride()
+    for grid, elem_off in _elementwise_launch_iters(n):
+        _silu_bwd_kernel[(grid,)](
+            y_pre, dy, out,
+            sy_n, sy_t, sy_d,
+            sdy_n, sdy_t, sdy_d,
+            so_n, so_t, so_d,
+            B, T, D,
+            ELEM_OFFSET=elem_off,
+            BLOCK=_ELEM_BLOCK,
+            num_warps=STATIC_WARPS,
+        )
+    return out
+def _postprocess_fwd(
+    y: torch.Tensor,
+    residual: torch.Tensor | None,
+    activation: str | None,
+) -> torch.Tensor:
+    if activation in ('swish', 'silu'):
+        y = _launch_silu(y)
+    if residual is not None:
+        if residual.stride() != y.stride():
+            residual = residual.contiguous()
+        y = _launch_add(y, residual)
+    return y
+def _use_seq_bwd(
+    T: int,
+    dtype: torch.dtype,
+    initial_state: torch.Tensor | None,
+    dht: torch.Tensor | None,
+    cu_seqlens: torch.Tensor | None,
+) -> bool:
+    return (
+        cu_seqlens is None
+        and initial_state is None
+        and dht is None
+        and dtype == torch.bfloat16
+        and T <= 16
+    )
+@triton.heuristics({
+    'HAS_WEIGHT': lambda args: args['dw'] is not None,
+    'HAS_BIAS': lambda args: args['db'] is not None,
+})
+@triton.jit
+def causal_conv1d_bwd_seq_kernel(
+    x,
+    weight,
+    dy,
+    dx,
+    dw,
+    db,
+    stride_x_n,
+    stride_x_t,
+    stride_x_d,
+    stride_dx_n,
+    stride_dx_t,
+    stride_dx_d,
+    stride_dy_n,
+    stride_dy_t,
+    stride_dy_d,
+    B,
+    TC: tl.constexpr,
+    D: tl.constexpr,
+    W: tl.constexpr,
+    HAS_WEIGHT: tl.constexpr,
+    HAS_BIAS: tl.constexpr,
+    BLOCK: tl.constexpr,
+):
+    pid = tl.program_id(0)
+    offs = pid * BLOCK + tl.arange(0, BLOCK)
+    n_elements = B * TC * D
+    mask = offs < n_elements
+    d = offs % D
+    tmp = offs // D
+    t = tmp % TC
+    b = tmp // TC
+    b_dx = tl.zeros((BLOCK,), dtype=tl.float32)
+    for i_w in tl.static_range(0, W):
+        t_dy = t + i_w
+        dy_off = b * stride_dy_n + t_dy * stride_dy_t + d * stride_dy_d
+        b_dy = tl.load(dy + dy_off, mask=mask & (t_dy < TC), other=0.).to(tl.float32)
+        if HAS_WEIGHT:
+            w_idx = W - i_w - 1
+            b_w = tl.load(weight + d * W + w_idx, mask=mask, other=0.).to(tl.float32)
+            b_dx += b_dy * b_w
+        else:
+            b_dx += b_dy
+    dx_off = b * stride_dx_n + t * stride_dx_t + d * stride_dx_d
+    tl.store(dx + dx_off, b_dx.to(dx.dtype.element_ty), mask=mask)
+    if HAS_WEIGHT:
+        x_off = b * stride_x_n + t * stride_x_t + d * stride_x_d
+        b_x = tl.load(x + x_off, mask=mask, other=0.).to(tl.float32)
+        i_tg = b * TC + t
+        for i_w in tl.static_range(0, W):
+            t_dy = t + i_w
+            dy_off = b * stride_dy_n + t_dy * stride_dy_t + d * stride_dy_d
+            b_dy = tl.load(dy + dy_off, mask=mask & (t_dy < TC), other=0.).to(tl.float32)
+            w_idx = W - i_w - 1
+            tl.store(
+                dw + (i_tg * D + d) * W + w_idx,
+                (b_dy * b_x).to(dw.dtype.element_ty),
+                mask=mask,
+            )
+    if HAS_BIAS:
+        i_tg = b * TC + t
+        dy_off = b * stride_dy_n + t * stride_dy_t + d * stride_dy_d
+        b_dy0 = tl.load(dy + dy_off, mask=mask, other=0.)
+        tl.store(db + i_tg * D + d, b_dy0.to(db.dtype.element_ty), mask=mask)
+@triton.heuristics({
+    'HAS_WEIGHT': lambda args: args['dw'] is not None,
+    'HAS_BIAS': lambda args: args['db'] is not None,
+    'USE_INITIAL_STATE': lambda args: args['initial_state'] is not None,
+    'USE_FINAL_STATE': lambda args: args['dht'] is not None,
+    'IS_VARLEN': lambda args: args['cu_seqlens'] is not None,
+})
+@triton.jit
+def causal_conv1d_bwd_kernel(
+    x,
+    weight,
+    initial_state,
+    dht,
+    dy,
+    dx,
+    dw,
+    db,
+    cu_seqlens,
+    chunk_indices,
+    B,
+    T,
+    stride_x_n,
+    stride_x_t,
+    stride_x_d,
+    stride_dx_n,
+    stride_dx_t,
+    stride_dx_d,
+    stride_dy_n,
+    stride_dy_t,
+    stride_dy_d,
+    D: tl.constexpr,
+    W: tl.constexpr,
+    BT: tl.constexpr,
+    BW: tl.constexpr,
+    BD: tl.constexpr,
+    HAS_WEIGHT: tl.constexpr,
+    HAS_BIAS: tl.constexpr,
+    USE_INITIAL_STATE: tl.constexpr,
+    USE_FINAL_STATE: tl.constexpr,
+    IS_VARLEN: tl.constexpr,
+    CHUNK_OFFSET: tl.constexpr,
+    NT: tl.constexpr,
+):
+    i_d, i_t, i_b = tl.program_id(0), tl.program_id(1), tl.program_id(2)
+    if IS_VARLEN:
+        i_tg = i_t
+        i_n, i_t = tl.load(chunk_indices + i_t * 2).to(tl.int32), tl.load(chunk_indices + i_t * 2 + 1).to(tl.int32)
+        bos, eos = tl.load(cu_seqlens + i_n).to(tl.int64), tl.load(cu_seqlens + i_n + 1).to(tl.int64)
+        T = eos - bos
+        p_x = x + bos * stride_x_t
+        p_dy = dy + bos * stride_dy_t
+        p_dx = dx + bos * stride_dx_t
+    else:
+        i_t = i_t + CHUNK_OFFSET
+        i_tg = i_b * NT + i_t
+        i_n = i_b
+        p_x = x + tl.cast(i_b, tl.int64) * stride_x_n
+        p_dy = dy + tl.cast(i_b, tl.int64) * stride_dy_n
+        p_dx = dx + tl.cast(i_b, tl.int64) * stride_dx_n
+    o_d = i_d * BD + tl.arange(0, BD)
+    o_w = tl.arange(0, BW) + W - BW
+    m_d = o_d < D
+    m_w = o_w >= 0
+    o_t = i_t * BT + tl.arange(0, BT)
+    m_t = (o_t >= 0) & (o_t < T)
+    b_x = tl.zeros((BT, BD), dtype=tl.float32)
+    if HAS_WEIGHT:
+        b_x = tl.load(
+            p_x + o_t[:, None] * stride_x_t + o_d[None, :] * stride_x_d,
+            mask=m_t[:, None] & m_d[None, :],
+            other=0,
+        ).to(tl.float32)
+        b_w = tl.load(weight + o_d[:, None] * W + o_w, mask=m_d[:, None] & m_w, other=0).to(tl.float32)
+    b_dx = tl.zeros((BT, BD), dtype=tl.float32)
+    if HAS_BIAS:
+        b_db = tl.zeros((BD,), dtype=tl.float32)
+    for i_w in tl.static_range(0, W):
+        o_dy = o_t + i_w
+        m_dy = ((o_dy >= 0) & (o_dy < T))[:, None] & m_d[None, :]
+        b_dy = tl.load(
+            p_dy + o_dy[:, None] * stride_dy_t + o_d[None, :] * stride_dy_d,
+            mask=m_dy,
+            other=0,
+        ).to(tl.float32)
+        if HAS_WEIGHT:
+            b_wdy = b_dy * tl.sum(b_w * (o_w == (W - i_w - 1)), 1)[None, :]
+            b_dw = tl.sum(b_dy * b_x, 0)
+            if USE_INITIAL_STATE:
+                mask_head_rows = (o_t < i_w) & (o_t < T)
+                b_dy_head = tl.load(
+                    p_dy + o_t[:, None] * stride_dy_t + o_d[None, :] * stride_dy_d,
+                    mask=(mask_head_rows[:, None] & m_d[None, :]),
+                    other=0.0,
+                ).to(tl.float32)
+                o_c = W - i_w + o_t
+                mask_c = (mask_head_rows & (o_c >= 1) & (o_c < W))
+                b_xc = tl.load(
+                    initial_state + i_n * D * W + o_d[None, :] * W + o_c[:, None],
+                    mask=(mask_c[:, None] & m_d[None, :]),
+                    other=0.0,
+                ).to(tl.float32)
+                b_dw += tl.sum(b_dy_head * b_xc, 0)
+            tl.store(dw + i_tg * D * W + o_d * W + W - i_w - 1, b_dw.to(dw.dtype.element_ty), mask=m_d)
+        else:
+            b_wdy = b_dy
+        if HAS_BIAS and i_w == 0:
+            b_db += tl.sum(b_dy, 0)
+        b_dx += b_wdy
+    if HAS_BIAS:
+        b_db = tl.cast(b_db, dtype=db.dtype.element_ty, fp_downcast_rounding='rtne')
+        tl.store(db + i_tg * D + o_d, b_db, mask=m_d)
+    if USE_FINAL_STATE:
+        if i_t * BT + BT >= T - W:
+            start_tok = T - (W - 1)
+            offset = i_t * BT + tl.arange(0, BT)
+            tok_idx = offset - start_tok
+            mask = (offset >= start_tok) & (offset < T)
+            w_idx = 1 + tok_idx
+            dht_off = i_n * D * W + o_d[None, :] * W + w_idx[:, None]
+            b_dht = tl.load(dht + dht_off, mask=mask[:, None] & m_d[None, :], other=0.).to(tl.float32)
+            b_dx += b_dht
+    tl.store(
+        p_dx + o_t[:, None] * stride_dx_t + o_d[None, :] * stride_dx_d,
+        tl.cast(b_dx, dtype=dx.dtype.element_ty, fp_downcast_rounding='rtne'),
+        mask=m_t[:, None] & m_d[None, :],
+    )
+@triton.heuristics({
+    'USE_ACTIVATION': lambda args: args['y'] is not None,
+    'IS_VARLEN': lambda args: args['cu_seqlens'] is not None,
+})
+@triton.jit
+def compute_dh0_kernel(
+    dy,
+    y,
+    weight,
+    dh0,
+    cu_seqlens,
+    stride_dy_n,
+    stride_dy_t,
+    stride_dy_d,
+    stride_y_n,
+    stride_y_t,
+    stride_y_d,
+    T,
+    D: tl.constexpr,
+    W: tl.constexpr,
+    BD: tl.constexpr,
+    USE_ACTIVATION: tl.constexpr,
+    IS_VARLEN: tl.constexpr,
+    CHUNK_OFFSET: tl.constexpr,
+):
+    i_d, i_n = tl.program_id(0), tl.program_id(1) + CHUNK_OFFSET
+    if IS_VARLEN:
+        bos = tl.load(cu_seqlens + i_n).to(tl.int64)
+        eos = tl.load(cu_seqlens + i_n + 1).to(tl.int64)
+        seq_len = eos - bos
+        dy_base = dy + bos * stride_dy_t
+    else:
+        seq_len = T
+        dy_base = dy + tl.cast(i_n, tl.int64) * stride_dy_n
+    o_d = i_d * BD + tl.arange(0, BD)
+    m_d = o_d < D
+    for i_w in tl.static_range(1, W):
+        b_dh0 = tl.zeros([BD], dtype=tl.float32)
+        for t in tl.static_range(0, W - 1):
+            if t < i_w:
+                w_idx = i_w - 1 - t
+                p_dy = dy_base + t * stride_dy_t + o_d * stride_dy_d
+                m_t = (t < seq_len) & m_d
+                b_dy = tl.load(p_dy, mask=m_t, other=0).to(tl.float32)
+                if USE_ACTIVATION:
+                    if IS_VARLEN:
+                        p_y = y + bos * stride_y_t + t * stride_y_t + o_d * stride_y_d
+                    else:
+                        p_y = y + tl.cast(i_n, tl.int64) * stride_y_n + t * stride_y_t + o_d * stride_y_d
+                    b_y = tl.load(p_y, mask=m_t, other=0).to(tl.float32)
+                    b_ys = tl.sigmoid(b_y)
+                    b_dy = b_dy * b_ys * (1 + b_y * (1 - b_ys))
+                b_w_col = tl.load(weight + o_d * W + w_idx, mask=m_d, other=0).to(tl.float32)
+                b_dh0 += tl.where(m_t, b_dy * b_w_col, 0)
+        p_dh0 = dh0 + i_n * D * W + o_d * W + i_w
+        tl.store(p_dh0, b_dh0.to(dh0.dtype.element_ty), mask=m_d)
+@triton.heuristics({
+    'USE_INITIAL_STATE': lambda args: args['initial_state'] is not None,
+    'IS_VARLEN': lambda args: args['cu_seqlens'] is not None,
+})
+@triton.jit
+def causal_conv1d_states_fwd_kernel(
+    x,
+    initial_state,
+    final_state,
+    cu_seqlens,
+    T,
+    D,
+    W,
+    stride_x_n,
+    stride_x_t,
+    stride_x_d,
+    BD: tl.constexpr,
+    BW: tl.constexpr,
+    USE_INITIAL_STATE: tl.constexpr,
+    IS_VARLEN: tl.constexpr,
+    CHUNK_OFFSET: tl.constexpr,
+):
+    i_d, i_n = tl.program_id(0), tl.program_id(1) + CHUNK_OFFSET
+    o_d = i_d * BD + tl.arange(0, BD)
+    m_d = o_d < D
+    if IS_VARLEN:
+        bos = tl.load(cu_seqlens + i_n).to(tl.int64)
+        eos = tl.load(cu_seqlens + i_n + 1).to(tl.int64)
+        seq_len = (eos - bos).to(tl.int32)
+        p_x = x + bos * stride_x_t
+    else:
+        seq_len = T
+        p_x = x + tl.cast(i_n, tl.int64) * stride_x_n
+    o_w = W - BW + tl.arange(0, BW)
+    m_w = o_w >= 0
+    o_t = seq_len - BW + tl.arange(0, BW)
+    m_t = (o_t >= 0) & (o_t < seq_len)
+    b_x = tl.load(
+        p_x + o_t[:, None] * stride_x_t + o_d[None, :] * stride_x_d,
+        mask=m_t[:, None] & m_d[None, :],
+        other=0,
+    ).to(tl.float32)
+    if USE_INITIAL_STATE:
+        if seq_len < BW:
+            o_c = W - (BW - seq_len) + tl.arange(0, BW)
+            m_c = (o_c >= 0) & (o_c < W)
+            b_cache = tl.load(
+                initial_state + i_n * D * W + o_d[None, :] * W + o_c[:, None],
+                mask=m_d[None, :] & m_c[:, None],
+                other=0,
+            ).to(tl.float32)
+            b_x += b_cache
+    p_final = final_state + tl.cast(i_n, tl.int64) * D * W + o_d[:, None] * W + o_w[None, :]
+    tl.store(p_final, tl.trans(b_x).to(final_state.dtype.element_ty), mask=m_d[:, None] & m_w[None, :])
+@triton.heuristics({
+    'HAS_WEIGHT': lambda args: args['weight'] is not None,
+    'HAS_BIAS': lambda args: args['bias'] is not None,
+})
+@triton.jit
+def causal_conv1d_update_kernel(
+    x,
+    cache,
+    y,
+    weight,
+    bias,
+    stride_x_n,
+    stride_x_d,
+    stride_y_n,
+    stride_y_d,
+    D: tl.constexpr,
+    W: tl.constexpr,
+    BD: tl.constexpr,
+    HAS_WEIGHT: tl.constexpr,
+    HAS_BIAS: tl.constexpr,
+    CHUNK_OFFSET: tl.constexpr,
+):
+    i_d, i_n = tl.program_id(0), tl.program_id(1) + CHUNK_OFFSET
+    o_d = i_d * BD + tl.arange(0, BD)
+    m_d = o_d < D
+    b_x = tl.load(x + i_n * stride_x_n + o_d * stride_x_d, mask=m_d, other=0).to(tl.float32)
+    b_y = tl.zeros((BD,), dtype=tl.float32)
+    for iw in tl.static_range(0, W):
+        if iw < W - 1:
+            b_c = tl.load(cache + i_n * D * W + o_d * W + (iw + 1), mask=m_d, other=0).to(tl.float32)
+        else:
+            b_c = b_x
+        tl.store(
+            cache + i_n * D * W + o_d * W + iw,
+            tl.cast(b_c, dtype=cache.dtype.element_ty, fp_downcast_rounding='rtne'),
+            mask=m_d,
+        )
+        if HAS_WEIGHT:
+            b_y += b_c * tl.load(weight + o_d * W + iw, mask=m_d, other=0).to(tl.float32)
+        else:
+            b_y += b_c
+    if HAS_BIAS:
+        b_y += tl.load(bias + o_d, mask=m_d)
+    tl.store(
+        y + i_n * stride_y_n + o_d * stride_y_d,
+        tl.cast(b_y, dtype=y.dtype.element_ty, fp_downcast_rounding='rtne'),
+        mask=m_d,
+    )
+def _postprocess_update(
+    y: torch.Tensor,
+    residual: torch.Tensor | None,
+    activation: str | None,
+) -> torch.Tensor:
+    if activation in ('swish', 'silu'):
+        y = _launch_silu(y)
+    if residual is not None:
+        if residual.stride() != y.stride():
+            residual = residual.contiguous()
+        y = _launch_add(y, residual)
+    return y
+def _launch_fwd_core(
+    x: torch.Tensor,
+    weight: torch.Tensor,
+    bias: torch.Tensor,
+    initial_state: torch.Tensor | None,
+    cu_seqlens: torch.LongTensor | None,
+    chunk_indices: torch.LongTensor | None,
+    B: int,
+    T: int,
+    D: int,
+    W: int,
+    BT: int,
+    BD: int | None = None,
+    num_warps: int | None = None,
+) -> torch.Tensor:
+    if BD is None or num_warps is None:
+        BD, BT, num_warps = _npu_tile_config(T, BT, D, x.dtype, initial_state)
+    NT = len(chunk_indices) if cu_seqlens is not None else triton.cdiv(T, BT)
+    BD = _clamp_bd_for_grid(B, NT, D, BD)
+    BW = triton.next_power_of_2(W)
+    stride_x_n, stride_x_t, stride_x_d = x.stride()
+    y = torch.zeros_like(x, memory_format=torch.contiguous_format)
+    stride_y_n, stride_y_t, stride_y_d = y.stride()
+    max_nt = _npu_max_axis_chunks(triton.cdiv(D, BD), B)
+    kernel_kwargs = dict(
+        x=x,
+        y=y,
+        weight=weight,
+        bias=bias,
+        cu_seqlens=cu_seqlens,
+        initial_state=initial_state,
+        B=B,
+        T=T,
+        D=D,
+        W=W,
+        BT=BT,
+        BW=BW,
+        BD=BD,
+        stride_x_n=stride_x_n,
+        stride_x_t=stride_x_t,
+        stride_x_d=stride_x_d,
+        stride_y_n=stride_y_n,
+        stride_y_t=stride_y_t,
+        stride_y_d=stride_y_d,
+        num_warps=num_warps,
+    )
+    for nt_off in range(0, NT, max_nt):
+        nt_len = min(max_nt, NT - nt_off)
+        grid = (triton.cdiv(D, BD), nt_len, B)
+        if cu_seqlens is not None:
+            kernel_kwargs['chunk_indices'] = chunk_indices[nt_off:nt_off + nt_len]
+            kernel_kwargs['CHUNK_OFFSET'] = 0
+        else:
+            kernel_kwargs['chunk_indices'] = chunk_indices
+            kernel_kwargs['CHUNK_OFFSET'] = nt_off
+        causal_conv1d_fwd_kernel[grid](**kernel_kwargs)
+    return y
+@input_guard(no_guard_contiguous=['x'])
+def causal_conv1d_fwd_npu(
+    x: torch.Tensor,
+    weight: torch.Tensor,
+    bias: torch.Tensor,
+    residual: torch.Tensor,
+    initial_state: torch.Tensor | None = None,
+    output_final_state: bool = False,
+    activation: str | None = None,
+    cu_seqlens: torch.LongTensor | None = None,
+    cu_seqlens_cpu: torch.LongTensor | None = None,
+    chunk_indices: torch.LongTensor | None = None,
+    BT: int = 64,
+    layout_fallback: bool = False,
+):
+    del layout_fallback
+    shape = x.shape
+    if x.shape[-1] != weight.shape[0]:
+        x = rearrange(x, 'b t ... -> b t (...)')
+    B, T, D = x.shape[0], x.shape[1], weight.shape[0]
+    W = weight.shape[1]
+    BD, BT, num_warps = _npu_tile_config(T, BT, D, x.dtype, initial_state)
+    if cu_seqlens is not None:
+        chunk_indices = prepare_chunk_indices(cu_seqlens, BT, cu_seqlens_cpu=cu_seqlens_cpu)
+    y = _launch_fwd_core(
+        x, weight, bias, initial_state, cu_seqlens, chunk_indices, B, T, D, W, BT, BD, num_warps,
+    )
+    y = _postprocess_fwd(y, residual, activation)
+    final_state = None
+    if output_final_state:
+        final_state = causal_conv1d_update_states_npu(
+            x=x,
+            state_len=W,
+            initial_state=initial_state,
+            cu_seqlens=cu_seqlens,
+        )
+    return y.view(shape), final_state
+def causal_conv1d_bwd_npu(
+    x: torch.Tensor,
+    dy: torch.Tensor,
+    dht: torch.Tensor,
+    weight: torch.Tensor | None = None,
+    bias: torch.Tensor | None = None,
+    residual: torch.Tensor | None = None,
+    initial_state: torch.Tensor | None = None,
+    activation: str | None = None,
+    cu_seqlens: torch.Tensor | None = None,
+    cu_seqlens_cpu: torch.LongTensor | None = None,
+    chunk_indices: torch.LongTensor | None = None,
+    BT: int = 64,
+    layout_fallback: bool = False,
+):
+    del layout_fallback
+    shape = x.shape
+    if x.shape[-1] != weight.shape[0]:
+        x = rearrange(x, 'b t ... -> b t (...)')
+    B, T, D = x.shape
+    W = weight.shape[1] if weight is not None else None
+    BD, BT, num_warps = _npu_bwd_tile_config(T, BT, D, x.dtype, initial_state)
+    if cu_seqlens is not None:
+        chunk_indices = prepare_chunk_indices(cu_seqlens, BT, cu_seqlens_cpu=cu_seqlens_cpu)
+    NT = len(chunk_indices) if cu_seqlens is not None else triton.cdiv(T, BT)
+    BD = _clamp_bd_for_grid(B, NT, D, BD)
+    BW = triton.next_power_of_2(W)
+    dr = dy if residual is not None else None
+    dy_conv = dy
+    y_pre = None
+    if activation in ('swish', 'silu'):
+        BD_f, BT_f, nw_f = _npu_tile_config(T, BT, D, x.dtype, initial_state)
+        chunk_indices_f = chunk_indices
+        if cu_seqlens is not None:
+            chunk_indices_f = prepare_chunk_indices(cu_seqlens, BT_f, cu_seqlens_cpu=cu_seqlens_cpu)
+        y_pre = _launch_fwd_core(
+            x, weight, bias, initial_state, cu_seqlens, chunk_indices_f,
+            B, T, D, W, BT_f, BD_f, nw_f,
+        )
+        dy_conv = _launch_silu_bwd(y_pre, dy)
+    stride_x_n, stride_x_t, stride_x_d = x.stride()
+    use_seq = _use_seq_bwd(T, x.dtype, initial_state, dht, cu_seqlens)
+    stride_dy_n, stride_dy_t, stride_dy_d = dy_conv.stride()
+    dx = torch.zeros_like(x)
+    stride_dx_n, stride_dx_t, stride_dx_d = dx.stride()
+    if use_seq:
+        block = 1024
+        dw = weight.new_empty(B * T, *weight.shape, dtype=torch.float) if weight is not None else None
+        db = bias.new_empty(B * T, *bias.shape, dtype=torch.float) if bias is not None else None
+        grid = (triton.cdiv(B * T * D, block),)
+        causal_conv1d_bwd_seq_kernel[grid](
+            x=x,
+            weight=weight,
+            dy=dy_conv,
+            dx=dx,
+            dw=dw,
+            db=db,
+            stride_x_n=stride_x_n,
+            stride_x_t=stride_x_t,
+            stride_x_d=stride_x_d,
+            stride_dx_n=stride_dx_n,
+            stride_dx_t=stride_dx_t,
+            stride_dx_d=stride_dx_d,
+            stride_dy_n=stride_dy_n,
+            stride_dy_t=stride_dy_t,
+            stride_dy_d=stride_dy_d,
+            B=B,
+            TC=T,
+            D=D,
+            W=W,
+            BLOCK=block,
+            num_warps=STATIC_WARPS,
+        )
+    else:
+        if not dy_conv.is_contiguous():
+            dy_conv = dy_conv.contiguous()
+        stride_dy_n, stride_dy_t, stride_dy_d = dy_conv.stride()
+        dw = weight.new_empty(B * NT, *weight.shape, dtype=torch.float) if weight is not None else None
+        db = bias.new_empty(B * NT, *bias.shape, dtype=torch.float) if bias is not None else None
+        max_nt = _npu_max_axis_chunks(triton.cdiv(D, BD), B)
+        kernel_kwargs = dict(
+            x=x,
+            weight=weight,
+            initial_state=initial_state,
+            dht=dht,
+            dy=dy_conv,
+            dx=dx,
+            cu_seqlens=cu_seqlens,
+            B=B,
+            T=T,
+            D=D,
+            W=W,
+            BT=BT,
+            BW=BW,
+            BD=BD,
+            stride_x_n=stride_x_n,
+            stride_x_t=stride_x_t,
+            stride_x_d=stride_x_d,
+            stride_dx_n=stride_dx_n,
+            stride_dx_t=stride_dx_t,
+            stride_dx_d=stride_dx_d,
+            stride_dy_n=stride_dy_n,
+            stride_dy_t=stride_dy_t,
+            stride_dy_d=stride_dy_d,
+            num_warps=num_warps,
+            NT=NT,
+        )
+        for nt_off in range(0, NT, max_nt):
+            nt_len = min(max_nt, NT - nt_off)
+            grid = (triton.cdiv(D, BD), nt_len, B)
+            if cu_seqlens is not None:
+                kernel_kwargs['chunk_indices'] = chunk_indices[nt_off:nt_off + nt_len]
+                kernel_kwargs['CHUNK_OFFSET'] = 0
+                kernel_kwargs['dw'] = dw[nt_off:nt_off + nt_len] if weight is not None else None
+                kernel_kwargs['db'] = db[nt_off:nt_off + nt_len] if bias is not None else None
+            else:
+                kernel_kwargs['chunk_indices'] = chunk_indices
+                kernel_kwargs['CHUNK_OFFSET'] = nt_off
+                kernel_kwargs['dw'] = dw
+                kernel_kwargs['db'] = db
+            causal_conv1d_bwd_kernel[grid](**kernel_kwargs)
+    if weight is not None:
+        dw = dw.sum(0).to(weight)
+    if bias is not None:
+        db = db.sum(0).to(bias)
+    dh0 = None
+    if initial_state is not None:
+        dh0 = compute_dh0_npu(
+            dy=dy,
+            y=y_pre,
+            weight=weight,
+            initial_state=initial_state,
+            activation=activation,
+            cu_seqlens=cu_seqlens,
+        )
+    return dx.view(shape), dw, db, dr, dh0
+def compute_dh0_npu(
+    dy: torch.Tensor,
+    y: torch.Tensor | None,
+    weight: torch.Tensor,
+    initial_state: torch.Tensor,
+    activation: str | None,
+    cu_seqlens: torch.Tensor | None,
+) -> torch.Tensor:
+    D, W = weight.shape
+    N = initial_state.shape[0]
+    T = dy.shape[1]
+    BD = 8 if dy.dtype == torch.float16 and activation in ('swish', 'silu') else 16
+    dh0 = torch.zeros_like(initial_state)
+    stride_dy_n = dy.stride(0)
+    stride_dy_t = dy.stride(1)
+    stride_dy_d = dy.stride(2) if dy.dim() == 3 else dy.stride(-1)
+    stride_y_n = stride_y_t = stride_y_d = 0
+    if y is not None:
+        stride_y_n = y.stride(0)
+        stride_y_t = y.stride(1)
+        stride_y_d = y.stride(2) if y.dim() == 3 else y.stride(-1)
+    max_n = _npu_max_axis_chunks(triton.cdiv(D, BD))
+    kernel_kwargs = dict(
+        dy=dy,
+        y=y if activation in ('swish', 'silu') else None,
+        weight=weight,
+        dh0=dh0,
+        cu_seqlens=cu_seqlens,
+        stride_dy_n=stride_dy_n,
+        stride_dy_t=stride_dy_t,
+        stride_dy_d=stride_dy_d,
+        stride_y_n=stride_y_n,
+        stride_y_t=stride_y_t,
+        stride_y_d=stride_y_d,
+        T=T,
+        D=D,
+        W=W,
+        BD=BD,
+        num_warps=STATIC_WARPS,
+    )
+    for n_off in range(0, N, max_n):
+        n_len = min(max_n, N - n_off)
+        kernel_kwargs['CHUNK_OFFSET'] = n_off
+        compute_dh0_kernel[(triton.cdiv(D, BD), n_len)](**kernel_kwargs)
+    return dh0
+@input_guard(no_guard_contiguous=['x'])
+def causal_conv1d_update_states_npu(
+    x: torch.Tensor,
+    state_len: int,
+    initial_state: torch.Tensor | None = None,
+    cu_seqlens: torch.Tensor | None = None,
+    layout_fallback: bool = False,
+) -> torch.Tensor:
+    del layout_fallback
+    if cu_seqlens is not None:
+        N = len(cu_seqlens) - 1
+        if x.dim() == 2:
+            stride_x_n = 0
+            stride_x_t, stride_x_d = x.stride()
+            T = x.shape[0]
+        else:
+            stride_x_n = x.stride(0)
+            stride_x_t, stride_x_d = x.stride(1), x.stride(2)
+            T = x.shape[1]
+        D = x.shape[-1]
+    else:
+        B, T, D = x.shape
+        N = B
+        stride_x_n, stride_x_t, stride_x_d = x.stride()
+    W = state_len
+    final_state = torch.empty(N, D, W, dtype=x.dtype, device=x.device)
+    BD = min(triton.next_power_of_2(D), 16)
+    BW = triton.next_power_of_2(W)
+    grid_dim0 = triton.cdiv(D, BD)
+    max_n = _npu_max_axis_chunks(grid_dim0)
+    kernel_kwargs = dict(
+        x=x,
+        initial_state=initial_state,
+        final_state=final_state,
+        cu_seqlens=cu_seqlens,
+        T=T,
+        D=D,
+        W=W,
+        stride_x_n=stride_x_n,
+        stride_x_t=stride_x_t,
+        stride_x_d=stride_x_d,
+        BW=BW,
+        BD=BD,
+        num_warps=STATIC_WARPS,
+    )
+    for n_off in range(0, N, max_n):
+        n_len = min(max_n, N - n_off)
+        kernel_kwargs['CHUNK_OFFSET'] = n_off
+        causal_conv1d_states_fwd_kernel[(grid_dim0, n_len)](**kernel_kwargs)
+    return final_state
+@input_guard(no_guard_contiguous=['x'])
+def causal_conv1d_update_npu(
+    x: torch.Tensor,
+    cache: torch.Tensor,
+    residual: torch.Tensor | None = None,
+    weight: torch.Tensor | None = None,
+    bias: torch.Tensor | None = None,
+    activation: str | None = None,
+) -> tuple[torch.Tensor, torch.Tensor]:
+    shape = x.shape
+    if weight is not None and x.shape[-1] != weight.shape[0]:
+        x = rearrange(x, 'b t ... -> b t (...)')
+    D = x.shape[-1]
+    N = x.numel() // D
+    W = weight.shape[1] if weight is not None else None
+    BD = min(triton.next_power_of_2(D), 16)
+    if x.dim() == 2:
+        stride_x_n = x.stride(0)
+        stride_x_d = x.stride(1)
+    elif x.dim() == 3 and x.shape[0] == 1:
+        stride_x_n = x.stride(1)
+        stride_x_d = x.stride(2)
+    elif x.dim() == 3:
+        stride_x_n = x.stride(0)
+        stride_x_d = x.stride(2)
+    else:
+        raise ValueError(f"Unsupported input shape: {x.shape}")
+    y = torch.zeros_like(x, memory_format=torch.contiguous_format)
+    if y.dim() == 2:
+        stride_y_n, stride_y_d = y.stride(0), y.stride(1)
+    elif y.dim() == 3 and y.shape[0] == 1:
+        stride_y_n, stride_y_d = y.stride(1), y.stride(2)
+    elif y.dim() == 3:
+        stride_y_n, stride_y_d = y.stride(0), y.stride(2)
+    grid_dim0 = triton.cdiv(D, BD)
+    max_n = _npu_max_axis_chunks(grid_dim0)
+    kernel_kwargs = dict(
+        x=x,
+        cache=cache,
+        y=y,
+        weight=weight,
+        bias=bias,
+        stride_x_n=stride_x_n,
+        stride_x_d=stride_x_d,
+        stride_y_n=stride_y_n,
+        stride_y_d=stride_y_d,
+        D=D,
+        W=W,
+        BD=BD,
+        num_warps=STATIC_WARPS,
+    )
+    for n_off in range(0, N, max_n):
+        n_len = min(max_n, N - n_off)
+        kernel_kwargs['CHUNK_OFFSET'] = n_off
+        causal_conv1d_update_kernel[(grid_dim0, n_len)](**kernel_kwargs)
+    y = _postprocess_update(y, residual, activation)
+    return y.view(shape), cache

build/torch-cuda/modules/backends/triton_ascend/fused_cross_entropy.py ADDED Viewed

	@@ -0,0 +1,469 @@

+# Copyright (c) 2023-2026, Songlin Yang, Yu Zhang, Zhiyuan Li
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+# For a list of all contributors, visit:
+#   https://github.com/fla-org/flash-linear-attention/graphs/contributors
+"""Fused cross-entropy kernels adapted for triton-ascend on Huawei NPU."""
+import torch
+import triton
+import triton.language as tl
+from triton.language.math import tanh
+from ....ops.utils.op import exp, log
+from ....utils import input_guard
+from ....utils.ascend_ub_manager import (
+    ASCEND_MAX_GRID_DIM,
+    compute_vocab_block_size,
+    iter_axis_launch_chunks,
+)
+# Cross-entropy fwd/bwd peak fp32 buffers along vocab dimension.
+_CE_FWD_MEM_MULT = 8.0
+_CE_BWD_MEM_MULT = 12.0
+@triton.heuristics({
+    "HAS_SMOOTHING": lambda args: args["label_smoothing"] > 0.0,
+})
+@triton.jit
+def cross_entropy_fwd_kernel(
+    loss_ptr,
+    lse_ptr,
+    z_loss_ptr,
+    logits_ptr,
+    labels_ptr,
+    label_smoothing,
+    logit_scale,
+    lse_square_scale,
+    logit_softcapping: tl.constexpr,
+    ignore_index,
+    total_classes,
+    class_start_idx,
+    n_cols,
+    n_rows,
+    logits_row_stride,
+    ROW_OFFSET,
+    BLOCK_SIZE: tl.constexpr,
+    HAS_SMOOTHING: tl.constexpr,
+    HAS_SOFTCAPPING: tl.constexpr,
+    SPLIT: tl.constexpr,
+):
+    row_idx = tl.program_id(0)
+    abs_row_idx = row_idx + ROW_OFFSET
+    col_block_idx = tl.program_id(1)
+    logits_ptr = logits_ptr + row_idx * logits_row_stride.to(tl.int64)
+    col_offsets = col_block_idx * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)
+    label_idx = tl.load(labels_ptr + row_idx)
+    logits = tl.load(logits_ptr + col_offsets, mask=col_offsets < n_cols, other=-float("inf"))
+    logits = logits.to(tl.float32) * logit_scale
+    if HAS_SOFTCAPPING:
+        logits = logit_softcapping * tanh(logits / logit_softcapping)
+    max_logits = tl.max(logits, 0)
+    if HAS_SMOOTHING:
+        sum_logits = tl.sum(tl.where(col_offsets < n_cols, logits, 0.0), 0)
+    lse = log(tl.sum(exp(logits - max_logits), 0)) + max_logits
+    tl.store(lse_ptr + col_block_idx * n_rows + abs_row_idx, lse)
+    if label_idx == ignore_index:
+        loss = 0.0
+        z_loss = 0.0
+    else:
+        label_idx -= class_start_idx
+        if label_idx >= col_block_idx * BLOCK_SIZE and label_idx < min(
+            n_cols, (col_block_idx + 1) * BLOCK_SIZE,
+        ):
+            logits_label = tl.load(logits_ptr + label_idx).to(tl.float32) * logit_scale
+            if HAS_SOFTCAPPING:
+                logits_label = logit_softcapping * tanh(logits_label / logit_softcapping)
+            if HAS_SMOOTHING:
+                loss = (
+                    (lse if not SPLIT else 0.0)
+                    - label_smoothing * sum_logits / total_classes
+                    - (1 - label_smoothing) * logits_label
+                )
+            else:
+                loss = (lse if not SPLIT else 0.0) - logits_label
+        else:
+            if HAS_SMOOTHING:
+                loss = label_smoothing * ((lse if not SPLIT else 0.0) - sum_logits / total_classes)
+            else:
+                loss = 0.0
+        if not SPLIT:
+            z_loss = lse_square_scale * lse * lse
+            loss += z_loss
+        else:
+            z_loss = 0.0
+    tl.store(loss_ptr + col_block_idx * n_rows + abs_row_idx, loss)
+    if not SPLIT:
+        tl.store(z_loss_ptr + col_block_idx * n_rows + abs_row_idx, z_loss)
+@triton.heuristics({
+    "HAS_SMOOTHING": lambda args: args["label_smoothing"] > 0.0,
+})
+@triton.jit
+def cross_entropy_bwd_kernel(
+    dlogits_ptr,
+    dloss_ptr,
+    logits_ptr,
+    lse_ptr,
+    labels_ptr,
+    label_smoothing,
+    logit_scale,
+    lse_square_scale,
+    logit_softcapping: tl.constexpr,
+    ignore_index,
+    total_classes,
+    class_start_idx,
+    n_cols,
+    logits_row_stride,
+    dlogits_row_stride,
+    dloss_row_stride,
+    ROW_OFFSET,
+    BLOCK_SIZE: tl.constexpr,
+    HAS_SMOOTHING: tl.constexpr,
+    HAS_SOFTCAPPING: tl.constexpr,
+):
+    row_idx = tl.program_id(0)
+    abs_row_idx = row_idx + ROW_OFFSET
+    col_block_idx = tl.program_id(1)
+    logits_ptr = logits_ptr + row_idx * logits_row_stride.to(tl.int64)
+    dlogits_ptr = dlogits_ptr + row_idx * dlogits_row_stride.to(tl.int64)
+    col_offsets = col_block_idx * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)
+    label_idx = tl.load(labels_ptr + row_idx)
+    if label_idx != ignore_index:
+        dloss = tl.load(dloss_ptr + abs_row_idx * dloss_row_stride)
+    else:
+        dloss = 0.0
+    logits = tl.load(logits_ptr + col_offsets, mask=col_offsets < n_cols, other=-float("inf")).to(
+        tl.float32,
+    ) * logit_scale
+    if HAS_SOFTCAPPING:
+        t = tanh(logits / logit_softcapping)
+        logits = logit_softcapping * t
+    lse = tl.load(lse_ptr + abs_row_idx)
+    probs = exp(logits - lse)
+    probs += 2.0 * lse_square_scale * lse * probs
+    label_idx -= class_start_idx
+    if HAS_SMOOTHING:
+        smooth_negative = label_smoothing / total_classes
+        probs = tl.where(col_offsets == label_idx, probs - (1 - label_smoothing), probs) - smooth_negative
+    else:
+        probs = tl.where(col_offsets == label_idx, probs - 1.0, probs)
+    if HAS_SOFTCAPPING:
+        probs = probs * (1.0 - t * t)
+    tl.store(dlogits_ptr + col_offsets, (dloss * logit_scale) * probs, mask=col_offsets < n_cols)
+def _npu_block_size(n_cols: int, n_rows: int, is_backward: bool = False) -> tuple[int, int]:
+    memory_multiplier = _CE_BWD_MEM_MULT if is_backward else _CE_FWD_MEM_MULT
+    block_size = compute_vocab_block_size(n_cols, n_rows, memory_multiplier)
+    num_warps = 2 if block_size <= 2048 else 4
+    return block_size, num_warps
+def _launch_cross_entropy_fwd(
+    losses,
+    lse,
+    z_losses,
+    logits,
+    target,
+    *,
+    label_smoothing,
+    logit_scale,
+    lse_square_scale,
+    softcap_val,
+    ignore_index,
+    total_classes,
+    class_start_idx,
+    n_cols,
+    n_rows,
+    logits_stride,
+    BLOCK_SIZE,
+    has_softcapping,
+    num_warps,
+    split,
+):
+    n_splits = triton.cdiv(n_cols, BLOCK_SIZE)
+    for row_off, row_len in iter_axis_launch_chunks(n_rows, n_splits, max_grid=ASCEND_MAX_GRID_DIM):
+        cross_entropy_fwd_kernel[(row_len, n_splits)](
+            losses,
+            lse,
+            z_losses,
+            logits[row_off:row_off + row_len],
+            target[row_off:row_off + row_len],
+            label_smoothing,
+            logit_scale,
+            lse_square_scale,
+            softcap_val,
+            ignore_index,
+            total_classes,
+            class_start_idx,
+            n_cols,
+            n_rows,
+            logits_stride,
+            row_off,
+            BLOCK_SIZE=BLOCK_SIZE,
+            HAS_SOFTCAPPING=has_softcapping,
+            num_warps=num_warps,
+            SPLIT=split,
+        )
+def _launch_cross_entropy_bwd(
+    dlogits,
+    grad_losses,
+    logits,
+    lse,
+    target,
+    *,
+    label_smoothing,
+    logit_scale,
+    lse_square_scale,
+    softcap_val,
+    ignore_index,
+    total_classes,
+    class_start_idx,
+    n_cols,
+    n_rows,
+    logits_stride,
+    dlogits_stride,
+    grad_losses_stride,
+    BLOCK_SIZE,
+    has_softcapping,
+    num_warps,
+):
+    n_splits = triton.cdiv(n_cols, BLOCK_SIZE)
+    for row_off, row_len in iter_axis_launch_chunks(n_rows, n_splits, max_grid=ASCEND_MAX_GRID_DIM):
+        cross_entropy_bwd_kernel[(row_len, n_splits)](
+            dlogits[row_off:row_off + row_len],
+            grad_losses,
+            logits[row_off:row_off + row_len],
+            lse,
+            target[row_off:row_off + row_len],
+            label_smoothing,
+            logit_scale,
+            lse_square_scale,
+            softcap_val,
+            ignore_index,
+            total_classes,
+            class_start_idx,
+            n_cols,
+            logits_stride,
+            dlogits_stride,
+            grad_losses_stride,
+            row_off,
+            BLOCK_SIZE=BLOCK_SIZE,
+            HAS_SOFTCAPPING=has_softcapping,
+            num_warps=num_warps,
+        )
+def fused_cross_entropy_forward_npu(
+    logits: torch.Tensor,
+    target: torch.Tensor,
+    label_smoothing: float = 0.0,
+    logit_scale: float = 1.0,
+    lse_square_scale: float = 0.0,
+    logit_softcapping: float = None,
+    ignore_index: int = -100,
+    process_group=None,
+):
+    n_rows, n_cols = logits.shape
+    assert target.shape == (n_rows,)
+    world_size = 1 if process_group is None else torch.distributed.get_world_size(process_group)
+    total_classes = world_size * n_cols
+    rank = 0 if process_group is None else torch.distributed.get_rank(process_group)
+    class_start_idx = rank * n_cols
+    if logits.stride(-1) != 1:
+        logits = logits.contiguous()
+    MAX_BLOCK_SIZE = 64 * 1024
+    BLOCK_SIZE, num_warps = _npu_block_size(n_cols, n_rows)
+    has_softcapping = logit_softcapping is not None
+    softcap_val = float(logit_softcapping) if has_softcapping else 0.0
+    n_splits = (n_cols + BLOCK_SIZE - 1) // BLOCK_SIZE
+    split = world_size > 1 or n_cols > MAX_BLOCK_SIZE or n_splits > 1
+    loss_shape = (n_splits, n_rows) if n_splits > 1 else (n_rows,)
+    losses = torch.empty(*loss_shape, dtype=torch.float, device=logits.device)
+    lse = torch.empty(*loss_shape, dtype=torch.float, device=logits.device)
+    z_losses = torch.empty(*loss_shape, dtype=torch.float, device=logits.device)
+    _launch_cross_entropy_fwd(
+        losses,
+        lse,
+        z_losses,
+        logits,
+        target,
+        label_smoothing=label_smoothing,
+        logit_scale=logit_scale,
+        lse_square_scale=lse_square_scale,
+        softcap_val=softcap_val,
+        ignore_index=ignore_index,
+        total_classes=total_classes,
+        class_start_idx=class_start_idx,
+        n_cols=n_cols,
+        n_rows=n_rows,
+        logits_stride=logits.stride(0),
+        BLOCK_SIZE=BLOCK_SIZE,
+        has_softcapping=has_softcapping,
+        num_warps=num_warps,
+        split=split,
+    )
+    if split:
+        if n_splits > 1:
+            lse = torch.logsumexp(lse, dim=0)
+            losses = losses.sum(dim=0)
+        if world_size > 1:
+            lse_allgather = torch.empty(world_size, n_rows, dtype=lse.dtype, device=lse.device)
+            torch.distributed.all_gather_into_tensor(lse_allgather, lse, group=process_group)
+            handle_losses = torch.distributed.all_reduce(
+                losses, op=torch.distributed.ReduceOp.SUM, group=process_group, async_op=True,
+            )
+            lse = torch.logsumexp(lse_allgather, dim=0)
+            handle_losses.wait()
+        losses += lse
+        if lse_square_scale != 0.0:
+            z_losses = lse_square_scale * lse.square()
+            z_losses.masked_fill_(target == ignore_index, 0.0)
+            losses += z_losses
+        else:
+            z_losses = torch.zeros_like(losses)
+        losses.masked_fill_(target == ignore_index, 0.0)
+    return losses, z_losses, lse, total_classes, class_start_idx
+def fused_cross_entropy_backward_npu(
+    dlogits: torch.Tensor,
+    grad_losses: torch.Tensor,
+    logits: torch.Tensor,
+    lse: torch.Tensor,
+    target: torch.Tensor,
+    label_smoothing: float,
+    logit_scale: float,
+    lse_square_scale: float,
+    logit_softcapping: float | None,
+    ignore_index: int,
+    total_classes: int,
+    class_start_idx: int,
+) -> torch.Tensor:
+    n_rows, n_cols = logits.shape
+    BLOCK_SIZE, num_warps = _npu_block_size(n_cols, n_rows, is_backward=True)
+    has_softcapping = logit_softcapping is not None
+    softcap_val = float(logit_softcapping) if has_softcapping else 0.0
+    _launch_cross_entropy_bwd(
+        dlogits,
+        grad_losses,
+        logits,
+        lse,
+        target,
+        label_smoothing=label_smoothing,
+        logit_scale=logit_scale,
+        lse_square_scale=lse_square_scale,
+        softcap_val=softcap_val,
+        ignore_index=ignore_index,
+        total_classes=total_classes,
+        class_start_idx=class_start_idx,
+        n_cols=n_cols,
+        n_rows=n_rows,
+        logits_stride=logits.stride(0),
+        dlogits_stride=dlogits.stride(0),
+        grad_losses_stride=grad_losses.stride(0),
+        BLOCK_SIZE=BLOCK_SIZE,
+        has_softcapping=has_softcapping,
+        num_warps=num_warps,
+    )
+    return dlogits
+class CrossEntropyLossFunctionNPU(torch.autograd.Function):
+    @staticmethod
+    @input_guard
+    def forward(
+        ctx,
+        logits,
+        target,
+        label_smoothing=0.0,
+        logit_scale=1.0,
+        lse_square_scale=0.0,
+        logit_softcapping=None,
+        ignore_index=-100,
+        inplace_backward=False,
+        process_group=None,
+    ):
+        losses, z_losses, lse, total_classes, class_start_idx = fused_cross_entropy_forward_npu(
+            logits,
+            target,
+            label_smoothing,
+            logit_scale,
+            lse_square_scale,
+            logit_softcapping,
+            ignore_index,
+            process_group,
+        )
+        ctx.save_for_backward(logits, lse, target)
+        ctx.mark_non_differentiable(z_losses)
+        ctx.label_smoothing = label_smoothing
+        ctx.logit_scale = logit_scale
+        ctx.lse_square_scale = lse_square_scale
+        ctx.logit_softcapping = logit_softcapping
+        ctx.ignore_index = ignore_index
+        ctx.total_classes = total_classes
+        ctx.class_start_idx = class_start_idx
+        ctx.inplace_backward = inplace_backward
+        return losses, z_losses
+    @staticmethod
+    @input_guard
+    def backward(ctx, grad_losses, grad_z_losses):
+        del grad_z_losses
+        logits, lse, target = ctx.saved_tensors
+        dlogits = logits if ctx.inplace_backward else torch.empty_like(logits)
+        fused_cross_entropy_backward_npu(
+            dlogits,
+            grad_losses,
+            logits,
+            lse,
+            target,
+            ctx.label_smoothing,
+            ctx.logit_scale,
+            ctx.lse_square_scale,
+            ctx.logit_softcapping,
+            ctx.ignore_index,
+            ctx.total_classes,
+            ctx.class_start_idx,
+        )
+        return dlogits, None, None, None, None, None, None, None, None, None
+def cross_entropy_loss_npu(
+    logits: torch.Tensor,
+    target: torch.Tensor,
+    label_smoothing: float = 0.0,
+    logit_scale: float = 1.0,
+    lse_square_scale: float = 0.0,
+    logit_softcapping: float = None,
+    ignore_index: int = -100,
+    inplace_backward: bool = False,
+    process_group=None,
+) -> tuple[torch.Tensor, torch.Tensor]:
+    return CrossEntropyLossFunctionNPU.apply(
+        logits,
+        target,
+        label_smoothing,
+        logit_scale,
+        lse_square_scale,
+        logit_softcapping,
+        ignore_index,
+        inplace_backward,
+        process_group,
+    )

build/torch-cuda/modules/backends/triton_ascend/fused_kl_div.py ADDED Viewed

	@@ -0,0 +1,188 @@

+# Copyright (c) 2023-2026, Songlin Yang, Yu Zhang, Zhiyuan Li
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+# For a list of all contributors, visit:
+#   https://github.com/fla-org/flash-linear-attention/graphs/contributors
+"""Fused KL divergence kernels adapted for triton-ascend on Huawei NPU."""
+import torch
+import torch.nn.functional as F
+import triton
+import triton.language as tl
+from ....ops.utils.op import exp, log
+from ....utils.ascend_ub_manager import ASCEND_MAX_GRID_DIM, compute_elementwise_block_size, compute_vocab_block_size
+_KLD_FWD_MEM_MULT = 12.0
+_ELEMENTWISE_MEM_MULT = 2.5
+STATIC_WARPS = 2
+@triton.jit
+def kl_div_kernel(
+    logits,
+    target_logits,
+    loss,
+    s_logits,
+    s_loss,
+    reduction: tl.constexpr,
+    N: tl.constexpr,
+    V: tl.constexpr,
+    BV: tl.constexpr,
+):
+    i_n = tl.program_id(0).to(tl.int64)
+    logits += i_n * s_logits
+    target_logits += i_n * s_logits
+    sm = float('-inf')
+    tm = float('-inf')
+    sd, td = 0.0, 0.0
+    NV = tl.cdiv(V, BV)
+    for iv in range(0, NV):
+        o_x = iv * BV + tl.arange(0, BV)
+        b_sl = tl.load(logits + o_x, mask=o_x < V, other=float('-inf'))
+        b_sm = tl.max(b_sl)
+        m_new = tl.maximum(sm, b_sm)
+        sd = sd * exp(sm - m_new) + tl.sum(exp(b_sl - m_new))
+        sm = m_new
+        b_tl = tl.load(target_logits + o_x, mask=o_x < V, other=float('-inf'))
+        b_tm = tl.max(b_tl)
+        m_new = tl.maximum(tm, b_tm)
+        td = td * exp(tm - m_new) + tl.sum(exp(b_tl - m_new))
+        tm = m_new
+    b_loss = 0.
+    for iv in range(0, NV):
+        o_x = iv * BV + tl.arange(0, BV)
+        b_sl = tl.load(logits + o_x, mask=o_x < V, other=float('-inf'))
+        b_tl = tl.load(target_logits + o_x, mask=o_x < V, other=float('-inf'))
+        b_sp_log = b_sl - sm - log(sd)
+        b_tp_log = b_tl - tm - log(td)
+        b_sp = exp(b_sp_log)
+        b_tp = exp(b_tp_log)
+        b_kl = tl.where(o_x < V, b_tp * (b_tp_log - b_sp_log), 0)
+        b_dl = -b_tp + b_sp
+        b_loss += tl.sum(b_kl)
+        if reduction == 'batchmean':
+            b_dl = b_dl / N
+        tl.store(logits + o_x, b_dl, mask=o_x < V)
+    if reduction == 'batchmean':
+        b_loss = b_loss / N
+    tl.store(loss + i_n * s_loss, b_loss)
+@triton.jit
+def elementwise_mul_kernel(
+    x,
+    g,
+    N: tl.constexpr,
+    B: tl.constexpr,
+):
+    i_x = tl.program_id(0).to(tl.int64)
+    o_x = i_x * B + tl.arange(0, B)
+    b_g = tl.load(g)
+    b_x = tl.load(x + o_x, mask=o_x < N)
+    tl.store(x + o_x, b_x * b_g, mask=o_x < N)
+def _npu_vocab_block_size(vocab_size: int, num_rows: int) -> int:
+    return compute_vocab_block_size(vocab_size, num_rows, _KLD_FWD_MEM_MULT)
+def fused_kl_div_forward_npu(
+    x: torch.Tensor,
+    target_x: torch.Tensor,
+    weight: torch.Tensor,
+    target_weight: torch.Tensor,
+    reduction: str = 'batchmean',
+    accumulate_grad_in_fp32: bool = True,
+):
+    device = x.device
+    N, H, V = *x.shape, weight.shape[0]
+    BV = _npu_vocab_block_size(V, N)
+    NC = min(8, triton.cdiv(V, H))
+    C = min(triton.next_power_of_2(triton.cdiv(N, NC)), ASCEND_MAX_GRID_DIM)
+    NC = triton.cdiv(N, C)
+    grad_dtype = torch.float32 if accumulate_grad_in_fp32 else weight.dtype
+    dx = torch.zeros_like(x, device=device)
+    dw = torch.zeros_like(weight, device=device, dtype=grad_dtype) if weight is not None else None
+    loss = torch.zeros(N, dtype=torch.float32, device=device)
+    for ic in range(NC):
+        start, end = ic * C, min((ic + 1) * C, N)
+        c_sx = x[start:end]
+        c_tx = target_x[start:end]
+        c_sl = F.linear(c_sx, weight)
+        c_tl = F.linear(c_tx, target_weight)
+        if weight is not None and c_sx.dtype != grad_dtype:
+            c_sx = c_sx.to(dtype=grad_dtype)
+        c_loss = loss[start:end]
+        kl_div_kernel[(c_sx.shape[0],)](
+            logits=c_sl,
+            target_logits=c_tl,
+            loss=c_loss,
+            s_logits=c_sl.stride(-2),
+            s_loss=c_loss.stride(-1),
+            reduction=reduction,
+            N=N,
+            V=V,
+            BV=BV,
+            num_warps=STATIC_WARPS,
+        )
+        c_grad = c_sl if c_sl.is_contiguous() else c_sl.contiguous()
+        dx[start:end] = torch.mm(c_grad, weight)
+        if weight is not None:
+            grad_w = c_grad.t().to(dtype=grad_dtype)
+            grad_x = c_sx if c_sx.dtype == grad_dtype else c_sx.to(dtype=grad_dtype)
+            dw.add_(grad_w @ grad_x)
+    loss = loss.sum()
+    if dw is not None:
+        dw = dw.to(weight)
+    return loss, dx, dw
+def fused_kl_div_backward_npu(
+    do: torch.Tensor,
+    dx: torch.Tensor,
+    dw: torch.Tensor,
+):
+    if torch.ne(do, torch.tensor(1.0, device=do.device)):
+        N, H = dx.shape
+        B = compute_elementwise_block_size(N * H, _ELEMENTWISE_MEM_MULT)
+        elementwise_mul_kernel[(triton.cdiv(N * H, B),)](
+            x=dx,
+            g=do,
+            N=N*H,
+            B=B,
+            num_warps=STATIC_WARPS,
+        )
+        if dw is not None:
+            V, H = dw.shape
+            B_dw = compute_elementwise_block_size(V * H, _ELEMENTWISE_MEM_MULT)
+            elementwise_mul_kernel[(triton.cdiv(V * H, B_dw),)](
+                x=dw,
+                g=do,
+                N=V*H,
+                B=B_dw,
+                num_warps=STATIC_WARPS,
+            )
+    return dx, dw

build/torch-cuda/modules/backends/triton_ascend/fused_linear_cross_entropy.py ADDED Viewed

	@@ -0,0 +1,347 @@

+# Copyright (c) 2023-2026, Songlin Yang, Yu Zhang, Zhiyuan Li
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+# For a list of all contributors, visit:
+#   https://github.com/fla-org/flash-linear-attention/graphs/contributors
+"""Fused linear cross-entropy kernels adapted for triton-ascend on Huawei NPU."""
+import torch
+import torch.nn.functional as F
+import triton
+import triton.language as tl
+from triton.language.math import tanh
+from ....ops.utils.op import exp, log
+from ....utils.ascend_ub_manager import (
+    ASCEND_MAX_GRID_DIM,
+    compute_elementwise_block_size,
+    compute_vocab_block_size,
+    iter_axis_launch_chunks,
+)
+# Fused linear CE: logsumexp forward vs gradient kernels along vocab.
+_LCE_FWD_MEM_MULT = 8.0
+_LCE_BWD_MEM_MULT = 12.0
+_ELEMENTWISE_MEM_MULT = 2.5
+STATIC_WARPS = 2
+@triton.heuristics({
+    'HAS_SCALE': lambda args: args['scale'] is not None,
+})
+@triton.jit
+def logsumexp_fwd_kernel(
+    x,
+    z,
+    scale,
+    softcapping: tl.constexpr,
+    D: tl.constexpr,
+    B: tl.constexpr,
+    ROWWISE: tl.constexpr,
+    HAS_SCALE: tl.constexpr,
+    HAS_SOFTCAPPING: tl.constexpr,
+):
+    i_n = tl.program_id(0).to(tl.int64)
+    if ROWWISE:
+        row = x + i_n * D
+        m = float('-inf')
+        d = 0.0
+        for start in range(0, D, B):
+            o = start + tl.arange(0, B)
+            b_x = tl.load(row + o, mask=o < D, other=float('-inf')).to(tl.float32)
+            if HAS_SCALE:
+                b_x = b_x * scale
+            if HAS_SOFTCAPPING:
+                b_x = softcapping * tanh(b_x / softcapping)
+            blk_max = tl.max(b_x, 0)
+            new_m = tl.maximum(m, blk_max)
+            d = d * exp(m - new_m) + tl.sum(exp(b_x - new_m), 0)
+            m = new_m
+        tl.store(z + i_n, m + log(d))
+    else:
+        i_d = tl.program_id(1).to(tl.int64)
+        o_d = i_d * B + tl.arange(0, B)
+        m_d = o_d < D
+        b_x = tl.load(x + i_n * D + o_d, mask=m_d, other=-float('inf'))
+        if HAS_SCALE:
+            b_x = b_x * scale
+        if HAS_SOFTCAPPING:
+            b_x = softcapping * tanh(b_x / softcapping)
+        b_m = tl.max(b_x, 0)
+        b_z = log(tl.sum(exp(b_x - b_m), 0)) + b_m
+        tl.store(z + i_n * tl.cdiv(D, B) + i_d, b_z)
+@triton.jit
+def cross_entropy_kernel(
+    logits,
+    lse,
+    target,
+    loss,
+    total,
+    ignore_index,
+    label_smoothing: tl.constexpr,
+    logit_scale: tl.constexpr,
+    logit_softcapping: tl.constexpr,
+    HAS_SOFTCAPPING: tl.constexpr,
+    reduction: tl.constexpr,
+    V: tl.constexpr,
+    BV: tl.constexpr,
+):
+    i_n = tl.program_id(0).to(tl.int64)
+    NV = tl.cdiv(V, BV)
+    b_y = tl.load(target + i_n)
+    logits += i_n * V
+    if b_y == ignore_index:
+        for i in range(0, V, BV):
+            o_v = i + tl.arange(0, BV)
+            tl.store(logits + o_v, 0.0, mask=o_v < V)
+        return
+    b_l = tl.load(logits + b_y).to(tl.float32) * logit_scale
+    if HAS_SOFTCAPPING:
+        b_t_y = tanh(b_l / logit_softcapping)
+        b_l = logit_softcapping * b_t_y
+        b_softcap_deriv_y = 1.0 - b_t_y * b_t_y
+    b_lse = tl.load(lse + i_n)
+    b_loss = b_lse - b_l
+    b_z = 0.0
+    eps = label_smoothing / V
+    for iv in range(0, NV):
+        o_v = iv * BV + tl.arange(0, BV)
+        b_logits = tl.load(logits + o_v, mask=o_v < V, other=float('-inf')).to(tl.float32) * logit_scale
+        if HAS_SOFTCAPPING:
+            b_t = tanh(b_logits / logit_softcapping)
+            b_capped = logit_softcapping * b_t
+        else:
+            b_capped = b_logits
+        if label_smoothing > 0:
+            b_z += tl.sum(tl.where(o_v < V, -eps * b_capped, 0.0))
+        b_p = (exp(b_capped - b_lse) - eps) * logit_scale
+        if HAS_SOFTCAPPING:
+            b_p = b_p * (1.0 - b_t * b_t)
+        if reduction == "mean":
+            b_p = b_p / total
+        tl.store(logits + o_v, b_p, mask=o_v < V)
+    if label_smoothing > 0:
+        b_loss = b_loss * (1 - label_smoothing) + (b_z + label_smoothing * b_lse)
+    b_l = tl.load(logits + b_y)
+    if HAS_SOFTCAPPING:
+        b_sc_factor = b_softcap_deriv_y
+    else:
+        b_sc_factor = 1.0
+    if reduction == 'mean':
+        b_loss = b_loss / total
+        b_l += (label_smoothing - 1) / total * logit_scale * b_sc_factor
+    else:
+        b_l += (label_smoothing - 1) * logit_scale * b_sc_factor
+    tl.store(loss + i_n, b_loss)
+    tl.store(logits + b_y, b_l)
+@triton.jit
+def elementwise_mul_kernel(
+    x,
+    g,
+    N: tl.constexpr,
+    B: tl.constexpr,
+):
+    i_x = tl.program_id(0).to(tl.int64)
+    o_x = i_x * B + tl.arange(0, B)
+    b_g = tl.load(g)
+    b_x = tl.load(x + o_x, mask=o_x < N)
+    tl.store(x + o_x, b_x * b_g, mask=o_x < N)
+def _npu_vocab_block_size(vocab_size: int, num_rows: int, is_backward: bool = True) -> int:
+    memory_multiplier = _LCE_BWD_MEM_MULT if is_backward else _LCE_FWD_MEM_MULT
+    return compute_vocab_block_size(vocab_size, num_rows, memory_multiplier)
+def logsumexp_fwd_npu(
+    x,
+    scale: float | None = None,
+    softcapping: float | None = None,
+    dtype: torch.dtype | None = None,
+):
+    shape = x.shape
+    x = x.view(-1, shape[-1])
+    N, D = x.shape
+    B = _npu_vocab_block_size(D, N, is_backward=False)
+    has_softcapping = softcapping is not None
+    softcap_val = float(softcapping) if has_softcapping else 0.0
+    z = x.new_empty(N, dtype=torch.float)
+    for row_off, row_len in iter_axis_launch_chunks(N, 1, max_grid=ASCEND_MAX_GRID_DIM):
+        logsumexp_fwd_kernel[(row_len,)](
+            x=x[row_off:row_off + row_len],
+            z=z[row_off:row_off + row_len],
+            scale=scale,
+            softcapping=softcap_val,
+            D=D,
+            B=B,
+            ROWWISE=True,
+            HAS_SOFTCAPPING=has_softcapping,
+        )
+    z = z.view(*shape[:-1])
+    if dtype is not None and dtype != torch.float:
+        z = z.to(dtype)
+    return z
+def fused_linear_cross_entropy_forward_npu(
+    x: torch.Tensor,
+    target: torch.LongTensor,
+    weight: torch.Tensor,
+    bias: torch.Tensor = None,
+    ignore_index: int = -100,
+    label_smoothing: float = 0.0,
+    logit_scale: float = 1.0,
+    logit_softcapping: float = None,
+    num_chunks: int = 8,
+    reduction: str = "mean",
+    use_l2warp: bool = False,
+    l2_penalty_factor: float = 1e-4,
+    accumulate_grad_in_fp32: bool = True,
+):
+    device = x.device
+    N, H, V = *x.shape, weight.shape[0]
+    BV = _npu_vocab_block_size(V, N)
+    has_softcapping = logit_softcapping is not None
+    softcap_val = float(logit_softcapping) if has_softcapping else 0.0
+    NC = min(num_chunks, triton.cdiv(V, H))
+    C = min(triton.next_power_of_2(triton.cdiv(N, NC)), ASCEND_MAX_GRID_DIM)
+    NC = triton.cdiv(N, C)
+    dx = torch.zeros_like(x, device=device)
+    grad_dtype = torch.float32 if accumulate_grad_in_fp32 else weight.dtype
+    bias_grad_dtype = None
+    if bias is not None:
+        bias_grad_dtype = torch.float32 if accumulate_grad_in_fp32 else bias.dtype
+    dw = torch.zeros_like(weight, device=device, dtype=grad_dtype) if weight is not None else None
+    db = torch.zeros_like(bias, device=device, dtype=bias_grad_dtype) if bias is not None else None
+    loss = torch.zeros(N, device=device, dtype=torch.float)
+    total = target.ne(ignore_index).sum().item()
+    for ic in range(NC):
+        start, end = ic * C, min((ic + 1) * C, N)
+        c_x = x[start:end]
+        c_logits = F.linear(c_x, weight, bias)
+        if weight is not None and c_x.dtype != grad_dtype:
+            c_x = c_x.to(dtype=grad_dtype)
+        c_target = target[start:end]
+        c_lse = logsumexp_fwd_npu(c_logits, scale=logit_scale, softcapping=logit_softcapping, dtype=torch.float)
+        c_loss = loss[start:end]
+        if use_l2warp:
+            c_maxx, c_ids = torch.max(c_logits, -1, keepdim=True)
+        cross_entropy_kernel[(c_logits.shape[0],)](
+            logits=c_logits,
+            lse=c_lse,
+            target=c_target,
+            loss=c_loss,
+            total=total,
+            ignore_index=ignore_index,
+            label_smoothing=label_smoothing,
+            logit_scale=logit_scale,
+            logit_softcapping=softcap_val,
+            HAS_SOFTCAPPING=has_softcapping,
+            reduction=reduction,
+            V=V,
+            BV=BV,
+            num_warps=STATIC_WARPS,
+        )
+        if use_l2warp:
+            g_logits_l2 = torch.zeros_like(c_logits)
+            l2_factor = l2_penalty_factor / N
+            penalty_grad = c_maxx * l2_factor
+            g_logits_l2.scatter_(-1, c_ids, penalty_grad)
+            if weight is not None:
+                torch.addmm(
+                    input=dw,
+                    mat1=g_logits_l2.t().to(dtype=grad_dtype),
+                    mat2=c_x,
+                    out=dw,
+                )
+            if bias is not None:
+                torch.add(input=db, other=g_logits_l2.sum(0, dtype=bias_grad_dtype), out=db)
+            dx_l2_contribution = torch.mm(g_logits_l2, weight)
+        else:
+            dx_l2_contribution = 0.0
+        c_grad = c_logits if c_logits.is_contiguous() else c_logits.contiguous()
+        dx[start:end] = torch.mm(c_grad, weight) + dx_l2_contribution
+        if weight is not None:
+            grad_w = c_grad.t().to(dtype=grad_dtype)
+            grad_x = c_x if c_x.dtype == grad_dtype else c_x.to(dtype=grad_dtype)
+            dw.add_(grad_w @ grad_x)
+        if bias is not None:
+            torch.add(input=db, other=c_logits.sum(0, dtype=bias_grad_dtype), out=db)
+    loss = loss.sum()
+    if dw is not None:
+        dw = dw.to(weight)
+    if db is not None:
+        db = db.to(bias)
+    return loss, dx, dw, db
+def fused_linear_cross_entropy_backward_npu(
+    do: torch.Tensor,
+    dx: torch.Tensor,
+    dw: torch.Tensor,
+    db: torch.Tensor,
+):
+    if torch.ne(do, torch.tensor(1.0, device=do.device)):
+        N, H = dx.shape
+        B = compute_elementwise_block_size(N * H, _ELEMENTWISE_MEM_MULT)
+        elementwise_mul_kernel[(triton.cdiv(N * H, B),)](
+            x=dx,
+            g=do,
+            N=N*H,
+            B=B,
+            num_warps=STATIC_WARPS,
+        )
+        if dw is not None:
+            V, H = dw.shape
+            B_dw = compute_elementwise_block_size(V * H, _ELEMENTWISE_MEM_MULT)
+            elementwise_mul_kernel[(triton.cdiv(V * H, B_dw),)](
+                x=dw,
+                g=do,
+                N=V*H,
+                B=B_dw,
+                num_warps=STATIC_WARPS,
+            )
+        if db is not None:
+            V = db.shape[0]
+            B_db = compute_elementwise_block_size(V, _ELEMENTWISE_MEM_MULT)
+            elementwise_mul_kernel[(triton.cdiv(V, B_db),)](
+                x=db,
+                g=do,
+                N=V,
+                B=B_db,
+                num_warps=STATIC_WARPS,
+            )
+    return dx, dw, db

build/torch-cuda/modules/backends/triton_ascend/grpo.py ADDED Viewed

	@@ -0,0 +1,266 @@

+# Copyright (c) 2023-2026, Songlin Yang, Yu Zhang, Zhiyuan Li
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+# For a list of all contributors, visit:
+#   https://github.com/fla-org/flash-linear-attention/graphs/contributors
+"""GRPO loss kernels adapted for triton-ascend on Huawei NPU."""
+import torch
+import triton
+import triton.language as tl
+from ....ops.utils.op import exp, log
+from ....utils import input_guard
+from ....utils.ascend_ub_manager import ASCEND_MAX_GRID_DIM, compute_vocab_block_size, iter_axis_launch_chunks
+# GRPO: use conservative multiplier covering both fwd softmax and bwd grad paths.
+_GRPO_MEM_MULT = 8.0
+STATIC_WARPS = 2
+def _npu_vocab_block_size(vocab_size: int, num_rows: int) -> int:
+    return compute_vocab_block_size(vocab_size, num_rows, _GRPO_MEM_MULT)
+@triton.jit
+def grpo_fwd_kernel(
+    logits_ptr,
+    ref_logp_ptr,
+    input_ids_ptr,
+    advantages_ptr,
+    completion_mask_ptr,
+    loss_ptr,
+    lse_ptr,
+    beta,
+    save_kl: tl.constexpr,
+    B,
+    M,
+    N,
+    L,
+    start_idx,
+    BLOCK_SIZE: tl.constexpr,
+    ROW_OFFSET: tl.constexpr,
+):
+    row_idx = tl.program_id(0) + ROW_OFFSET
+    off_b = row_idx // L
+    N = tl.cast(N, tl.int64)
+    loss_ptr += row_idx
+    completion_mask_ptr += row_idx
+    not_skip = tl.load(completion_mask_ptr).to(tl.int1)
+    if not_skip == 1:
+        ref_logp_ptr += row_idx
+        lse_ptr += row_idx
+        advantages_ptr += off_b
+        logits_ptr += N * (row_idx + off_b)
+        input_ids_ptr += row_idx + (off_b + 1) * start_idx
+        base_cols = tl.arange(0, BLOCK_SIZE)
+        m_i = -float("inf")
+        l_i = 0.0
+        for start_n in tl.range(0, N, BLOCK_SIZE):
+            cols = start_n + base_cols
+            mask = cols < N
+            logits = tl.load(logits_ptr + cols, mask=mask, other=-float('inf')).to(tl.float32)
+            m_ij = tl.max(logits)
+            new_m_i = tl.maximum(m_i, m_ij)
+            l_i = l_i * exp(m_i - new_m_i) + tl.sum(exp(logits - new_m_i))
+            m_i = new_m_i
+        lse = log(l_i) + m_i
+        idx = tl.load(input_ids_ptr)
+        x = tl.load(logits_ptr + idx).to(tl.float32)
+        advantage = tl.load(advantages_ptr).to(tl.float32)
+        ref_logp = tl.load(ref_logp_ptr)
+        logp = x - lse
+        diff = ref_logp - logp
+        kl = exp(diff) - diff - 1
+        loss = kl * beta - advantage
+        tl.store(loss_ptr, loss.to(loss_ptr.dtype.element_ty))
+        tl.store(lse_ptr, lse.to(lse_ptr.dtype.element_ty))
+        if save_kl:
+            tl.store(loss_ptr + M, kl.to(loss_ptr.dtype.element_ty))
+    else:
+        tl.store(loss_ptr, 0.0)
+        if save_kl:
+            tl.store(loss_ptr + M, 0.0)
+@triton.jit
+def grpo_bwd_kernel(
+    dloss_ptr,
+    dlogits_ptr,
+    logits_ptr,
+    ref_logp_ptr,
+    input_ids_ptr,
+    advantages_ptr,
+    completion_mask_ptr,
+    lse_ptr,
+    beta,
+    B,
+    N,
+    L,
+    start_idx,
+    BLOCK_SIZE: tl.constexpr,
+    ROW_OFFSET: tl.constexpr,
+):
+    row_idx = tl.program_id(0) + ROW_OFFSET
+    off_b = row_idx // L
+    N = tl.cast(N, tl.int64)
+    dlogits_ptr += N * (row_idx + off_b)
+    base_cols = tl.arange(0, BLOCK_SIZE)
+    completion_mask_ptr += row_idx
+    not_skip = tl.load(completion_mask_ptr).to(tl.int1)
+    if not_skip == 1:
+        lse_ptr += row_idx
+        dloss_ptr += row_idx
+        advantages_ptr += off_b
+        ref_logp_ptr += row_idx
+        logits_ptr += N * (row_idx + off_b)
+        input_ids_ptr += row_idx + (off_b + 1) * start_idx
+        dloss = tl.load(dloss_ptr).to(tl.float32)
+        lse = tl.load(lse_ptr).to(tl.float32)
+        idx = tl.load(input_ids_ptr)
+        x = tl.load(logits_ptr + idx).to(tl.float32)
+        advantage = tl.load(advantages_ptr).to(tl.float32)
+        ref_logp = tl.load(ref_logp_ptr)
+        logp = x - lse
+        dlogp = (beta * (-1.0 * exp(ref_logp - logp) + 1) - advantage) * dloss
+        for start_n in tl.range(0, N, BLOCK_SIZE):
+            cols = start_n + base_cols
+            mask = cols < N
+            logits = tl.load(logits_ptr + cols, mask=mask, other=-float('inf')).to(tl.float32)
+            probs = exp(logits - lse)
+            dlogits = tl.where(cols == idx, 1 - probs, -probs) * dlogp
+            tl.store(dlogits_ptr + cols, dlogits.to(dlogits_ptr.dtype.element_ty), mask=mask)
+    else:
+        dlogits = tl.zeros((BLOCK_SIZE,), dtype=tl.float32)
+        for start_n in tl.range(0, N, BLOCK_SIZE):
+            cols = start_n + base_cols
+            mask = cols < N
+            tl.store(dlogits_ptr + cols, dlogits.to(dlogits_ptr.dtype.element_ty), mask=mask)
+class GrpoLossNPU(torch.autograd.Function):
+    @input_guard
+    @staticmethod
+    def forward(ctx, logits, ref_logp, input_ids, advantages, beta, completion_mask, save_kl, inplace=True):
+        ctx.input_shape = logits.shape
+        B, L_ADD_1, N = ctx.input_shape
+        L = L_ADD_1 - 1
+        M = B * L
+        input_ids_start_index = input_ids.size(1) - L
+        block_size = _npu_vocab_block_size(N, M)
+        if not save_kl:
+            loss = torch.empty(B, L, device=logits.device, dtype=torch.float32)
+        else:
+            loss = torch.empty(B * 2, L, device=logits.device, dtype=torch.float32)
+        lse = torch.empty(B, L, device=logits.device, dtype=torch.float32)
+        if completion_mask is None:
+            completion_mask = torch.ones(B, L, device=logits.device, dtype=torch.int32)
+        else:
+            loss[:B].masked_fill_(completion_mask.logical_not(), 0.0)
+        for row_off, row_len in iter_axis_launch_chunks(M, 1, max_grid=ASCEND_MAX_GRID_DIM):
+            grpo_fwd_kernel[(row_len,)](
+                logits_ptr=logits,
+                ref_logp_ptr=ref_logp,
+                input_ids_ptr=input_ids,
+                advantages_ptr=advantages,
+                completion_mask_ptr=completion_mask,
+                loss_ptr=loss,
+                lse_ptr=lse,
+                beta=beta,
+                save_kl=save_kl,
+                B=B,
+                M=M,
+                N=N,
+                L=L,
+                start_idx=input_ids_start_index,
+                BLOCK_SIZE=block_size,
+                ROW_OFFSET=row_off,
+                num_warps=STATIC_WARPS,
+            )
+        ctx.beta = beta
+        ctx.save_for_backward(lse, logits, input_ids, advantages, completion_mask)
+        ctx.ref_logp = ref_logp
+        ctx.inplace = inplace
+        ctx.block_size = block_size
+        return loss
+    @input_guard
+    @staticmethod
+    def backward(ctx, dloss):
+        lse, logits, input_ids, advantages, completion_mask = ctx.saved_tensors
+        inplace = ctx.inplace
+        B, L_ADD_1, N = ctx.input_shape
+        L = L_ADD_1 - 1
+        M = B * L
+        block_size = ctx.block_size
+        input_ids_start_index = input_ids.size(1) - L
+        dlogits = logits if inplace else torch.empty_like(logits)
+        for row_off, row_len in iter_axis_launch_chunks(M, 1, max_grid=ASCEND_MAX_GRID_DIM):
+            grpo_bwd_kernel[(row_len,)](
+                dloss_ptr=dloss,
+                dlogits_ptr=dlogits,
+                logits_ptr=logits,
+                ref_logp_ptr=ctx.ref_logp,
+                input_ids_ptr=input_ids,
+                advantages_ptr=advantages,
+                completion_mask_ptr=completion_mask,
+                lse_ptr=lse,
+                beta=ctx.beta,
+                B=B,
+                N=N,
+                L=L,
+                BLOCK_SIZE=block_size,
+                start_idx=input_ids_start_index,
+                ROW_OFFSET=row_off,
+                num_warps=STATIC_WARPS,
+            )
+        dlogits[:, -1, :].fill_(0.0)
+        return dlogits.view(*ctx.input_shape), None, None, None, None, None, None, None
+def fused_grpo_loss_npu(
+    logits,
+    ref_logp,
+    input_ids,
+    advantages,
+    beta=0.1,
+    completion_mask=None,
+    save_kl=False,
+    inplace=False,
+) -> torch.Tensor:
+    out = GrpoLossNPU.apply(
+        logits,
+        ref_logp,
+        input_ids,
+        advantages,
+        beta,
+        completion_mask,
+        save_kl,
+        inplace,
+    )
+    if not save_kl:
+        return out
+    return out.chunk(2, axis=0)

build/torch-cuda/modules/backends/triton_ascend/layernorm.py ADDED Viewed

	@@ -0,0 +1,355 @@

+# Copyright (c) 2023-2026, Songlin Yang, Yu Zhang, Zhiyuan Li
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+# For a list of all contributors, visit:
+#   https://github.com/fla-org/flash-linear-attention/graphs/contributors
+"""LayerNorm / RMSNorm / GroupNorm kernels adapted for triton-ascend on Huawei NPU."""
+import torch
+import triton
+import triton.language as tl
+from ....utils import get_multiprocessor_count
+from ....utils.ascend_ub_manager import ASCEND_MAX_GRID_DIM, compute_ub_block_size, iter_axis_launch_chunks
+# Peak live fp32 vectors in row-wise kernel1 (see Liger Ascend layer_norm).
+_FWD_MEM_MULT = 6.0
+_BWD_MEM_MULT = 8.0
+_UB_SAFETY_MARGIN = 0.85
+# Legacy byte cap when UB capacity cannot be detected (65536 // fp32).
+_FALLBACK_MAX_BD = 65536 // 4
+def _get_layer_norm_bd(D: int, is_forward: bool) -> int:
+    """Return power-of-2 block size for feature dim D under UB constraints."""
+    memory_multiplier = _FWD_MEM_MULT if is_forward else _BWD_MEM_MULT
+    return compute_ub_block_size(
+        D,
+        memory_multiplier,
+        safety_margin=_UB_SAFETY_MARGIN,
+        fallback=_FALLBACK_MAX_BD,
+        desired=triton.next_power_of_2(D),
+    )
+def _layer_norm_bwd_launch_config(T: int, G: int, device_index: int) -> tuple[int, int, int]:
+    """Return (NS, BS, GS) capped under Ascend grid limit."""
+    NS = min(triton.cdiv(get_multiprocessor_count(device_index), G), T // G) * G
+    NS = min(NS, ASCEND_MAX_GRID_DIM)
+    BS = triton.cdiv(T, NS) if NS > 0 else T
+    GS = NS // G if G > 0 else NS
+    return NS, BS, GS
+@triton.jit
+def layer_norm_fwd_kernel1(
+    x,
+    y,
+    w,
+    b,
+    res,
+    res_out,
+    mean,
+    rstd,
+    eps,
+    G: tl.constexpr,
+    D: tl.constexpr,
+    BD: tl.constexpr,
+    IS_RMS_NORM: tl.constexpr,
+    HAS_RESIDUAL: tl.constexpr,
+    STORE_RESIDUAL_OUT: tl.constexpr,
+    HAS_WEIGHT: tl.constexpr,
+    HAS_BIAS: tl.constexpr,
+):
+    i_t = tl.program_id(0)
+    i_g = i_t % G
+    x += i_t * D
+    y += i_t * D
+    if HAS_RESIDUAL:
+        res += i_t * D
+    if STORE_RESIDUAL_OUT:
+        res_out += i_t * D
+    o_d = tl.arange(0, BD)
+    m_d = o_d < D
+    b_x = tl.load(x + o_d, mask=m_d, other=0.0).to(tl.float32)
+    if HAS_RESIDUAL:
+        b_x += tl.load(res + o_d, mask=m_d, other=0.0).to(tl.float32)
+    if STORE_RESIDUAL_OUT:
+        tl.store(res_out + o_d, b_x, mask=m_d)
+    if not IS_RMS_NORM:
+        b_mean = tl.sum(b_x, axis=0) / D
+        tl.store(mean + i_t, b_mean)
+        b_xbar = tl.where(m_d, b_x - b_mean, 0.0)
+        b_var = tl.sum(b_xbar * b_xbar, axis=0) / D
+    else:
+        b_xbar = tl.where(m_d, b_x, 0.0)
+        b_var = tl.sum(b_xbar * b_xbar, axis=0) / D
+    b_rstd = 1 / tl.sqrt(b_var + eps)
+    tl.store(rstd + i_t, b_rstd)
+    if HAS_WEIGHT:
+        b_w = tl.load(w + i_g * D + o_d, mask=m_d).to(tl.float32)
+    if HAS_BIAS:
+        b_b = tl.load(b + i_g * D + o_d, mask=m_d).to(tl.float32)
+    b_x_hat = (b_x - b_mean) * b_rstd if not IS_RMS_NORM else b_x * b_rstd
+    b_y = b_x_hat * b_w if HAS_WEIGHT else b_x_hat
+    if HAS_BIAS:
+        b_y = b_y + b_b
+    tl.store(y + o_d, b_y, mask=m_d)
+@triton.heuristics({
+    'RECOMPUTE_OUTPUT': lambda args: args['y'] is not None,
+})
+@triton.jit
+def layer_norm_bwd_kernel1(
+    x,
+    w,
+    b,
+    y,
+    dy,
+    dx,
+    dw,
+    db,
+    dres,
+    dres_in,
+    mean,
+    rstd,
+    T,
+    G: tl.constexpr,
+    D: tl.constexpr,
+    BS: tl.constexpr,
+    BD: tl.constexpr,
+    GS: tl.constexpr,
+    IS_RMS_NORM: tl.constexpr,
+    HAS_DRESIDUAL: tl.constexpr,
+    STORE_DRESIDUAL: tl.constexpr,
+    HAS_WEIGHT: tl.constexpr,
+    HAS_BIAS: tl.constexpr,
+    RECOMPUTE_OUTPUT: tl.constexpr,
+):
+    i_s = tl.program_id(0)
+    i_g, i_sg = i_s // GS, i_s % GS
+    o_d = tl.arange(0, BD)
+    mask = o_d < D
+    if HAS_WEIGHT:
+        b_w = tl.load(w + i_g * D + o_d, mask=mask).to(tl.float32)
+        b_dw = tl.zeros((BD,), dtype=tl.float32)
+    if RECOMPUTE_OUTPUT and HAS_BIAS:
+        b_b = tl.load(b + i_g * D + o_d, mask=mask, other=0.0).to(tl.float32)
+    if HAS_BIAS:
+        b_db = tl.zeros((BD,), dtype=tl.float32)
+    for i_t in range(i_sg * BS * G + i_g, min((i_sg * BS + BS) * G + i_g, T), G):
+        b_x = tl.load(x + i_t * D + o_d, mask=mask, other=0).to(tl.float32)
+        b_dy = tl.load(dy + i_t * D + o_d, mask=mask, other=0).to(tl.float32)
+        if not IS_RMS_NORM:
+            b_mean = tl.load(mean + i_t)
+        b_rstd = tl.load(rstd + i_t)
+        b_xhat = (b_x - b_mean) * b_rstd if not IS_RMS_NORM else b_x * b_rstd
+        b_xhat = tl.where(mask, b_xhat, 0.0)
+        if RECOMPUTE_OUTPUT:
+            b_y = b_xhat * b_w if HAS_WEIGHT else b_xhat
+            if HAS_BIAS:
+                b_y = b_y + b_b
+            tl.store(y + i_t * D + o_d, b_y, mask=mask)
+        b_wdy = b_dy
+        if HAS_WEIGHT:
+            b_wdy = b_dy * b_w
+            b_dw += b_dy * b_xhat
+        if HAS_BIAS:
+            b_db += b_dy
+        if not IS_RMS_NORM:
+            b_c1 = tl.sum(b_xhat * b_wdy, axis=0) / D
+            b_c2 = tl.sum(b_wdy, axis=0) / D
+            b_dx = (b_wdy - (b_xhat * b_c1 + b_c2)) * b_rstd
+        else:
+            b_c1 = tl.sum(b_xhat * b_wdy, axis=0) / D
+            b_dx = (b_wdy - b_xhat * b_c1) * b_rstd
+        if HAS_DRESIDUAL:
+            b_dres = tl.load(dres + i_t * D + o_d, mask=mask, other=0).to(tl.float32)
+            b_dx += b_dres
+        b_dx = tl.cast(b_dx, dtype=dx.dtype.element_ty, fp_downcast_rounding='rtne')
+        if STORE_DRESIDUAL:
+            tl.store(dres_in + i_t * D + o_d, b_dx, mask=mask)
+        tl.store(dx + i_t * D + o_d, b_dx, mask=mask)
+    if HAS_WEIGHT:
+        tl.store(dw + i_s * D + o_d, b_dw, mask=mask)
+    if HAS_BIAS:
+        tl.store(db + i_s * D + o_d, b_db, mask=mask)
+def _launch_layer_norm_fwd_kernel1(
+    x: torch.Tensor,
+    y: torch.Tensor,
+    weight: torch.Tensor,
+    bias: torch.Tensor,
+    residual: torch.Tensor,
+    res_out: torch.Tensor,
+    mean: torch.Tensor,
+    rstd: torch.Tensor,
+    eps: float,
+    G: int,
+    D: int,
+    BD: int,
+    is_rms_norm: bool,
+):
+    chunk_T = x.shape[0]
+    layer_norm_fwd_kernel1[(chunk_T,)](
+        x,
+        y,
+        weight,
+        bias,
+        residual,
+        res_out,
+        mean,
+        rstd,
+        eps,
+        G=G,
+        D=D,
+        BD=BD,
+        IS_RMS_NORM=is_rms_norm,
+        HAS_RESIDUAL=residual is not None,
+        STORE_RESIDUAL_OUT=res_out is not None,
+        HAS_WEIGHT=weight is not None,
+        HAS_BIAS=bias is not None,
+    )
+def layer_norm_fwd_npu(
+    x: torch.Tensor,
+    weight: torch.Tensor,
+    bias: torch.Tensor,
+    eps: float = 1e-5,
+    residual: torch.Tensor = None,
+    out_dtype: torch.dtype = None,
+    residual_dtype: torch.dtype = None,
+    is_rms_norm: bool = False,
+    num_groups: int = 1,
+):
+    if residual is not None:
+        residual_dtype = residual.dtype
+    T, D, G = *x.shape, num_groups
+    if residual is not None:
+        assert residual.shape == (T, D)
+    if weight is not None:
+        assert weight.shape == (G * D,)
+    if bias is not None:
+        assert bias.shape == (G * D,)
+    y = torch.empty_like(x, dtype=x.dtype if out_dtype is None else out_dtype)
+    if residual is not None or (residual_dtype is not None and residual_dtype != x.dtype):
+        res_out = torch.empty(T, D, device=x.device, dtype=residual_dtype)
+    else:
+        res_out = None
+    mean = torch.empty((T,), dtype=torch.float, device=x.device) if not is_rms_norm else None
+    rstd = torch.empty((T,), dtype=torch.float, device=x.device)
+    BD = _get_layer_norm_bd(D, is_forward=True)
+    if D > BD:
+        raise RuntimeError(
+            f"LayerNorm feature dim {D} exceeds UB-safe block size {BD}. "
+            "Column-tiled kernels are not yet implemented for this size."
+        )
+    # Ascend: use row-wise kernel1 (no make_block_ptr) for all feature dims.
+    # Split along rows when T exceeds the Ascend grid limit.
+    for row_start, row_len in iter_axis_launch_chunks(T, 1, max_grid=ASCEND_MAX_GRID_DIM):
+        row_end = row_start + row_len
+        _launch_layer_norm_fwd_kernel1(
+            x[row_start:row_end],
+            y[row_start:row_end],
+            weight,
+            bias,
+            None if residual is None else residual[row_start:row_end],
+            None if res_out is None else res_out[row_start:row_end],
+            None if mean is None else mean[row_start:row_end],
+            rstd[row_start:row_end],
+            eps,
+            G,
+            D,
+            BD,
+            is_rms_norm,
+        )
+    return y, mean, rstd, res_out if res_out is not None else x
+def layer_norm_bwd_npu(
+    dy: torch.Tensor,
+    x: torch.Tensor,
+    weight: torch.Tensor,
+    bias: torch.Tensor,
+    mean: torch.Tensor = None,
+    rstd: torch.Tensor = None,
+    dres: torch.Tensor = None,
+    has_residual: bool = False,
+    is_rms_norm: bool = False,
+    x_dtype: torch.dtype = None,
+    recompute_output: bool = False,
+    num_groups: int = 1,
+):
+    T, D, G = *x.shape, num_groups
+    assert dy.shape == (T, D)
+    if dres is not None:
+        assert dres.shape == (T, D)
+    if weight is not None:
+        assert weight.shape == (G * D,)
+    if bias is not None:
+        assert bias.shape == (G * D,)
+    dx = torch.empty_like(x) if x_dtype is None else torch.empty(T, D, dtype=x_dtype, device=x.device)
+    dres_in = torch.empty_like(x) if has_residual and dx.dtype != x.dtype else None
+    y = torch.empty(T, D, dtype=dy.dtype, device=dy.device) if recompute_output else None
+    BD = _get_layer_norm_bd(D, is_forward=False)
+    if D > BD:
+        raise RuntimeError(
+            f"LayerNorm feature dim {D} exceeds UB-safe block size {BD}. "
+            "Column-tiled kernels are not yet implemented for this size."
+        )
+    NS, BS, GS = _layer_norm_bwd_launch_config(T, G, x.device.index)
+    dw = torch.empty((NS, D), dtype=torch.float, device=weight.device) if weight is not None else None
+    db = torch.empty((NS, D), dtype=torch.float, device=bias.device) if bias is not None else None
+    grid = (NS,)
+    layer_norm_bwd_kernel1[grid](
+        x,
+        weight,
+        bias,
+        y,
+        dy,
+        dx,
+        dw,
+        db,
+        dres,
+        dres_in,
+        mean,
+        rstd,
+        T=T,
+        G=G,
+        D=D,
+        BS=BS,
+        BD=BD,
+        GS=GS,
+        IS_RMS_NORM=is_rms_norm,
+        HAS_DRESIDUAL=dres is not None,
+        STORE_DRESIDUAL=dres_in is not None,
+        HAS_WEIGHT=weight is not None,
+        HAS_BIAS=bias is not None,
+    )
+    dw = dw.view(G, -1, D).sum(1).to(weight).view_as(weight) if weight is not None else None
+    db = db.view(G, -1, D).sum(1).to(bias).view_as(bias) if bias is not None else None
+    if has_residual and dx.dtype == x.dtype:
+        dres_in = dx
+    return (dx, dw, db, dres_in) if not recompute_output else (dx, dw, db, dres_in, y)

build/torch-cuda/modules/backends/triton_ascend/rotary.py ADDED Viewed

	@@ -0,0 +1,211 @@

+# Copyright (c) 2023-2026, Songlin Yang, Yu Zhang, Zhiyuan Li
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+# For a list of all contributors, visit:
+#   https://github.com/fla-org/flash-linear-attention/graphs/contributors
+"""Rotary embedding kernels adapted for triton-ascend on Huawei NPU."""
+import torch
+import triton
+import triton.language as tl
+from ....ops.utils import prepare_chunk_indices
+from ....utils import autotune_cache_kwargs, get_multiprocessor_count
+from ....utils.ascend_ub_manager import (
+    ASCEND_MAX_GRID_DIM,
+    compute_grid_limited_tile_size,
+    compute_row_tile_block_size,
+    max_grid_axis_chunks,
+)
+# Peak live fp32 tiles in rotary kernel: cos, sin, x0, x1, o0, o1.
+_ROTARY_MEM_MULT = 6.0
+_ROTARY_SAFETY_MARGIN = 0.90
+# Ascend vector UB is small; large num_warps / stages explodes compile-time UB (see bishengir ub overflow).
+NUM_WARPS_AUTOTUNE = [2, 4]
+NUM_STAGES_AUTOTUNE = [1, 2]
+@triton.autotune(
+    configs=[
+        triton.Config({}, num_warps=num_warps, num_stages=num_stages)
+        for num_warps in NUM_WARPS_AUTOTUNE
+        for num_stages in NUM_STAGES_AUTOTUNE
+    ],
+    key=['B', 'H', 'D', 'INTERLEAVED'],
+    **autotune_cache_kwargs,
+)
+@triton.jit(do_not_specialize=['T'])
+def rotary_embedding_kernel(
+    x,
+    cos,
+    sin,
+    y,
+    cu_seqlens,
+    chunk_indices,
+    seq_offsets,
+    T,
+    B: tl.constexpr,
+    H: tl.constexpr,
+    D: tl.constexpr,
+    R: tl.constexpr,
+    TR: tl.constexpr,
+    BT: tl.constexpr,
+    BD: tl.constexpr,
+    IS_SEQLEN_OFFSETS_TENSOR: tl.constexpr,
+    IS_VARLEN: tl.constexpr,
+    INTERLEAVED: tl.constexpr,
+    CONJUGATE: tl.constexpr,
+    NT_OFFSET: tl.constexpr,
+):
+    i_t, i_b, i_h = tl.program_id(0), tl.program_id(1), tl.program_id(2)
+    i_t += NT_OFFSET
+    if IS_VARLEN:
+        i_n, i_t = tl.load(chunk_indices + i_t * 2).to(tl.int32), tl.load(chunk_indices + i_t * 2 + 1).to(tl.int32)
+        bos, eos = tl.load(cu_seqlens + i_n), tl.load(cu_seqlens + i_n + 1)
+        T = eos - bos
+        x = x + bos * H*D + i_h * D
+        y = y + bos * H*D + i_h * D
+    else:
+        i_n = i_b
+        x = x + i_n * T*H*D + i_h * D
+        y = y + i_n * T*H*D + i_h * D
+    if i_t * BT >= T:
+        return
+    o_t = i_t * BT + tl.arange(0, BT)
+    if not IS_SEQLEN_OFFSETS_TENSOR:
+        o_cs = o_t + seq_offsets
+    else:
+        o_cs = o_t + tl.load(seq_offsets + i_n)
+    m_t = (o_t >= 0) & (o_t < T) & (o_cs >= 0) & (o_cs < TR)
+    if not INTERLEAVED:
+        o_r = tl.arange(0, BD // 2)
+        p_x = x + o_t[:, None] * H*D + o_r[None, :]
+        p_cos = cos + (o_cs[:, None] * R + o_r[None, :])
+        p_sin = sin + (o_cs[:, None] * R + o_r[None, :])
+        mask = m_t[:, None] & (o_r < R)[None, :]
+        b_cos = tl.load(p_cos, mask=mask, other=1.0).to(tl.float32)
+        b_sin = tl.load(p_sin, mask=mask, other=0.0).to(tl.float32)
+        b_x0 = tl.load(p_x, mask=mask, other=0.0).to(tl.float32)
+        b_x1 = tl.load(p_x + R, mask=mask, other=0.0).to(tl.float32)
+        if CONJUGATE:
+            b_sin = -b_sin
+        b_o0 = b_x0 * b_cos - b_x1 * b_sin
+        b_o1 = b_x0 * b_sin + b_x1 * b_cos
+        p_y = y + (o_t[:, None] * H*D + o_r[None, :])
+        tl.store(p_y, b_o0, mask=mask)
+        tl.store(p_y + R, b_o1, mask=mask)
+    else:
+        o_d = tl.arange(0, BD)
+        o_d_swap = o_d + ((o_d + 1) % 2) * 2 - 1
+        o_d_repeat = tl.arange(0, BD) // 2
+        p_x0 = x + o_t[:, None] * H*D + o_d[None, :]
+        p_x1 = x + o_t[:, None] * H*D + o_d_swap[None, :]
+        p_cos = cos + (o_cs[:, None] * R + o_d_repeat[None, :])
+        p_sin = sin + (o_cs[:, None] * R + o_d_repeat[None, :])
+        mask = m_t[:, None] & (o_d_repeat < R)[None, :]
+        b_cos = tl.load(p_cos, mask=mask, other=1.0).to(tl.float32)
+        b_sin = tl.load(p_sin, mask=mask, other=0.0).to(tl.float32)
+        b_x0 = tl.load(p_x0, mask=mask, other=0.0).to(tl.float32)
+        b_x1 = tl.load(p_x1, mask=mask, other=0.0).to(tl.float32)
+        if CONJUGATE:
+            b_sin = -b_sin
+        b_o0 = b_x0 * b_cos
+        b_o1 = b_x1 * b_sin
+        b_y = tl.where(o_d[None, :] % 2 == 0, b_o0 - b_o1, b_o0 + b_o1)
+        p_y = y + (o_t[:, None] * H*D + o_d[None, :])
+        tl.store(p_y, b_y, mask=mask)
+def rotary_embedding_fwdbwd_npu(
+    x: torch.Tensor,
+    cos: torch.Tensor,
+    sin: torch.Tensor,
+    seqlen_offsets: int | torch.Tensor = 0,
+    cu_seqlens: torch.Tensor | None = None,
+    interleaved: bool = False,
+    inplace: bool = False,
+    conjugate: bool = False,
+    chunk_indices: torch.LongTensor | None = None,
+) -> torch.Tensor:
+    is_varlen = cu_seqlens is not None
+    B, T, H, D = x.shape
+    N = B if not is_varlen else cu_seqlens.shape[0] - 1
+    TR, R = cos.shape
+    R2 = R * 2
+    assert D <= 256, "Only support D <= 256"
+    assert TR >= T, f"TR must be >= T, got {TR} and {T}"
+    assert cos.dtype == sin.dtype, f"cos and sin must have the same dtype, got {cos.dtype} and {sin.dtype}"
+    assert x.dtype == cos.dtype, f"Input and cos/sin must have the same dtype, got {x.dtype} and {cos.dtype}"
+    if isinstance(seqlen_offsets, torch.Tensor):
+        assert seqlen_offsets.shape == (N,)
+        assert seqlen_offsets.dtype in [torch.int32, torch.int64]
+    else:
+        assert seqlen_offsets + T <= TR
+    y = torch.zeros_like(x) if not inplace else x
+    if R2 < D and not inplace:
+        y[..., R2:].copy_(x[..., R2:])
+    BD = triton.next_power_of_2(R2)
+    desired_bt = triton.next_power_of_2(triton.cdiv(T, get_multiprocessor_count(x.device.index)))
+    bt_cap = compute_row_tile_block_size(
+        desired_bt,
+        R2,
+        _ROTARY_MEM_MULT,
+        safety_margin=_ROTARY_SAFETY_MARGIN,
+        dtype_size=x.element_size(),
+        fallback=16 if R >= 128 else (32 if R >= 64 else 64),
+        min_block=1,
+    )
+    BT = min(bt_cap, desired_bt)
+    BT = compute_grid_limited_tile_size(T, B * H, BT, max_grid=ASCEND_MAX_GRID_DIM)
+    if chunk_indices is None and is_varlen:
+        chunk_indices = prepare_chunk_indices(cu_seqlens, BT)
+    NT = len(chunk_indices) if is_varlen else triton.cdiv(T, BT)
+    kernel_kwargs = dict(
+        x=x,
+        cos=cos,
+        sin=sin,
+        y=y,
+        cu_seqlens=cu_seqlens,
+        chunk_indices=chunk_indices,
+        seq_offsets=seqlen_offsets,
+        B=B,
+        T=T,
+        H=H,
+        D=D,
+        R=R,
+        TR=TR,
+        BT=BT,
+        BD=BD,
+        IS_SEQLEN_OFFSETS_TENSOR=isinstance(seqlen_offsets, torch.Tensor),
+        IS_VARLEN=is_varlen,
+        INTERLEAVED=interleaved,
+        CONJUGATE=conjugate,
+    )
+    max_nt = max_grid_axis_chunks(NT, B * H, max_grid=ASCEND_MAX_GRID_DIM)
+    for nt_off in range(0, NT, max_nt):
+        nt_len = min(max_nt, NT - nt_off)
+        if is_varlen:
+            kernel_kwargs['chunk_indices'] = chunk_indices[nt_off:nt_off + nt_len]
+            kernel_kwargs['NT_OFFSET'] = 0
+        else:
+            kernel_kwargs['chunk_indices'] = chunk_indices
+            kernel_kwargs['NT_OFFSET'] = nt_off
+        rotary_embedding_kernel[(nt_len, B, H)](**kernel_kwargs)
+    return y

build/torch-cuda/modules/conv/__init__.py ADDED Viewed

	@@ -0,0 +1,19 @@

+# Copyright (c) 2023-2026, Songlin Yang, Yu Zhang, Zhiyuan Li
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+# For a list of all contributors, visit:
+#   https://github.com/fla-org/flash-linear-attention/graphs/contributors
+from .causal_conv1d import causal_conv1d
+from .long_conv import ImplicitLongConvolution, LongConvolution, PositionalEmbedding, fft_conv
+from .short_conv import ShortConvolution
+__all__ = [
+    'ImplicitLongConvolution',
+    'LongConvolution',
+    'PositionalEmbedding',
+    'ShortConvolution',
+    'causal_conv1d',
+    'fft_conv',
+]

build/torch-cuda/modules/conv/causal_conv1d.py ADDED Viewed

	@@ -0,0 +1,129 @@

+# Copyright (c) 2023-2026, Songlin Yang, Yu Zhang, Zhiyuan Li
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+# For a list of all contributors, visit:
+#   https://github.com/fla-org/flash-linear-attention/graphs/contributors
+"""Main interface for causal 1D convolution operations."""
+import torch
+from ...ops.cp import FLACPContext
+from ...utils import input_guard
+@input_guard(no_guard_contiguous=["x"])
+def causal_conv1d(
+    x: torch.Tensor,
+    weight: torch.Tensor | None = None,
+    bias: torch.Tensor | None = None,
+    residual: torch.Tensor | None = None,
+    initial_state: torch.Tensor | None = None,
+    output_final_state: bool | None = False,
+    activation: str | None = None,
+    backend: str | None = 'triton',
+    cu_seqlens: torch.Tensor | None = None,
+    cu_seqlens_cpu: torch.LongTensor | None = None,
+    chunk_indices: torch.LongTensor | None = None,
+    cp_context: FLACPContext | None = None,
+    **kwargs,
+):
+    """
+    A causal 1D convolution implementation that powers Mamba/Mamba2 and DeltaNet architectures.
+    When a residual connection is provided, this implements the Canon operation
+    described in the paper at https://papers.ssrn.com/sol3/papers.cfm?abstract_id=5240330.
+    Args:
+        x (torch.Tensor):
+            Input tensor of shape [B, T, D].
+        weight (Optional[torch.Tensor]):
+            Weight tensor of shape [D, W]. Default: `None`.
+        bias (Optional[torch.Tensor]):
+            Bias tensor of shape [D]. Default: `None`.
+        residual (Optional[torch.Tensor]):
+            Residual tensor of shape [B, T, D]. Default: `None`.
+        initial_state (Optional[torch.Tensor]):
+            Initial state tensor of shape [N, D, W],
+            where `N` is the number of sequences in the batch and `W` is the kernel size.
+            If provided, the initial state is used to initialize the cache. Default: `None`.
+        output_final_state (Optional[bool]):
+            Whether to output the final state of shape [N, D, W]. Default: `False`.
+        activation (Optional[str]):
+            Activations applied to output, only `swish`/`silu` or `None` (i.e., no activation) are supported.
+            Default: `None`.
+        backend (Optional[str]):
+            Specifies the backend to use for the convolution operation. Supported values are `'cuda'` 、 `'triton'` and `'mix'`.
+            Default: `'triton'`.
+        cu_seqlens (Optional[torch.Tensor]):
+            Cumulative sequence lengths (optional)
+        chunk_indices (Optional[torch.LongTensor]):
+            Chunk indices for variable-length sequences (optional)
+    Returns:
+        Tuple of (output, final_state).
+        If `output_final_state` is `False`, the final state is `None`.
+    """
+    # Import here to avoid circular dependencies
+    from ...modules.conv.cp import causal_conv1d_cp
+    from ...modules.conv.cuda import causal_conv1d_cuda, fast_causal_conv1d_fn
+    from ...modules.conv.triton import CausalConv1dFunction
+    if cp_context is not None:
+        assert initial_state is None, "Initial state is not supported for CP"
+        assert output_final_state is False, "Output final state is not supported for CP"
+        output = causal_conv1d_cp(
+            x=x,
+            weight=weight,
+            bias=bias,
+            activation=activation,
+            chunk_indices=chunk_indices,
+            cp_context=cp_context,
+        )
+        return output, None
+    if backend == 'triton':
+        y, final_state = CausalConv1dFunction.apply(
+            x,
+            weight,
+            bias,
+            residual,
+            initial_state,
+            output_final_state,
+            activation,
+            cu_seqlens,
+            cu_seqlens_cpu,
+            chunk_indices,
+        )
+        return y, final_state
+    elif backend == 'mix':
+        seq_idx = kwargs.get('seq_idx')
+        return fast_causal_conv1d_fn(
+            x,
+            weight,
+            bias,
+            residual,
+            initial_state,
+            output_final_state,
+            activation,
+            cu_seqlens,
+            cu_seqlens_cpu=cu_seqlens_cpu,
+            chunk_indices=chunk_indices,
+            seq_idx=seq_idx,
+        )
+    elif backend == 'cuda':
+        return causal_conv1d_cuda(
+            x,
+            weight,
+            bias,
+            residual,
+            initial_state,
+            output_final_state,
+            activation,
+            cu_seqlens,
+            cu_seqlens_cpu=cu_seqlens_cpu,
+            **kwargs,
+        )
+    else:
+        raise ValueError(f"Unsupported backend: {backend}")

build/torch-cuda/modules/conv/cp/__init__.py ADDED Viewed

	@@ -0,0 +1,13 @@

+# Copyright (c) 2023-2026, Songlin Yang, Yu Zhang, Zhiyuan Li
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+# For a list of all contributors, visit:
+#   https://github.com/fla-org/flash-linear-attention/graphs/contributors
+from .ops import CausalConv1dFunctionCP, causal_conv1d_cp
+__all__ = [
+    'CausalConv1dFunctionCP',
+    'causal_conv1d_cp',
+]

build/torch-cuda/modules/conv/cp/ops.py ADDED Viewed

	@@ -0,0 +1,258 @@

+# Copyright (c) 2023-2026, Songlin Yang, Yu Zhang, Zhiyuan Li
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+# For a list of all contributors, visit:
+#   https://github.com/fla-org/flash-linear-attention/graphs/contributors
+import torch
+import torch.distributed as dist
+from ....ops.cp import FLACPContext, conv_cp_send_recv_bwd, conv_cp_send_recv_fwd
+from ....ops.utils import prepare_chunk_indices
+class CausalConv1dFunctionCP(torch.autograd.Function):
+    """
+    Context Parallel version of CausalConv1dFunction.
+    Forward:
+        1. Get tails from previous rank to construct initial_state
+        2. Call causal_conv1d_fwd
+    Backward:
+        1. Call causal_conv1d_bwd to get dx
+        2. Sync communication: add next rank's first W-1 token gradients to current rank's last W-1 tokens
+    """
+    @staticmethod
+    def _prepare_initial_state_for_cp(
+        x: torch.Tensor,
+        weight: torch.Tensor,
+        cu_seqlens: torch.Tensor | None,
+        context: FLACPContext,
+        group: dist.ProcessGroup | None,
+    ) -> torch.Tensor | None:
+        """Prepare initial_state for CP forward pass by communicating with previous rank.
+        Args:
+            x: Input tensor of shape [1, T, D]
+            weight: Weight tensor of shape [D, W]
+            cu_seqlens: Cumulative sequence lengths
+            context: CP context
+            group: Process group for communication
+        Returns:
+            initial_state: Initial state tensor of shape [N, D, W] or None
+        """
+        if group is None:
+            return None
+        W = weight.shape[-1]  # weight: [D, W]
+        D = weight.shape[0]
+        initial_state = None
+        if not context.is_first_rank:
+            # Non-first rank needs initial_state
+            assert x.dim() == 3 and x.shape[0] == 1, f"CP requires [1, T, D], got {x.shape}"
+            x_2d = x.squeeze(0)  # [T, D]
+            tails = x_2d[-(W-1):].contiguous()  # [W-1, D]
+            heads = conv_cp_send_recv_fwd(tails, group)  # [W-1, D]
+            # Construct initial_state: [N, D, W]
+            N = len(cu_seqlens) - 1
+            initial_state = torch.zeros(N, D, W, device=x.device, dtype=x.dtype)
+            valid_len = min(W - 1, context.pre_num_conv_tokens)
+            if valid_len > 0:
+                # heads[-valid_len:]: [valid_len, D] -> [D, valid_len]
+                initial_state[0, :, -valid_len:] = heads[-valid_len:].T
+        else:
+            # First rank also needs to participate in communication (send tails)
+            x_2d = x.squeeze(0)
+            tails = x_2d[-(W-1):].contiguous()
+            _ = conv_cp_send_recv_fwd(tails, group)  # Send but don't use
+        return initial_state
+    @staticmethod
+    def _correct_dx_for_cp(
+        dx: torch.Tensor,
+        dh0: torch.Tensor | None,
+        W: int,
+        group: dist.ProcessGroup | None,
+        is_first_rank: bool,
+        pre_num_conv_tokens: int = 0,
+    ) -> None:
+        """Correct dx gradients for CP backward pass by communicating with next rank.
+        Args:
+            dx: Gradient tensor to be corrected, shape [1, T, D]
+            dh0: Gradient w.r.t. initial_state, shape [N, D, W] or None
+            W: Kernel size
+            group: Process group for communication
+            is_first_rank: Whether this is the first rank in the sequence's processing chain
+            pre_num_conv_tokens: Number of tokens from the previous rank that
+                belong to the first sequence on the current rank. Must match the
+                value used in the forward pass to construct initial_state.
+        """
+        if group is None:
+            return
+        D = dx.shape[-1]
+        # dh0: [N, D, W] or None
+        # We only care about the first sequence's initial_state gradient
+        if dh0 is not None:
+            # Only keep gradients for positions that had real data from the
+            # previous rank. The forward fills only the last valid_len positions
+            # of initial_state; gradients for the remaining (zero-padded) positions
+            # must not flow back, otherwise they leak into unrelated sequences.
+            valid_len = min(W - 1, pre_num_conv_tokens)
+            d_initial_state = torch.zeros(W-1, D, device=dx.device, dtype=dx.dtype)
+            if valid_len > 0:
+                d_initial_state[-valid_len:] = dh0[0, :, -valid_len:].T
+        else:
+            # dh0 is None only when this is the first rank (no initial_state needed)
+            assert is_first_rank, "dh0 should not be None when is_first_rank=False"
+            d_initial_state = torch.zeros(W-1, D, device=dx.device, dtype=dx.dtype)
+        # Sync communication: send d_initial_state to previous rank, receive from next rank
+        recv_d_init = conv_cp_send_recv_bwd(d_initial_state, group)  # [W-1, D]
+        # Add to current rank's last W-1 tokens (these tokens are used as initial_state by next rank)
+        dx[0, -(W-1):, :].add_(recv_d_init)
+    @staticmethod
+    def forward(
+        ctx,
+        x: torch.Tensor,
+        weight: torch.Tensor,
+        bias: torch.Tensor | None,
+        activation: str | None,
+        chunk_indices: torch.Tensor | None,
+        cp_context: FLACPContext | None,
+        chunk_size: int | None,
+        backend: str = 'triton',
+    ):
+        # Import here to avoid circular dependency
+        from ....modules.conv.triton.ops import causal_conv1d_fwd
+        if cp_context is None:
+            raise ValueError("cp_context must be provided for CausalConv1dFunctionCP")
+        cu_seqlens = cp_context.cu_seqlens
+        cu_seqlens_cpu = cp_context.cu_seqlens_cpu
+        group = cp_context.group
+        # Get kernel_size
+        W = weight.shape[-1]  # weight: [D, W]
+        # Prepare initial_state for CP
+        initial_state = CausalConv1dFunctionCP._prepare_initial_state_for_cp(
+            x=x,
+            weight=weight,
+            cu_seqlens=cu_seqlens,
+            context=cp_context,
+            group=group,
+        )
+        ctx.save_for_backward(x, weight, bias, initial_state)
+        ctx.activation = activation
+        ctx.cu_seqlens = cu_seqlens
+        ctx.cu_seqlens_cpu = cu_seqlens_cpu
+        ctx.chunk_indices = chunk_indices
+        ctx.chunk_size = chunk_size
+        ctx.group = group
+        ctx.W = W
+        ctx.is_first_rank = cp_context.is_first_rank
+        ctx.pre_num_conv_tokens = cp_context.pre_num_conv_tokens
+        # Call original forward
+        y, _ = causal_conv1d_fwd(
+            x=x,
+            weight=weight,
+            bias=bias,
+            residual=None,
+            initial_state=initial_state,
+            output_final_state=False,
+            activation=activation,
+            cu_seqlens=cu_seqlens,
+            cu_seqlens_cpu=cu_seqlens_cpu,
+            chunk_indices=chunk_indices,
+            BT=chunk_size,
+        )
+        return y
+    @staticmethod
+    def backward(ctx, dy: torch.Tensor):
+        # Import here to avoid circular dependency
+        from ....modules.conv.triton.ops import causal_conv1d_bwd
+        x, weight, bias, initial_state = ctx.saved_tensors
+        group = ctx.group
+        W = ctx.W
+        # Call original backward
+        dx, dw, db, _, dh0 = causal_conv1d_bwd(
+            x=x,
+            dy=dy,
+            dht=None,
+            weight=weight,
+            bias=bias,
+            residual=None,
+            initial_state=initial_state,
+            activation=ctx.activation,
+            cu_seqlens=ctx.cu_seqlens,
+            cu_seqlens_cpu=ctx.cu_seqlens_cpu,
+            chunk_indices=ctx.chunk_indices,
+            BT=ctx.chunk_size,
+        )
+        # Correct dx gradients for CP
+        CausalConv1dFunctionCP._correct_dx_for_cp(
+            dx=dx,
+            dh0=dh0,
+            W=W,
+            group=group,
+            is_first_rank=ctx.is_first_rank,
+            pre_num_conv_tokens=ctx.pre_num_conv_tokens,
+        )
+        return dx, dw, db, None, None, None, None, None
+def causal_conv1d_cp(
+    x: torch.Tensor,
+    weight: torch.Tensor,
+    bias: torch.Tensor | None = None,
+    activation: str | None = None,
+    chunk_indices: torch.Tensor | None = None,
+    cp_context: FLACPContext | None = None,
+    chunk_size: int | None = None,
+    backend: str = 'triton',
+):
+    """
+    Context Parallel version of causal_conv1d.
+    Automatically handles communication in CP environment:
+    - Forward: get initial_state from previous rank
+    - Backward: correct dx gradients
+    Args:
+        x: Input tensor of shape [1, T, D]
+        weight: Weight tensor of shape [D, W]
+        bias: Bias tensor of shape [D] or None
+        activation: Activation function name or None
+        cu_seqlens: Cumulative sequence lengths
+        cu_seqlens_cpu: Cumulative sequence lengths on CPU
+        chunk_indices: Chunk indices for variable-length sequences
+        cp_context: CP context (required for CP mode)
+    """
+    if cp_context is None:
+        raise ValueError("cp_context must be provided for causal_conv1d_cp")
+    assert cp_context.conv1d_kernel_size is not None, "conv1d_kernel_size must be provided for causal_conv1d_cp"
+    assert cp_context.cu_seqlens is not None, "cu_seqlens must be provided for causal_conv1d_cp"
+    assert backend in ['triton'], "backend must be 'triton'"
+    chunk_size = chunk_size or 64
+    if chunk_indices is None:
+        chunk_indices = prepare_chunk_indices(cp_context.cu_seqlens, chunk_size, cu_seqlens_cpu=cp_context.cu_seqlens_cpu)
+    return CausalConv1dFunctionCP.apply(
+        x, weight, bias, activation,
+        chunk_indices, cp_context, chunk_size, backend
+    )

build/torch-cuda/modules/conv/cuda/__init__.py ADDED Viewed

	@@ -0,0 +1,14 @@

+# Copyright (c) 2023-2026, Songlin Yang, Yu Zhang, Zhiyuan Li
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+# For a list of all contributors, visit:
+#   https://github.com/fla-org/flash-linear-attention/graphs/contributors
+from .ops import FastCausalConv1dFn, causal_conv1d_cuda, fast_causal_conv1d_fn
+__all__ = [
+    'FastCausalConv1dFn',
+    'causal_conv1d_cuda',
+    'fast_causal_conv1d_fn',
+]

build/torch-cuda/modules/conv/cuda/ops.py ADDED Viewed

	@@ -0,0 +1,233 @@

+# Copyright (c) 2023-2026, Songlin Yang, Yu Zhang, Zhiyuan Li
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+# For a list of all contributors, visit:
+#   https://github.com/fla-org/flash-linear-attention/graphs/contributors
+"""CUDA-based mixed-mode implementation for causal convolution."""
+import torch
+from einops import rearrange
+from ....modules.conv.triton import causal_conv1d_update_states
+from ....ops.utils import prepare_sequence_ids
+from ....utils import input_guard
+try:
+    from causal_conv1d.cpp_functions import causal_conv1d_bwd_function
+except ImportError:
+    causal_conv1d_bwd_function = None
+try:
+    from causal_conv1d import causal_conv1d_fn as causal_conv1d_fn_cuda
+except ImportError:
+    causal_conv1d_fn_cuda = None
+class FastCausalConv1dFn(torch.autograd.Function):
+    """
+    Mixed-mode (Mix) Causal Convolution Implementation - Combining Triton Forward and CUDA Backward Propagation
+    This class implements forward propagation using FLA's Triton kernel, while using the optimized
+    implementation from TriDao's causal_conv1d CUDA package for backward propagation.
+    This hybrid strategy combines the advantages of both technologies:
+    - Forward: Uses FLA's Triton implementation, optimized for the FLA framework
+    - Backward: Uses TriDao's causal_conv1d_bwd_function CUDA implementation for faster speed
+    Performance Benefits:
+    - CUDA backward implementation is typically faster than the Triton version, reducing training time
+    - Maintains the flexibility and compatibility of forward propagation
+    Note:
+    - Input/Output format is (batch, seqlen, dim)
+    - Backward propagation requires causal_conv1d package: pip install causal-conv1d
+    - Supports SILU/Swish activation functions
+    - Current limitations (not yet supported):
+        * output_final_state must be False
+        * initial_states must be None
+        * residual must be None
+    """
+    @staticmethod
+    @input_guard(no_guard_contiguous=["x"])
+    def forward(
+        ctx,
+        x,
+        weight,
+        bias=None,
+        residual: torch.Tensor | None = None,
+        initial_states=None,
+        output_final_state=False,
+        activation=None,
+        cu_seqlens: torch.LongTensor | None = None,
+        cu_seqlens_cpu: torch.LongTensor | None = None,
+        chunk_indices: torch.LongTensor | None = None,
+        seq_idx: torch.LongTensor | None = None,
+    ):
+        if activation not in [None, "silu", "swish"]:
+            raise NotImplementedError("activation must be None, silu, or swish")
+        assert output_final_state is False, "output_final_state must be False for FastCausalConv1dFn"
+        assert initial_states is None, "initial_states must be None for FastCausalConv1dFn"
+        assert residual is None, "residual must be None for FastCausalConv1dFn"
+        bias = bias.contiguous() if bias is not None else None
+        if cu_seqlens is not None and seq_idx is None:
+            seq_idx = prepare_sequence_ids(cu_seqlens, cu_seqlens_cpu=cu_seqlens_cpu).to(
+                torch.int32).unsqueeze(0)
+        seq_idx = seq_idx.contiguous() if seq_idx is not None else None
+        # Import here to avoid circular dependency
+        from ....modules.conv.triton.ops import causal_conv1d_fwd
+        ctx.activation = activation in ["silu", "swish"]
+        out, _ = causal_conv1d_fwd(
+            x=x,
+            weight=weight,
+            bias=bias,
+            residual=None,
+            initial_state=None,
+            output_final_state=output_final_state,
+            activation=activation,
+            cu_seqlens=cu_seqlens,
+            cu_seqlens_cpu=cu_seqlens_cpu,
+            chunk_indices=chunk_indices,
+        )
+        ctx.save_for_backward(x, weight, bias, seq_idx, initial_states)
+        ctx.return_final_states = output_final_state
+        ctx.return_dinitial_states = (
+            initial_states is not None and initial_states.requires_grad
+        )
+        return out, None
+    @staticmethod
+    @input_guard
+    def backward(ctx, dout, *args):
+        x, weight, bias, seq_idx, initial_states = ctx.saved_tensors
+        dx = torch.empty_like(x, memory_format=torch.contiguous_format)
+        x = rearrange(x, 'b t d -> b d t')
+        dx = rearrange(dx, 'b t d -> b d t')
+        dout = rearrange(dout, 'b t d -> b d t')
+        dfinal_states = args[0] if ctx.return_final_states else None
+        if dout.stride(2) != 1 and dout.stride(1) != 1:
+            dout = dout.contiguous()
+        # The kernel supports passing in a pre-allocated dx (e.g., in case we want to fuse the
+        # backward of conv1d with the backward of chunk).
+        # Here we just pass in None and dx will be allocated in the C++ code.
+        dx, dweight, dbias, dinitial_states = causal_conv1d_bwd_function(
+            x,
+            weight,
+            bias,
+            dout,
+            seq_idx,
+            initial_states,
+            dfinal_states,
+            dx,
+            ctx.return_dinitial_states,
+            ctx.activation,
+        )
+        dx = rearrange(dx, 'b d t -> b t d')
+        return (
+            dx,
+            dweight,
+            dbias if bias is not None else None,
+            None,
+            None,
+            None,
+            None,
+            None,
+            None,
+            None,
+            None,
+        )
+def fast_causal_conv1d_fn(
+    x: torch.Tensor,
+    weight: torch.Tensor | None = None,
+    bias: torch.Tensor | None = None,
+    residual: torch.Tensor | None = None,
+    initial_state: torch.Tensor | None = None,
+    output_final_state: bool | None = False,
+    activation: str | None = None,
+    cu_seqlens: torch.Tensor | None = None,
+    cu_seqlens_cpu: torch.LongTensor | None = None,
+    chunk_indices: torch.LongTensor | None = None,
+    seq_idx: torch.LongTensor | None = None,
+):
+    """
+    x: (batch, seqlen, dim)
+    weight: (dim, width)
+    bias: (dim,)
+    seq_idx: (batch, seqlen)
+    initial_states: (batch, dim, width - 1)
+    final_states_out: (batch, dim, width - 1), to be written to
+    activation: either None or "silu" or "swish"
+    out: (batch, seqlen, dim)
+    """
+    assert causal_conv1d_bwd_function is not None, "causal_conv1d_bwd_function is not available"
+    return FastCausalConv1dFn.apply(
+        x,
+        weight,
+        bias,
+        residual,
+        initial_state,
+        output_final_state,
+        activation,
+        cu_seqlens,
+        cu_seqlens_cpu,
+        chunk_indices,
+        seq_idx,
+    )
+def causal_conv1d_cuda(
+    x: torch.Tensor,
+    weight: torch.Tensor,
+    bias: torch.Tensor | None = None,
+    residual: torch.Tensor | None = None,
+    initial_state: torch.Tensor | None = None,
+    output_final_state: bool | None = False,
+    activation: str | None = None,
+    cu_seqlens: torch.Tensor | None = None,
+    cu_seqlens_cpu: torch.LongTensor | None = None,
+    **kwargs,
+):
+    assert causal_conv1d_fn_cuda is not None, "causal_conv1d_fn_cuda is not available"
+    seq_idx = kwargs.get('seq_idx')
+    if cu_seqlens is not None or seq_idx is not None:
+        assert initial_state is None, "For CUDA backend, initial_state must be None if cu_seqlens or seq_idx is provided"
+    W = weight.shape[-1]
+    if x.stride(-1) != 1:
+        x = x.contiguous()
+    x_conv1d = rearrange(x, 'b t d -> b d t')
+    if cu_seqlens is not None and seq_idx is None:
+        seq_idx = prepare_sequence_ids(cu_seqlens, cu_seqlens_cpu=cu_seqlens_cpu).to(torch.int32).unsqueeze(0)
+    y = causal_conv1d_fn_cuda(
+        x=x_conv1d,
+        weight=weight,
+        bias=bias,
+        activation=activation,
+        seq_idx=seq_idx,
+        initial_states=None,
+        return_final_states=False,
+    )
+    y = rearrange(y, 'b d t -> b t d')
+    if output_final_state:
+        final_state = causal_conv1d_update_states(
+            x=x,
+            state_len=W,
+            initial_state=initial_state,
+            cu_seqlens=cu_seqlens,
+        )
+    else:
+        final_state = None
+    if residual is not None:
+        y.add_(residual)
+    return y, final_state

build/torch-cuda/modules/conv/long_conv.py ADDED Viewed

	@@ -0,0 +1,172 @@

+# Copyright (c) 2023-2026, Songlin Yang, Yu Zhang, Zhiyuan Li
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+# For a list of all contributors, visit:
+#   https://github.com/fla-org/flash-linear-attention/graphs/contributors
+import math
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from einops import rearrange
+def fft_conv(u, k, dropout_mask, gelu=True, k_rev=None):
+    seqlen = u.shape[-1]
+    fft_size = 2 * seqlen
+    k_f = torch.fft.rfft(k, n=fft_size) / fft_size
+    if k_rev is not None:
+        k_rev_f = torch.fft.rfft(k_rev, n=fft_size) / fft_size
+        k_f = k_f + k_rev_f.conj()
+    u_f = torch.fft.rfft(u.to(dtype=k.dtype), n=fft_size)
+    if len(u.shape) > 3:
+        k_f = k_f.unsqueeze(1)
+    y = torch.fft.irfft(u_f * k_f, n=fft_size, norm="forward")[..., :seqlen]
+    out = y + u
+    if gelu:
+        out = F.gelu(out)
+    if dropout_mask is not None:
+        return (out * rearrange(dropout_mask, "b H -> b H 1")).to(dtype=u.dtype)
+    else:
+        return out.to(dtype=u.dtype)
+class LongConvolution(nn.Module):
+    """
+    LongConvolution applies a convolution operation on the input tensor using a fixed
+    filter of length max_len.
+    The filter is learned during training and is applied using FFT convolution.
+    Args:
+        hidden_size (int): The number of expected features in the input and output.
+        max_len (int): The maximum sequence length.
+    Returns:
+        y: [batch_size, seq_len, hidden_size] tensor
+    """
+    def __init__(
+        self,
+        hidden_size: int,
+        max_len: int,
+        **kwargs,
+    ):
+        """
+        Initializes the LongConvolution module.
+        Args:
+            hidden_size (int): The number of expected features in the input and output.
+            max_len (int): The maximum sequence length.
+        """
+        super().__init__()
+        self.hidden_size = hidden_size
+        self.filter = nn.Parameter(torch.randn(self.hidden_size, max_len), requires_grad=True)
+    def forward(self, x: torch.Tensor, *args, **kwargs):
+        """
+        Applies the LongConvolution operation on the input tensor.
+        Args:
+            x: [batch_size, seq_len, hidden_size] tensor
+        Returns:
+            y: [batch_size, seq_len, hidden_size] tensor
+        """
+        x = x.transpose(1, 2)
+        y = fft_conv(x, self.filter, dropout_mask=None, gelu=False)
+        y = y.transpose(1, 2)
+        return y.to(dtype=x.dtype)
+class PositionalEmbedding(nn.Module):
+    def __init__(self, emb_dim: int, seq_len: int, **kwargs):
+        """Complex exponential positional embeddings for implicit long convolution filters."""
+        super().__init__()
+        self.seq_len = seq_len
+        # The time embedding fed to the filteres is normalized so that t_f = 1
+        t = torch.linspace(0, 1, self.seq_len)[None, :, None]  # 1, L, 1
+        if emb_dim > 1:
+            bands = (emb_dim - 1) // 2
+        # To compute the right embeddings we use the "proper" linspace
+        t_rescaled = torch.linspace(0, seq_len - 1, seq_len)[None, :, None]
+        w = 2 * math.pi * t_rescaled / seq_len  # 1, L, 1
+        f = torch.linspace(1e-4, bands - 1, bands)[None, None]
+        z = torch.exp(-1j * f * w)
+        z = torch.cat([t, z.real, z.imag], dim=-1)
+        self.z = nn.Parameter(z, requires_grad=False)
+    def forward(self, L):
+        return self.z[:, :L]
+class ImplicitLongConvolution(nn.Module):
+    """
+    Long convolution with implicit filter parameterized by an MLP.
+    Args:
+        hidden_size (int):
+            The number of expected features in the input and output.
+        max_len (int):
+            The maximum sequence length.
+        d_emb (Optional[int]):
+            The dimension of the positional embeddings. Must be odd and greater or equal to 3 (time, sine and cosine).
+            Defaults to 3.
+        d_hidden (Optional[int]):
+            The number of features in the hidden layer of the MLP. Defaults to 16.
+    Attributes:
+        pos_emb (`PositionalEmbedding`): The positional embedding layer.
+        mlp (`nn.Sequential`): The MLP that parameterizes the implicit filter.
+    """
+    def __init__(
+        self,
+        hidden_size: int,
+        max_len: int,
+        d_emb: int = 3,
+        d_hidden: int = 16,
+        **kwargs,
+    ):
+        """
+        Long convolution with implicit filter parameterized by an MLP.
+        """
+        super().__init__()
+        self.hidden_size = hidden_size
+        self.d_emb = d_emb
+        assert (
+            d_emb % 2 != 0 and d_emb >= 3
+        ), "d_emb must be odd and greater or equal to 3 (time, sine and cosine)"
+        self.pos_emb = PositionalEmbedding(d_emb, max_len)
+        # final linear layer
+        self.mlp = nn.Sequential(
+            nn.Linear(d_emb, d_hidden),
+            torch.nn.ReLU(),
+            nn.Linear(d_hidden, hidden_size),
+        )
+    def filter(self, seq_len: int, *args, **kwargs):
+        return self.mlp(self.pos_emb(seq_len)).transpose(1, 2)
+    def forward(self, x: torch.Tensor, *args, **kwargs):
+        """
+        Args:
+            x: [batch_size, seq_len, hidden_size] tensor
+        Returns:
+            y: [batch_size, seq_len, hidden_size] tensor
+        """
+        x = x.transpose(1, 2)
+        k = self.filter(x.shape[-1])
+        y = fft_conv(x, k, dropout_mask=None, gelu=False)
+        y = y.transpose(1, 2)
+        return y.to(dtype=x.dtype)

build/torch-cuda/modules/conv/short_conv.py ADDED Viewed

	@@ -0,0 +1,250 @@

+# Copyright (c) 2023-2026, Songlin Yang, Yu Zhang, Zhiyuan Li
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+# For a list of all contributors, visit:
+#   https://github.com/fla-org/flash-linear-attention/graphs/contributors
+"""Short convolution implementation for efficient causal convolutions."""
+import warnings
+import torch
+import torch.nn as nn
+from einops import rearrange
+try:
+    from causal_conv1d import causal_conv1d_fn as causal_conv1d_fn_cuda
+    from causal_conv1d import causal_conv1d_update as causal_conv1d_update_cuda
+except ImportError:
+    causal_conv1d_fn_cuda = None
+    causal_conv1d_update_cuda = None
+class ShortConvolution(nn.Conv1d):
+    """Short convolution layer for efficient causal convolution operations.
+    This class implements a depthwise 1D convolution with causal padding,
+    designed for efficient sequence processing. It supports multiple backends (Triton/CUDA)
+    and optional activation functions.
+    Args:
+        hidden_size (int): Number of input/output channels (must be equal for depthwise conv)
+        kernel_size (int): Size of the convolution kernel
+        bias (bool, optional): Whether to include learnable bias. Defaults to False.
+        activation (Optional[str], optional): Activation function ('silu' or 'swish'). Defaults to 'silu'.
+        backend (Optional[str], optional): Backend implementation ('triton' or 'cuda'). Defaults to 'triton'.
+        device (Optional[torch.device], optional): Device to place the layer on. Defaults to None.
+        dtype (Optional[torch.dtype], optional): Data type for layer parameters. Defaults to None.
+        **kwargs: Additional keyword arguments (deprecated 'use_fast_conv1d' supported for compatibility)
+    Attributes:
+        hidden_size (int): Number of channels
+        activation (Optional[str]): Selected activation function
+        backend (str): Actual backend being used (may differ from input due to availability)
+    Note:
+        - Uses depthwise convolution (groups=hidden_size) for efficiency
+        - Applies causal padding (kernel_size-1) to ensure no future information leakage
+        - Falls back to Triton backend if CUDA backend is unavailable
+    """
+    def __init__(
+        self,
+        hidden_size: int,
+        kernel_size: int,
+        bias: bool = False,
+        activation: str | None = 'silu',
+        backend: str | None = 'triton',
+        device: torch.device | None = None,
+        dtype: torch.dtype | None = None,
+        **kwargs,
+    ):
+        super().__init__(
+            in_channels=hidden_size,
+            out_channels=hidden_size,
+            kernel_size=kernel_size,
+            groups=hidden_size,
+            bias=bias,
+            padding=kernel_size - 1,
+            device=device,
+            dtype=dtype,
+        )
+        self.hidden_size = hidden_size
+        self.activation = None
+        if activation is not None:
+            assert activation in ['silu', 'swish'], f"Activation `{activation}` not supported yet."
+            self.activation = activation
+        if 'use_fast_conv1d' in kwargs:
+            warnings.warn(
+                "The `use_fast_conv1d` parameter is deprecated and will be ignored. "
+                "Please use the `backend` parameter instead.",
+            )
+        import os
+        self.backend = os.environ.get('FLA_CONV_BACKEND', backend)
+        if backend not in ['cuda', 'triton']:
+            raise ValueError(f"Invalid backend: {backend}, must be one of ['cuda', 'triton']")
+        if backend == 'cuda':
+            if causal_conv1d_fn_cuda is None:
+                warnings.warn(
+                    "The `backend` parameter is set to `cuda`, but `causal_conv1d_fn` is not available. "
+                    "Switching to the Triton implementation instead. "
+                    "Consider installing `causal_conv1d` to enable the CUDA backend.",
+                )
+                self.backend = 'triton'
+    def extra_repr(self):
+        s = ('{in_channels}, {out_channels}, kernel_size={kernel_size}'
+             ', stride={stride}')
+        if self.padding != (0,) * len(self.padding):
+            s += ', padding={padding}'
+        if self.dilation != (1,) * len(self.dilation):
+            s += ', dilation={dilation}'
+        if self.output_padding != (0,) * len(self.output_padding):
+            s += ', output_padding={output_padding}'
+        if self.groups != 1:
+            s += ', groups={groups}'
+        if self.bias is None:
+            s += ', bias=False'
+        if self.padding_mode != 'zeros':
+            s += ', padding_mode={padding_mode}'
+        if self.activation is not None:
+            s += ', activation={activation}'
+        s += f', backend={self.backend}'
+        return s.format(**self.__dict__)
+    def forward(
+        self,
+        x: torch.Tensor,
+        residual: torch.Tensor | None = None,
+        mask: torch.Tensor | None = None,
+        cache: torch.Tensor | None = None,
+        output_final_state: bool = False,
+        cu_seqlens: torch.LongTensor | None = None,
+        chunk_indices: torch.LongTensor | None = None,
+        **kwargs,
+    ) -> tuple[torch.Tensor, torch.Tensor | None]:
+        """
+        Args:
+            x (`torch.Tensor`):
+                Tensor of shape `[B, T, D]`. `B` must be 1 if `cu_seqlens` is provided.
+            residual (`Optional[torch.Tensor]`):
+                Residual tensor of shape `[B, T, D]`. Default: `None`.
+            mask (`Optional[torch.Tensor]`):
+                Attention mask dealing with padded positions.
+            cache (`Optional[torch.Tensor]`):
+                Previous cache tensor of shape `[N, D, W]`, where `W` is the kernel size.
+                If provided, the cache is updated **inplace**.
+            output_final_state (Optional[bool]):
+                Whether to output the final state of shape `[N, D, W]`. Default: `False`.
+            cu_seqlens (Optional[torch.LongTensor]):
+                Cumulative sequence lengths for each batch. Used for varlen. Default: `None`.
+                Shape: [B+1]
+            chunk_indices (Optional[torch.LongTensor]):
+                Chunk indices for variable-length sequences. Default: `None`.
+        Returns:
+            Tensor of shape `[B, T, D]`.
+        """
+        # Import here to avoid circular dependency
+        from ...modules.conv.causal_conv1d import causal_conv1d
+        B, T, *_ = x.shape
+        N = B if cu_seqlens is None else len(cu_seqlens) - 1
+        if mask is not None:
+            if cu_seqlens is not None:
+                raise ValueError("`mask` and `cu_seqlens` cannot be provided at the same time")
+            x = x.mul_(mask.unsqueeze(-1))
+        # in decoding phase, the cache (if provided) is updated inplace
+        if B * T == N:
+            y, cache = self.step(
+                x=x,
+                residual=residual,
+                cache=cache,
+                output_final_state=output_final_state,
+                cu_seqlens=cu_seqlens,
+            )
+            return y, cache
+        # cuda backend do not support:
+        # 1. both `cu_seqlens` and `cache` being provided
+        # 2. both `cu_seqlens` and `output_final_state` being provided
+        # and other small issues
+        # to simplify the implementation, we just switch to triton backend
+        if self.backend == 'cuda' and cache is not None:
+            warnings.warn(
+                "The CUDA backend does not support both `cu_seqlens` and `cache` being provided, "
+                "or both `cu_seqlens` and `output_final_state` being provided. "
+                "Switching to the Triton backend instead. ",
+                stacklevel=2,
+            )
+            self.backend = 'triton'
+        return causal_conv1d(
+            x=x,
+            weight=rearrange(self.weight, "d 1 w -> d w"),
+            bias=self.bias,
+            residual=residual,
+            initial_state=cache,
+            output_final_state=output_final_state,
+            activation=self.activation,
+            backend=self.backend,
+            cu_seqlens=cu_seqlens,
+            chunk_indices=chunk_indices,
+            **kwargs,
+        )
+    def step(
+        self,
+        x: torch.Tensor,
+        residual: torch.Tensor | None,
+        cache: torch.Tensor | None,
+        output_final_state: bool = False,
+        cu_seqlens: torch.LongTensor | None = None,
+    ) -> tuple[torch.Tensor, torch.Tensor | None]:
+        from ...modules.conv.triton.ops import causal_conv1d_update
+        B, _, D, W = *x.shape, self.kernel_size[0]
+        N = B if cu_seqlens is None else len(cu_seqlens) - 1
+        # Always initialise cache when None so the Triton kernel never
+        # receives a None tensor. Return value still respects output_final_state
+        # to maintain consistency with the non-step path in forward().
+        if cache is None:
+            cache = x.new_zeros(N, D, W)
+        # NOTE: we follow the fast mode that updates the cache in-place
+        if self.backend == 'triton':
+            y, cache = causal_conv1d_update(
+                x=x,
+                cache=cache,
+                residual=residual,
+                weight=rearrange(self.weight, "d 1 w -> d w"),
+                bias=self.bias,
+                activation=self.activation,
+            )
+            return y, (cache if output_final_state else None)
+        shape = x.shape
+        x = x.squeeze(0) if cu_seqlens is not None else x.squeeze(1)
+        # equivalent to:
+        # cache.copy_(cache.roll(shifts=-1, dims=-1))
+        # cache[:, :, -1] = x
+        # y = torch.sum(cache * rearrange(self.weight, "d 1 w -> d w"), dim=-1)
+        y = causal_conv1d_update_cuda(
+            x=x,
+            conv_state=cache,
+            weight=rearrange(self.weight, "d 1 w -> d w"),
+            bias=self.bias,
+            activation=self.activation,
+        )
+        y = y.view(shape)
+        if residual is not None:
+            y.add_(residual)
+        return y, (cache if output_final_state else None)
+    @property
+    def state_size(self) -> int:
+        return self.hidden_size * self.kernel_size

build/torch-cuda/modules/conv/triton/__init__.py ADDED Viewed

	@@ -0,0 +1,24 @@

+# Copyright (c) 2023-2026, Songlin Yang, Yu Zhang, Zhiyuan Li
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+# For a list of all contributors, visit:
+#   https://github.com/fla-org/flash-linear-attention/graphs/contributors
+from .ops import (
+    CausalConv1dFunction,
+    causal_conv1d_bwd,
+    causal_conv1d_fwd,
+    causal_conv1d_update,
+    causal_conv1d_update_states,
+    compute_dh0_triton,
+)
+__all__ = [
+    'CausalConv1dFunction',
+    'causal_conv1d_bwd',
+    'causal_conv1d_fwd',
+    'causal_conv1d_update',
+    'causal_conv1d_update_states',
+    'compute_dh0_triton',
+]

build/torch-cuda/modules/conv/triton/kernels.py ADDED Viewed

	@@ -0,0 +1,683 @@

+# Copyright (c) 2023-2026, Songlin Yang, Yu Zhang, Zhiyuan Li
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+# For a list of all contributors, visit:
+#   https://github.com/fla-org/flash-linear-attention/graphs/contributors
+import torch
+import triton
+import triton.language as tl
+from einops import rearrange
+from ....ops.utils.cache import fla_cache_autotune
+from ....utils import IS_AMD, autotune_cache_kwargs, input_guard
+NUM_WARPS_AUTOTUNE = [2, 4, 8, 16] if IS_AMD else [4, 8, 16, 32]
+STATIC_WARPS = 32 if not IS_AMD else 16
+@triton.heuristics({
+    'HAS_WEIGHT': lambda args: args['weight'] is not None,
+    'HAS_BIAS': lambda args: args['bias'] is not None,
+    'HAS_RESIDUAL': lambda args: args['residual'] is not None,
+    'USE_INITIAL_STATE': lambda args: args['initial_state'] is not None,
+    'IS_VARLEN': lambda args: args['cu_seqlens'] is not None,
+})
+@fla_cache_autotune(
+    configs=[
+        triton.Config({'BD': BD}, num_warps=num_warps)
+        for BD in [16, 32, 64, 128]
+        for num_warps in NUM_WARPS_AUTOTUNE
+    ],
+    key=['D', 'W', 'NB'],
+    **autotune_cache_kwargs,
+)
+@triton.jit
+def causal_conv1d_fwd_kernel(
+    x,
+    y,
+    weight,
+    bias,
+    residual,
+    cu_seqlens,
+    initial_state,
+    chunk_indices,
+    B,
+    T,
+    stride_x_n,
+    stride_x_t,
+    stride_x_d,
+    D: tl.constexpr,
+    W: tl.constexpr,
+    BT: tl.constexpr,
+    BW: tl.constexpr,
+    BD: tl.constexpr,
+    NB: tl.constexpr,
+    ACTIVATION: tl.constexpr,
+    HAS_WEIGHT: tl.constexpr,
+    HAS_BIAS: tl.constexpr,
+    HAS_RESIDUAL: tl.constexpr,
+    USE_INITIAL_STATE: tl.constexpr,
+    IS_VARLEN: tl.constexpr,
+):
+    i_d, i_t, i_b = tl.program_id(0), tl.program_id(1), tl.program_id(2)
+    if IS_VARLEN:
+        i_n, i_t = tl.load(chunk_indices + i_t * 2).to(tl.int32), tl.load(chunk_indices + i_t * 2 + 1).to(tl.int32)
+        bos, eos = tl.load(cu_seqlens + i_n).to(tl.int64), tl.load(cu_seqlens + i_n + 1).to(tl.int64)
+        T = eos - bos
+        p_x = x + bos * stride_x_t
+    else:
+        i_n = i_b
+        bos, eos = (i_b * T).to(tl.int64), (i_b * T + T).to(tl.int64)
+        p_x = x + tl.cast(i_b, tl.int64) * stride_x_n
+    o_d = i_d * BD + tl.arange(0, BD)
+    o_w = tl.arange(0, BW) + W - BW
+    m_d = o_d < D
+    m_w = o_w >= 0
+    if HAS_WEIGHT:
+        # [BD, BW]
+        b_w = tl.load(weight + o_d[:, None] * W + o_w, mask=m_d[:, None] & m_w, other=0).to(tl.float32)
+    b_y = tl.zeros((BT, BD), dtype=tl.float32)
+    if not USE_INITIAL_STATE:
+        for i_w in tl.static_range(-W + 1, 1):
+            p_yi = tl.make_block_ptr(p_x, (T, D), (stride_x_t, stride_x_d), (i_t * BT + i_w, i_d * BD), (BT, BD), (1, 0))
+            # [BT, BD]
+            b_yi = tl.load(p_yi, boundary_check=(0, 1)).to(tl.float32)
+            if HAS_WEIGHT:
+                b_yi *= tl.sum(b_w * (o_w == (i_w + W - 1)), 1)
+            b_y += b_yi
+    elif i_t * BT >= W:
+        # to make Triton compiler happy, we need to copy codes
+        for i_w in tl.static_range(-W + 1, 1):
+            p_yi = tl.make_block_ptr(p_x, (T, D), (stride_x_t, stride_x_d), (i_t * BT + i_w, i_d * BD), (BT, BD), (1, 0))
+            # [BT, BD]
+            b_yi = tl.load(p_yi, boundary_check=(0, 1)).to(tl.float32)
+            if HAS_WEIGHT:
+                b_yi *= tl.sum(b_w * (o_w == (i_w + W - 1)), 1)
+            b_y += b_yi
+    else:
+        o_t = i_t * BT + tl.arange(0, BT)
+        for i_w in tl.static_range(-W + 1, 1):
+            o_x = o_t + i_w
+            m_x = ((o_x >= 0) & (o_x < T))[:, None] & m_d
+            m_c = ((o_x + W >= 0) & (o_x < 0))[:, None] & m_d
+            b_yi = tl.load(
+                p_x + o_x[:, None] * stride_x_t + o_d * stride_x_d,
+                mask=m_x,
+                other=0
+            ).to(tl.float32)
+            b_yi += tl.load(initial_state + i_n * D*W + o_d * W + (o_x + W)[:, None], mask=m_c, other=0).to(tl.float32)
+            if HAS_WEIGHT:
+                b_yi *= tl.sum(b_w * (o_w == (i_w + W - 1)), 1)
+            b_y += b_yi
+    if HAS_BIAS:
+        b_y += tl.load(bias + o_d, mask=m_d).to(tl.float32)
+    if ACTIVATION == 'swish' or ACTIVATION == 'silu':
+        b_y = b_y * tl.sigmoid(b_y)
+    if HAS_RESIDUAL:
+        p_residual = tl.make_block_ptr(residual + bos * D, (T, D), (D, 1), (i_t * BT, i_d * BD), (BT, BD), (1, 0))
+        b_residual = tl.load(p_residual, boundary_check=(0, 1))
+        b_y += b_residual
+    p_y = tl.make_block_ptr(y + bos * D, (T, D), (D, 1), (i_t * BT, i_d * BD), (BT, BD), (1, 0))
+    tl.store(p_y, tl.cast(b_y, dtype=p_y.dtype.element_ty, fp_downcast_rounding='rtne'), boundary_check=(0, 1))
+@triton.heuristics({
+    'HAS_WEIGHT': lambda args: args['dw'] is not None,
+    'HAS_BIAS': lambda args: args['db'] is not None,
+    'USE_INITIAL_STATE': lambda args: args['initial_state'] is not None,
+    'USE_FINAL_STATE': lambda args: args['dht'] is not None,
+    'IS_VARLEN': lambda args: args['cu_seqlens'] is not None,
+})
+@fla_cache_autotune(
+    configs=[
+        triton.Config({'BD': BD}, num_warps=num_warps)
+        for BD in [16, 32, 64, 128]
+        for num_warps in [4, 8, 16, 32]
+    ],
+    key=['D', 'W', 'NB'],
+    **autotune_cache_kwargs,
+)
+@triton.jit
+def causal_conv1d_bwd_kernel(
+    x,
+    y,
+    weight,
+    initial_state,
+    dht,
+    dy,
+    dx,
+    dw,
+    db,
+    cu_seqlens,
+    chunk_indices,
+    B,
+    T,
+    stride_x_n,   # x batch stride
+    stride_x_t,   # x time stride
+    stride_x_d,   # x dim stride
+    stride_dx_n,  # dx batch stride
+    stride_dx_t,  # dx time stride
+    stride_dx_d,  # dx dim stride
+    stride_dy_n,  # dy batch stride
+    stride_dy_t,  # dy time stride
+    stride_dy_d,  # dy dim stride
+    D: tl.constexpr,
+    W: tl.constexpr,
+    BT: tl.constexpr,
+    BW: tl.constexpr,
+    BD: tl.constexpr,
+    NB: tl.constexpr,
+    ACTIVATION: tl.constexpr,
+    HAS_WEIGHT: tl.constexpr,
+    HAS_BIAS: tl.constexpr,
+    USE_INITIAL_STATE: tl.constexpr,
+    USE_FINAL_STATE: tl.constexpr,
+    IS_VARLEN: tl.constexpr,
+):
+    i_d, i_t, i_b = tl.program_id(0), tl.program_id(1), tl.program_id(2)
+    if IS_VARLEN:
+        i_tg = i_t
+        i_n, i_t = tl.load(chunk_indices + i_t * 2).to(tl.int32), tl.load(chunk_indices + i_t * 2 + 1).to(tl.int32)
+        bos, eos = tl.load(cu_seqlens + i_n).to(tl.int64), tl.load(cu_seqlens + i_n + 1).to(tl.int64)
+        T = eos - bos
+        p_x = x + bos * stride_x_t
+    else:
+        i_tg = i_b * tl.num_programs(1) + i_t
+        i_n = i_b
+        bos, eos = (i_b * T).to(tl.int64), (i_b * T + T).to(tl.int64)
+        p_x = x + tl.cast(i_b, tl.int64) * stride_x_n
+    if IS_VARLEN:
+        p_dy = dy + bos * stride_dy_t
+    else:
+        p_dy = dy + tl.cast(i_b, tl.int64) * stride_dy_n
+    o_d = i_d * BD + tl.arange(0, BD)
+    o_w = tl.arange(0, BW) + W - BW
+    m_d = o_d < D
+    m_w = o_w >= 0
+    if HAS_WEIGHT:
+        p_x = tl.make_block_ptr(p_x, (T, D), (stride_x_t, stride_x_d), (i_t * BT, i_d * BD), (BT, BD), (1, 0))
+        b_x = tl.load(p_x, boundary_check=(0, 1))
+        # [BD, BW]
+        b_w = tl.load(weight + o_d[:, None] * W + o_w, mask=m_d[:, None] & m_w, other=0)
+    b_dx = tl.zeros((BT, BD), dtype=tl.float32)
+    if HAS_BIAS:
+        b_db = tl.zeros((BD,), dtype=tl.float32)
+    if not USE_FINAL_STATE and not USE_INITIAL_STATE:
+        for i_w in tl.static_range(0, W):
+            p_dy_blk = tl.make_block_ptr(p_dy, (T, D), (stride_dy_t, stride_dy_d),
+                                         (i_t * BT + i_w, i_d * BD), (BT, BD), (1, 0))
+            # [BT, BD]
+            b_dy = tl.load(p_dy_blk, boundary_check=(0, 1)).to(tl.float32)
+            if ACTIVATION == 'swish' or ACTIVATION == 'silu':
+                p_y = tl.make_block_ptr(y + bos * D, (T, D), (D, 1), (i_t * BT + i_w, i_d * BD), (BT, BD), (1, 0))
+                b_y = tl.load(p_y, boundary_check=(0, 1)).to(tl.float32)
+                b_ys = tl.sigmoid(b_y)
+                b_dy = b_dy * b_ys * (1 + b_y * (1 - b_ys))
+            b_wdy = b_dy
+            if HAS_WEIGHT:
+                # [BT, BD]
+                b_wdy = b_wdy * tl.sum(b_w * (o_w == (W - i_w - 1)), 1)
+                # [BD]
+                b_dw = tl.sum(b_dy * b_x, 0)
+                tl.store(dw + i_tg * D*W + o_d * W + W - i_w - 1, b_dw.to(dw.dtype.element_ty), mask=m_d)
+            if HAS_BIAS and i_w == 0:
+                b_db += tl.sum(b_dy, 0)
+            b_dx += b_wdy
+    elif i_t * BT >= W:
+        # to make Triton compiler happy, we need to copy codes
+        for i_w in tl.static_range(0, W):
+            p_dy_blk = tl.make_block_ptr(p_dy, (T, D), (stride_dy_t, stride_dy_d),
+                                         (i_t * BT + i_w, i_d * BD), (BT, BD), (1, 0))
+            # [BT, BD]
+            b_dy = tl.load(p_dy_blk, boundary_check=(0, 1)).to(tl.float32)
+            if ACTIVATION == 'swish' or ACTIVATION == 'silu':
+                p_y = tl.make_block_ptr(y + bos * D, (T, D), (D, 1), (i_t * BT + i_w, i_d * BD), (BT, BD), (1, 0))
+                b_y = tl.load(p_y, boundary_check=(0, 1)).to(tl.float32)
+                b_ys = tl.sigmoid(b_y)
+                b_dy = b_dy * b_ys * (1 + b_y * (1 - b_ys))
+            b_wdy = b_dy
+            if HAS_WEIGHT:
+                # [BT, BD]
+                b_wdy = b_wdy * tl.sum(b_w * (o_w == (W - i_w - 1)), 1)
+                # [BD]
+                b_dw = tl.sum(b_dy * b_x, 0)
+                tl.store(dw + i_tg * D*W + o_d * W + W - i_w - 1, b_dw.to(dw.dtype.element_ty), mask=m_d)
+            if HAS_BIAS and i_w == 0:
+                b_db += tl.sum(b_dy, 0)
+            b_dx += b_wdy
+    else:
+        # which may use initial state
+        o_t = i_t * BT + tl.arange(0, BT)
+        for i_w in tl.static_range(0, W):
+            p_dy_blk = tl.make_block_ptr(p_dy, (T, D), (stride_dy_t, stride_dy_d),
+                                         (i_t * BT + i_w, i_d * BD), (BT, BD), (1, 0))
+            b_dy_shift = tl.load(p_dy_blk, boundary_check=(0, 1)).to(tl.float32)
+            if ACTIVATION == 'swish' or ACTIVATION == 'silu':
+                p_y = tl.make_block_ptr(y + bos * D, (T, D), (D, 1), (i_t * BT + i_w, i_d * BD), (BT, BD), (1, 0))
+                b_y_shift = tl.load(p_y, boundary_check=(0, 1)).to(tl.float32)
+                b_ys = tl.sigmoid(b_y_shift)
+                b_dy_shift = b_dy_shift * b_ys * (1 + b_y_shift * (1 - b_ys))
+            if HAS_WEIGHT:
+                # gradient comes from x：sum_t dy[t+i_w] * x[t]
+                b_dw = tl.sum(b_dy_shift * b_x, 0)
+                # index of cache：c = W - i_w + t
+                if USE_INITIAL_STATE:
+                    mask_head_rows = (o_t < i_w) & (o_t < T)
+                    # dy_head = dy[t]
+                    b_dy_head = tl.load(p_dy + o_t[:, None] * stride_dy_t + o_d * stride_dy_d, mask=(mask_head_rows[:, None] & m_d[None, :]),
+                                        other=0.0).to(tl.float32)
+                    if ACTIVATION == 'swish' or ACTIVATION == 'silu':
+                        # use y[t] （not y[t+i_w]）
+                        b_y_head = tl.load(y + bos * D + o_t[:, None] * D + o_d,
+                                           mask=(mask_head_rows[:, None] & m_d[None, :]), other=0.0).to(tl.float32)
+                        b_ys_head = tl.sigmoid(b_y_head)
+                        b_dy_head = b_dy_head * b_ys_head * (1 + b_y_head * (1 - b_ys_head))
+                    o_c = W - i_w + o_t
+                    # index 0 is padding 0
+                    mask_c = (mask_head_rows & (o_c >= 1) & (o_c < W))
+                    b_xc = tl.load(initial_state + i_n * D * W + o_d[None, :] * W + o_c[:, None],
+                                   mask=(mask_c[:, None] & m_d[None, :]), other=0.0).to(tl.float32)
+                    # add the gradient comes from initial_state
+                    b_dw += tl.sum(b_dy_head * b_xc, 0)
+                tl.store(dw + i_tg * D * W + o_d * W + W - i_w - 1, b_dw.to(dw.dtype.element_ty), mask=m_d)
+            if HAS_BIAS and i_w == 0:
+                b_db += tl.sum(b_dy_shift, 0)
+            b_wdy = b_dy_shift if not HAS_WEIGHT else (b_dy_shift * tl.sum(b_w * (o_w == (W - i_w - 1)), 1))
+            b_dx += b_wdy
+    if HAS_BIAS:
+        b_db = tl.cast(b_db, dtype=db.dtype.element_ty, fp_downcast_rounding='rtne')
+        tl.store(db + i_tg * D + o_d, b_db, mask=m_d)
+    if USE_FINAL_STATE:
+        if i_t * BT + BT >= T-W:
+            start_tok = max(0, T - (W - 1))
+            offset = i_t * BT + tl.arange(0, BT)
+            tok_idx = offset - start_tok
+            mask = (offset >= start_tok) & (offset < T)
+            w_idx = 1 + tok_idx
+            dht_off = i_n * D * W + o_d[None, :] * W + w_idx[:, None]
+            b_dht = tl.load(dht + dht_off, mask=mask[:, None] & m_d[None, :], other=0.).to(tl.float32)
+            b_dx += b_dht
+    if IS_VARLEN:
+        p_dx = dx + bos * stride_dx_t
+    else:
+        p_dx = dx + tl.cast(i_b, tl.int64) * stride_dx_n
+    p_dx = tl.make_block_ptr(p_dx, (T, D), (stride_dx_t, stride_dx_d), (i_t * BT, i_d * BD), (BT, BD), (1, 0))
+    tl.store(p_dx, tl.cast(b_dx, dtype=p_dx.dtype.element_ty, fp_downcast_rounding='rtne'), boundary_check=(0, 1))
+@triton.heuristics({
+    'USE_INITIAL_STATE': lambda args: args['cache'] is not None,
+    'HAS_WEIGHT': lambda args: args['weight'] is not None,
+    'HAS_BIAS': lambda args: args['bias'] is not None,
+    'HAS_RESIDUAL': lambda args: args['residual'] is not None,
+})
+@fla_cache_autotune(
+    configs=[
+        triton.Config({'BD': BD}, num_warps=num_warps)
+        for BD in [8, 16, 32, 64, 128, 256]
+        for num_warps in NUM_WARPS_AUTOTUNE
+    ],
+    key=['D', 'W'],
+    restore_value=['cache'],
+    **autotune_cache_kwargs,
+)
+@triton.jit
+def causal_conv1d_update_kernel(
+    x,
+    cache,
+    residual,
+    y,
+    weight,
+    bias,
+    stride_x_n,  # batch stride
+    stride_x_d,  # dim stride
+    stride_y_n,  # batch stride
+    stride_y_d,  # dim stride
+    D: tl.constexpr,
+    W: tl.constexpr,
+    BD: tl.constexpr,
+    BW: tl.constexpr,
+    ACTIVATION: tl.constexpr,
+    USE_INITIAL_STATE: tl.constexpr,
+    HAS_WEIGHT: tl.constexpr,
+    HAS_BIAS: tl.constexpr,
+    HAS_RESIDUAL: tl.constexpr,
+):
+    i_d, i_n = tl.program_id(0), tl.program_id(1)
+    o_d = i_d * BD + tl.arange(0, BD)
+    o_w = tl.arange(0, BW)
+    m_d = o_d < D
+    m_w = o_w < W
+    # [BD]
+    b_x = tl.load(x + i_n * stride_x_n + o_d * stride_x_d, mask=m_d, other=0).to(tl.float32)
+    b_cache = tl.zeros((BD, BW), dtype=tl.float32)
+    if USE_INITIAL_STATE:
+        # 2. Shift Cache (Read [1:])
+        p_cache_read = tl.make_block_ptr(
+            cache + i_n * D*W,
+            shape=(D, W),
+            strides=(W, 1),
+            offsets=(i_d * BD, 1),
+            block_shape=(BD, BW),
+            order=(1, 0)
+        )
+        b_cache = tl.load(p_cache_read, boundary_check=(0, 1)).to(tl.float32)
+        # 3. Fill x to the last position
+        m_update = o_w == (W - 1)
+        b_cache = tl.where(m_update[None, :], b_x[:, None], b_cache)
+    if HAS_WEIGHT:
+        b_w = tl.load(weight + o_d[:, None] * W + o_w, mask=m_d[:, None] & m_w, other=0)
+        b_y = tl.sum(b_cache * b_w, 1)
+    else:
+        b_y = tl.sum(b_cache, 1)
+    if HAS_BIAS:
+        b_y += tl.load(bias + o_d, mask=m_d)
+    if ACTIVATION == 'swish' or ACTIVATION == 'silu':
+        b_y = b_y * tl.sigmoid(b_y)
+    if HAS_RESIDUAL:
+        b_y += tl.load(residual + i_n * D + o_d, mask=m_d, other=0)
+    tl.store(y + i_n * stride_y_n + o_d * stride_y_d, tl.cast(b_y,
+             dtype=y.dtype.element_ty, fp_downcast_rounding='rtne'), mask=m_d)
+    if USE_INITIAL_STATE:
+        p_cache_write = tl.make_block_ptr(
+            cache + i_n * D*W,
+            shape=(D, W),
+            strides=(W, 1),
+            offsets=(i_d * BD, 0),
+            block_shape=(BD, BW),
+            order=(1, 0)
+        )
+        tl.store(p_cache_write, tl.cast(b_cache, dtype=cache.dtype.element_ty,
+                 fp_downcast_rounding='rtne'), boundary_check=(0, 1))
+@triton.heuristics({
+    'USE_ACTIVATION': lambda args: args['y'] is not None,
+    'IS_VARLEN': lambda args: args['cu_seqlens'] is not None,
+})
+@triton.jit
+def compute_dh0_kernel(
+    dy,
+    y,
+    weight,
+    dh0,
+    cu_seqlens,
+    stride_dy_n,
+    stride_dy_t,
+    T,
+    D: tl.constexpr,
+    W: tl.constexpr,
+    BD: tl.constexpr,
+    USE_ACTIVATION: tl.constexpr,
+    IS_VARLEN: tl.constexpr,
+):
+    """
+    Compute dh0 (gradient w.r.t. initial_state) in a separate kernel.
+    This avoids Triton compiler bugs on some architectures (e.g., GB200).
+    Grid: (cdiv(D, BD), N)
+    """
+    i_d, i_n = tl.program_id(0), tl.program_id(1)
+    # Get sequence boundaries
+    if IS_VARLEN:
+        bos = tl.load(cu_seqlens + i_n).to(tl.int64)
+        eos = tl.load(cu_seqlens + i_n + 1).to(tl.int64)
+        seq_len = eos - bos
+        # For varlen, dy is [1, total_T, D], offset by bos
+        dy_base = dy + bos * stride_dy_t
+    else:
+        seq_len = T
+        # For non-varlen, dy is [B, T, D], offset by i_n * stride_dy_n
+        dy_base = dy + tl.cast(i_n, tl.int64) * stride_dy_n
+    o_d = i_d * BD + tl.arange(0, BD)
+    m_d = o_d < D
+    # For each i_w in [1, W), compute dh0[i_n, :, i_w]
+    for i_w in tl.static_range(1, W):
+        b_dh0 = tl.zeros([BD], dtype=tl.float32)
+        # Accumulate contributions from t = 0 to min(i_w, seq_len) - 1
+        for t in tl.static_range(0, W - 1):
+            if t < i_w:
+                w_idx = i_w - 1 - t
+                # Load dy[t, :] relative to dy_base
+                p_dy = dy_base + t * stride_dy_t + o_d
+                m_t = (t < seq_len) & m_d
+                b_dy = tl.load(p_dy, mask=m_t, other=0).to(tl.float32)
+                if USE_ACTIVATION:
+                    if IS_VARLEN:
+                        p_y = y + bos * stride_dy_t + t * stride_dy_t + o_d
+                    else:
+                        p_y = y + tl.cast(i_n, tl.int64) * stride_dy_n + t * stride_dy_t + o_d
+                    b_y = tl.load(p_y, mask=m_t, other=0).to(tl.float32)
+                    b_ys = tl.sigmoid(b_y)
+                    b_dy = b_dy * b_ys * (1 + b_y * (1 - b_ys))
+                # Get weight[:, w_idx]
+                b_w_col = tl.load(weight + o_d * W + w_idx, mask=m_d, other=0).to(tl.float32)
+                # Accumulate
+                b_dh0 += tl.where(m_t, b_dy * b_w_col, 0)
+        # Store dh0[i_n, :, i_w]
+        p_dh0 = dh0 + i_n * D * W + o_d * W + i_w
+        tl.store(p_dh0, b_dh0.to(dh0.dtype.element_ty), mask=m_d)
+@triton.heuristics({
+    'USE_INITIAL_STATE': lambda args: args['initial_state'] is not None,
+    'IS_VARLEN': lambda args: args['cu_seqlens'] is not None,
+})
+@triton.jit
+def causal_conv1d_states_fwd_kernel(
+    x,
+    initial_state,
+    final_state,
+    cu_seqlens,
+    T,
+    D,
+    W,
+    stride_x_n,
+    stride_x_t,
+    stride_x_d,
+    BD: tl.constexpr,
+    BW: tl.constexpr,
+    USE_INITIAL_STATE: tl.constexpr,
+    IS_VARLEN: tl.constexpr,
+):
+    i_d, i_n = tl.program_id(0), tl.program_id(1)
+    # o_d Shape: [BD]
+    o_d = i_d * BD + tl.arange(0, BD)
+    m_d = o_d < D
+    if IS_VARLEN:
+        bos = tl.load(cu_seqlens + i_n).to(tl.int64)
+        eos = tl.load(cu_seqlens + i_n + 1).to(tl.int64)
+        seq_len = (eos - bos).to(tl.int32)
+        p_x = x + bos * stride_x_t
+    else:
+        seq_len = T
+        p_x = x + tl.cast(i_n, tl.int64) * stride_x_n
+    p_x = tl.make_block_ptr(p_x, (seq_len, D), (stride_x_t, stride_x_d), (seq_len - BW, i_d * BD), (BW, BD), (1, 0))
+    # b_x Shape: [BW, BD]
+    b_x = tl.load(p_x, boundary_check=(0, 1), padding_option="zero").to(tl.float32)
+    if USE_INITIAL_STATE:
+        if seq_len < BW:
+            o_c = W - (BW - seq_len) + tl.arange(0, BW)
+            m_c = (o_c >= 0) & (o_c < W)
+            p_init = initial_state + i_n * D*W + o_d[None, :] * W + o_c[:, None]
+            mask_init = m_d[None, :] & m_c[:, None]
+            b_cache = tl.load(p_init, mask=mask_init, other=0)
+            b_x += b_cache
+    # final_state: [N, D, W] (Channel Major inside sample)
+    # o_w Shape: [BW]
+    o_w = W - BW + tl.arange(0, BW)
+    # o_d[:, None] -> [BD, 1]
+    # o_w[None, :] -> [1, BW]
+    # p_final Shape -> [BD, BW]
+    p_final = final_state + tl.cast(i_n, tl.int64) * D*W + o_d[:, None] * W + o_w[None, :]
+    # m_final Shape -> [BD, BW]
+    m_final = m_d[:, None] & (o_w[None, :] >= 0)
+    tl.store(p_final, tl.trans(b_x).to(final_state.dtype.element_ty), mask=m_final)
+@input_guard(no_guard_contiguous=["x"])
+def causal_conv1d_update_states(
+    x: torch.Tensor,
+    state_len: int,
+    initial_state: torch.Tensor | None = None,
+    cu_seqlens: torch.Tensor | None = None,
+) -> torch.Tensor:
+    if cu_seqlens is not None:
+        N = len(cu_seqlens) - 1
+        if x.dim() == 2:
+            stride_x_n = 0
+            stride_x_t, stride_x_d = x.stride()
+            T = x.shape[0]
+        else:
+            stride_x_n = x.stride(0)
+            stride_x_t, stride_x_d = x.stride(1), x.stride(2)
+            T = x.shape[1]
+        D = x.shape[-1]
+    else:
+        B, T, D = x.shape
+        N = B
+        stride_x_n, stride_x_t, stride_x_d = x.stride()
+    W = state_len
+    final_state = torch.empty(N, D, W, dtype=x.dtype, device=x.device)
+    BD = min(triton.next_power_of_2(D), 256)
+    BW = triton.next_power_of_2(W)
+    grid = (triton.cdiv(D, BD), N)
+    causal_conv1d_states_fwd_kernel[grid](
+        x=x,
+        initial_state=initial_state,
+        final_state=final_state,
+        cu_seqlens=cu_seqlens,
+        T=T,
+        D=D,
+        W=W,
+        stride_x_n=stride_x_n,
+        stride_x_t=stride_x_t,
+        stride_x_d=stride_x_d,
+        BW=BW,
+        BD=BD,
+    )
+    return final_state
+@input_guard(no_guard_contiguous=["x"])
+def causal_conv1d_update(
+    x: torch.Tensor,
+    cache: torch.Tensor,
+    residual: torch.Tensor | None = None,
+    weight: torch.Tensor | None = None,
+    bias: torch.Tensor | None = None,
+    activation: str | None = None,
+) -> torch.Tensor:
+    shape = x.shape
+    if weight is not None and x.shape[-1] != weight.shape[0]:
+        x = rearrange(x, 'b t ... -> b t (...)')
+    D = x.shape[-1]
+    N = x.numel() // D
+    W = weight.shape[1] if weight is not None else None
+    BW = triton.next_power_of_2(W)
+    if x.dim() == 2:
+        # Case: (N, D)
+        stride_x_n = x.stride(0)
+        stride_x_d = x.stride(1)
+    elif x.dim() == 3 and x.shape[0] == 1:
+        # Case: (1, N, D) -> Time=1, Batch=N, Dim=D
+        # Batch 在 dim 1
+        stride_x_n = x.stride(1)
+        stride_x_d = x.stride(2)
+    elif x.dim() == 3:
+        # Case: (N, 1, D) -> Batch=N, Time=1, Dim=D
+        # Batch 在 dim 0
+        stride_x_n = x.stride(0)
+        stride_x_d = x.stride(2)
+    else:
+        # Fallback / Error case
+        raise ValueError(f"Unsupported input shape: {x.shape}")
+    y = torch.empty_like(x, memory_format=torch.contiguous_format)
+    if y.dim() == 2:
+        stride_y_n, stride_y_d = y.stride(0), y.stride(1)
+    elif y.dim() == 3 and y.shape[0] == 1:
+        stride_y_n, stride_y_d = y.stride(1), y.stride(2)
+    elif y.dim() == 3:
+        stride_y_n, stride_y_d = y.stride(0), y.stride(2)
+    def grid(meta): return (triton.cdiv(D, meta['BD']), N)
+    causal_conv1d_update_kernel[grid](
+        x=x,
+        cache=cache,
+        residual=residual,
+        y=y,
+        weight=weight,
+        bias=bias,
+        stride_x_n=stride_x_n,
+        stride_x_d=stride_x_d,
+        stride_y_n=stride_y_n,
+        stride_y_d=stride_y_d,
+        D=D,
+        W=W,
+        BW=BW,
+        ACTIVATION=activation,
+    )
+    return y.view(shape), cache

build/torch-cuda/modules/conv/triton/ops.py ADDED Viewed

	@@ -0,0 +1,424 @@

+# Copyright (c) 2023-2026, Songlin Yang, Yu Zhang, Zhiyuan Li
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+# For a list of all contributors, visit:
+#   https://github.com/fla-org/flash-linear-attention/graphs/contributors
+import torch
+import triton
+from einops import rearrange
+from ....modules.backends import dispatch
+from ....ops.utils import prepare_chunk_indices
+from ....utils import input_guard
+from .kernels import (
+    causal_conv1d_bwd_kernel,
+    causal_conv1d_fwd_kernel,
+    causal_conv1d_states_fwd_kernel,
+    causal_conv1d_update_kernel,
+    compute_dh0_kernel,
+)
+def _has_non_standard_layout(x: torch.Tensor) -> bool:
+    """QKV-style views (stride_t != D) break triton-ascend masked kernels."""
+    if x.dtype not in (torch.float16, torch.bfloat16):
+        return False
+    if x.dim() != 3:
+        return not x.is_contiguous()
+    _, stride_t, stride_d = x.stride()
+    return stride_d == 1 and stride_t != x.shape[-1]
+@dispatch('modules')
+@input_guard(no_guard_contiguous=["x"])
+def causal_conv1d_fwd(
+    x: torch.Tensor,
+    weight: torch.Tensor,
+    bias: torch.Tensor,
+    residual: torch.Tensor,
+    initial_state: torch.Tensor | None = None,
+    output_final_state: bool = False,
+    activation: str | None = None,
+    cu_seqlens: torch.LongTensor | None = None,
+    cu_seqlens_cpu: torch.LongTensor | None = None,
+    chunk_indices: torch.LongTensor | None = None,
+    BT: int = 64,
+    layout_fallback: bool = False,
+) -> torch.Tensor:
+    shape = x.shape
+    if x.shape[-1] != weight.shape[0]:
+        x = rearrange(x, 'b t ... -> b t (...)')
+    B, T, D = x.shape[0], x.shape[1], weight.shape[0]
+    W = weight.shape[1]
+    stride_x_n, stride_x_t, stride_x_d = x.stride()
+    BW = triton.next_power_of_2(W)
+    if cu_seqlens is not None and chunk_indices is None:
+        chunk_indices = prepare_chunk_indices(cu_seqlens, BT, cu_seqlens_cpu=cu_seqlens_cpu)
+    NT = len(chunk_indices) if cu_seqlens is not None else triton.cdiv(T, BT)
+    NB = triton.cdiv(B*T, 1024)
+    y = torch.empty_like(x, memory_format=torch.contiguous_format)
+    def grid(meta): return (triton.cdiv(D, meta['BD']), NT, B)
+    causal_conv1d_fwd_kernel[grid](
+        x=x,
+        y=y,
+        weight=weight,
+        bias=bias,
+        residual=residual,
+        cu_seqlens=cu_seqlens,
+        initial_state=initial_state,
+        chunk_indices=chunk_indices,
+        B=B,
+        T=T,
+        D=D,
+        W=W,
+        BT=BT,
+        BW=BW,
+        NB=NB,
+        stride_x_n=stride_x_n,
+        stride_x_t=stride_x_t,
+        stride_x_d=stride_x_d,
+        ACTIVATION=activation,
+    )
+    final_state = None
+    if output_final_state:
+        final_state = causal_conv1d_update_states(
+            x=x,
+            state_len=W,
+            initial_state=initial_state,
+            cu_seqlens=cu_seqlens,
+        )
+    return y.view(shape), final_state
+@dispatch('modules')
+def compute_dh0_triton(
+    dy: torch.Tensor,
+    y: torch.Tensor | None,
+    weight: torch.Tensor,
+    initial_state: torch.Tensor,
+    activation: str | None,
+    cu_seqlens: torch.Tensor | None,
+) -> torch.Tensor:
+    """
+    Compute dh0 (gradient w.r.t. initial_state) using a separate Triton kernel.
+    This is a workaround for Triton compiler bugs on some architectures (e.g., GB200).
+    """
+    D, W = weight.shape
+    N = initial_state.shape[0]
+    T = dy.shape[1]
+    # Initialize dh0
+    dh0 = torch.zeros_like(initial_state)
+    BD = 32
+    grid = (triton.cdiv(D, BD), N)
+    y_to_pass = y if activation in ('swish', 'silu') else None
+    # dy is [B, T, D], stride_n = T*D, stride_t = D
+    stride_dy_n = dy.stride(0)
+    stride_dy_t = dy.stride(1)
+    compute_dh0_kernel[grid](
+        dy=dy,
+        y=y_to_pass,
+        weight=weight,
+        dh0=dh0,
+        cu_seqlens=cu_seqlens,
+        stride_dy_n=stride_dy_n,
+        stride_dy_t=stride_dy_t,
+        T=T,
+        D=D,
+        W=W,
+        BD=BD,
+    )
+    return dh0
+@dispatch('modules')
+def causal_conv1d_bwd(
+    x: torch.Tensor,
+    dy: torch.Tensor,
+    dht: torch.Tensor,
+    weight: torch.Tensor | None = None,
+    bias: torch.Tensor | None = None,
+    residual: torch.Tensor | None = None,
+    initial_state: torch.Tensor | None = None,
+    activation: str | None = None,
+    cu_seqlens: torch.Tensor | None = None,
+    cu_seqlens_cpu: torch.LongTensor | None = None,
+    chunk_indices: torch.LongTensor | None = None,
+    BT: int = 64,
+    layout_fallback: bool = False,
+):
+    shape = x.shape
+    if x.shape[-1] != weight.shape[0]:
+        x = rearrange(x, 'b t ... -> b t (...)')
+    B, T, D = x.shape
+    W = weight.shape[1] if weight is not None else None
+    stride_x_n, stride_x_t, stride_x_d = x.stride()
+    stride_dy_n, stride_dy_t, stride_dy_d = dy.stride()
+    BW = triton.next_power_of_2(W)
+    if cu_seqlens is not None and chunk_indices is None:
+        chunk_indices = prepare_chunk_indices(cu_seqlens, BT, cu_seqlens_cpu=cu_seqlens_cpu)
+    NT = len(chunk_indices) if cu_seqlens is not None else triton.cdiv(T, BT)
+    NB = triton.cdiv(B*T, 1024)
+    y = None
+    if activation is not None:
+        y, _ = causal_conv1d_fwd(
+            x=x,
+            weight=weight,
+            bias=bias,
+            residual=None,
+            initial_state=initial_state,
+            activation=None,
+            cu_seqlens=cu_seqlens,
+            cu_seqlens_cpu=cu_seqlens_cpu,
+            output_final_state=False,
+            chunk_indices=chunk_indices,
+        )
+    dx = torch.empty_like(x)
+    dw = weight.new_empty(B*NT, *weight.shape, dtype=torch.float) if weight is not None else None
+    db = bias.new_empty(B*NT, *bias.shape, dtype=torch.float) if bias is not None else None
+    dr = dy if residual is not None else None
+    stride_dx_n, stride_dx_t, stride_dx_d = dx.stride()
+    def grid(meta): return (triton.cdiv(D, meta['BD']), NT, B)
+    causal_conv1d_bwd_kernel[grid](
+        x=x,
+        y=y,
+        weight=weight,
+        initial_state=initial_state,
+        dht=dht,
+        dy=dy,
+        dx=dx,
+        dw=dw,
+        db=db,
+        cu_seqlens=cu_seqlens,
+        chunk_indices=chunk_indices,
+        B=B,
+        T=T,
+        D=D,
+        W=W,
+        BT=BT,
+        BW=BW,
+        NB=NB,
+        stride_x_n=stride_x_n,
+        stride_x_t=stride_x_t,
+        stride_x_d=stride_x_d,
+        stride_dx_n=stride_dx_n,
+        stride_dx_t=stride_dx_t,
+        stride_dx_d=stride_dx_d,
+        stride_dy_n=stride_dy_n,
+        stride_dy_t=stride_dy_t,
+        stride_dy_d=stride_dy_d,
+        ACTIVATION=activation,
+    )
+    if weight is not None:
+        dw = dw.sum(0).to(weight)
+    if bias is not None:
+        db = db.sum(0).to(bias)
+    # Compute dh0 using separate Triton kernel to avoid compiler bugs on some architectures (e.g., GB200)
+    dh0 = None
+    if initial_state is not None:
+        dh0 = compute_dh0_triton(
+            dy=dy,
+            y=y,
+            weight=weight,
+            initial_state=initial_state,
+            activation=activation,
+            cu_seqlens=cu_seqlens,
+        )
+    return dx.view(shape), dw, db, dr, dh0
+@dispatch('modules')
+@input_guard(no_guard_contiguous=["x"])
+def causal_conv1d_update_states(
+    x: torch.Tensor,
+    state_len: int,
+    initial_state: torch.Tensor | None = None,
+    cu_seqlens: torch.Tensor | None = None,
+) -> torch.Tensor:
+    if cu_seqlens is not None:
+        N = len(cu_seqlens) - 1
+        if x.dim() == 2:
+            stride_x_n = 0
+            stride_x_t, stride_x_d = x.stride()
+            T = x.shape[0]
+        else:
+            stride_x_n = x.stride(0)
+            stride_x_t, stride_x_d = x.stride(1), x.stride(2)
+            T = x.shape[1]
+        D = x.shape[-1]
+    else:
+        B, T, D = x.shape
+        N = B
+        stride_x_n, stride_x_t, stride_x_d = x.stride()
+    W = state_len
+    final_state = torch.empty(N, D, W, dtype=x.dtype, device=x.device)
+    BD = min(triton.next_power_of_2(D), 256)
+    BW = triton.next_power_of_2(W)
+    grid = (triton.cdiv(D, BD), N)
+    causal_conv1d_states_fwd_kernel[grid](
+        x=x,
+        initial_state=initial_state,
+        final_state=final_state,
+        cu_seqlens=cu_seqlens,
+        T=T,
+        D=D,
+        W=W,
+        stride_x_n=stride_x_n,
+        stride_x_t=stride_x_t,
+        stride_x_d=stride_x_d,
+        BW=BW,
+        BD=BD,
+    )
+    return final_state
+@dispatch('modules')
+@input_guard(no_guard_contiguous=["x"])
+def causal_conv1d_update(
+    x: torch.Tensor,
+    cache: torch.Tensor,
+    residual: torch.Tensor | None = None,
+    weight: torch.Tensor | None = None,
+    bias: torch.Tensor | None = None,
+    activation: str | None = None,
+) -> torch.Tensor:
+    shape = x.shape
+    if weight is not None and x.shape[-1] != weight.shape[0]:
+        x = rearrange(x, 'b t ... -> b t (...)')
+    D = x.shape[-1]
+    N = x.numel() // D
+    W = weight.shape[1] if weight is not None else None
+    BW = triton.next_power_of_2(W)
+    if x.dim() == 2:
+        # Case: (N, D)
+        stride_x_n = x.stride(0)
+        stride_x_d = x.stride(1)
+    elif x.dim() == 3 and x.shape[0] == 1:
+        # Case: (1, N, D) -> Time=1, Batch=N, Dim=D
+        # Batch 在 dim 1
+        stride_x_n = x.stride(1)
+        stride_x_d = x.stride(2)
+    elif x.dim() == 3:
+        # Case: (N, 1, D) -> Batch=N, Time=1, Dim=D
+        # Batch 在 dim 0
+        stride_x_n = x.stride(0)
+        stride_x_d = x.stride(2)
+    else:
+        # Fallback / Error case
+        raise ValueError(f"Unsupported input shape: {x.shape}")
+    y = torch.empty_like(x, memory_format=torch.contiguous_format)
+    if y.dim() == 2:
+        stride_y_n, stride_y_d = y.stride(0), y.stride(1)
+    elif y.dim() == 3 and y.shape[0] == 1:
+        stride_y_n, stride_y_d = y.stride(1), y.stride(2)
+    elif y.dim() == 3:
+        stride_y_n, stride_y_d = y.stride(0), y.stride(2)
+    def grid(meta): return (triton.cdiv(D, meta['BD']), N)
+    causal_conv1d_update_kernel[grid](
+        x=x,
+        cache=cache,
+        residual=residual,
+        y=y,
+        weight=weight,
+        bias=bias,
+        stride_x_n=stride_x_n,
+        stride_x_d=stride_x_d,
+        stride_y_n=stride_y_n,
+        stride_y_d=stride_y_d,
+        D=D,
+        W=W,
+        BW=BW,
+        ACTIVATION=activation,
+    )
+    return y.view(shape), cache
+class CausalConv1dFunction(torch.autograd.Function):
+    @staticmethod
+    @input_guard(no_guard_contiguous=["x"])
+    def forward(
+        ctx,
+        x: torch.Tensor,
+        weight: torch.Tensor | None = None,
+        bias: torch.Tensor | None = None,
+        residual: torch.Tensor | None = None,
+        initial_state: torch.Tensor | None = None,
+        output_final_state: bool | None = False,
+        activation: str | None = None,
+        cu_seqlens: torch.Tensor | None = None,
+        cu_seqlens_cpu: torch.LongTensor | None = None,
+        chunk_indices: torch.LongTensor | None = None,
+        chunk_size: int = 64,
+    ):
+        BT = chunk_size
+        if cu_seqlens is not None and chunk_indices is None:
+            chunk_indices = prepare_chunk_indices(cu_seqlens, BT, cu_seqlens_cpu=cu_seqlens_cpu)
+        ctx.activation = activation
+        ctx.cu_seqlens = cu_seqlens
+        ctx.cu_seqlens_cpu = cu_seqlens_cpu
+        ctx.chunk_indices = chunk_indices
+        ctx.layout_fallback = _has_non_standard_layout(x)
+        ctx.save_for_backward(x, weight, bias, residual, initial_state)
+        y, final_state = causal_conv1d_fwd(
+            x=x,
+            weight=weight,
+            bias=bias,
+            residual=residual,
+            initial_state=initial_state,
+            output_final_state=output_final_state,
+            activation=activation,
+            cu_seqlens=cu_seqlens,
+            cu_seqlens_cpu=cu_seqlens_cpu,
+            chunk_indices=chunk_indices,
+            BT=BT,
+            layout_fallback=ctx.layout_fallback,
+        )
+        return y, final_state
+    @staticmethod
+    @input_guard(no_guard_contiguous=["dy"])
+    def backward(ctx, dy: torch.Tensor, dht: torch.Tensor | None = None):
+        x, weight, bias, residual, initial_state = ctx.saved_tensors
+        dx, dw, db, dr, dh0 = causal_conv1d_bwd(
+            x=x,
+            dy=dy,
+            dht=dht,
+            weight=weight,
+            bias=bias,
+            residual=residual,
+            initial_state=initial_state,
+            activation=ctx.activation,
+            cu_seqlens=ctx.cu_seqlens,
+            cu_seqlens_cpu=ctx.cu_seqlens_cpu,
+            chunk_indices=ctx.chunk_indices,
+            layout_fallback=ctx.layout_fallback,
+        )
+        return dx, dw, db, dr, dh0, None, None, None, None, None, None

build/torch-cuda/modules/convolution.py ADDED Viewed

	@@ -0,0 +1,42 @@

+# Copyright (c) 2023-2026, Songlin Yang, Yu Zhang, Zhiyuan Li
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+# For a list of all contributors, visit:
+#   https://github.com/fla-org/flash-linear-attention/graphs/contributors
+from ..modules.conv import (
+    ImplicitLongConvolution,
+    LongConvolution,
+    PositionalEmbedding,
+    ShortConvolution,
+    causal_conv1d,
+    fft_conv,
+)
+from ..modules.conv.cp import CausalConv1dFunctionCP, causal_conv1d_cp
+from ..modules.conv.cuda import FastCausalConv1dFn, fast_causal_conv1d_fn
+from ..modules.conv.triton import (
+    CausalConv1dFunction,
+    causal_conv1d_bwd,
+    causal_conv1d_fwd,
+    causal_conv1d_update,
+    causal_conv1d_update_states,
+)
+__all__ = [
+    'CausalConv1dFunction',
+    'CausalConv1dFunctionCP',
+    'FastCausalConv1dFn',
+    'ImplicitLongConvolution',
+    'LongConvolution',
+    'PositionalEmbedding',
+    'ShortConvolution',
+    'causal_conv1d',
+    'causal_conv1d_bwd',
+    'causal_conv1d_cp',
+    'causal_conv1d_fwd',
+    'causal_conv1d_update',
+    'causal_conv1d_update_states',
+    'fast_causal_conv1d_fn',
+    'fft_conv',
+]

build/torch-cuda/modules/feature_map.py ADDED Viewed

	@@ -0,0 +1,315 @@

+# Copyright (c) 2023-2026, Songlin Yang, Yu Zhang, Zhiyuan Li
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+# For a list of all contributors, visit:
+#   https://github.com/fla-org/flash-linear-attention/graphs/contributors
+from __future__ import annotations
+import functools
+import math
+import torch
+import torch.nn.functional as F
+from torch import nn
+from ..modules.activations import fast_gelu_impl, sigmoid, sqrelu, swish
+from ..modules.layernorm import layer_norm
+from ..utils import checkpoint
+@functools.cache
+def _triu_indices(n: int, offset: int, device: torch.device) -> torch.Tensor:
+    # cache the upper-triangular gather indices per (size, offset, device) to avoid rebuilding
+    # them and copying host -> device on every forward
+    return torch.triu_indices(n, n, offset, device=device)
+@checkpoint
+def flatten_diag_outer_product(x: torch.Tensor, y: torch.Tensor) -> torch.Tensor:
+    z = torch.einsum("...i,...j->...ij", x, y)
+    N = z.size(-1)
+    indices = _triu_indices(N, 0, z.device)
+    return z[..., indices[0], indices[1]]
+@checkpoint
+def flatten_diag_outer_product_off1(x: torch.Tensor, y: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
+    z = torch.einsum("...i,...j->...ij", x, y)
+    N = z.size(-1)
+    indices = _triu_indices(N, 1, z.device)
+    diag = torch.arange(N, device=z.device)
+    return z[..., indices[0], indices[1]], z[..., diag, diag]
+def is_power_of_2(n: int) -> bool:
+    return (n & (n - 1) == 0) and n != 0
+class HedgehogFeatureMap(nn.Module):
+    r"""
+    Hedgehog feature map as introduced in
+    `The Hedgehog & the Porcupine: Expressive Linear Attentions with Softmax Mimicry <https://arxiv.org/abs/2402.04347>`_
+    """
+    def __init__(
+        self,
+        head_dim: int,
+    ) -> None:
+        super().__init__()
+        # Trainable map
+        self.layer = nn.Linear(head_dim, head_dim)
+        self.init_weights_()
+    def init_weights_(self):
+        """Initialize trainable map as identity"""
+        nn.init.eye_(self.layer.weight)
+        nn.init.zeros_(self.layer.bias)
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        x = self.layer(x)  # b, h, l, d
+        y = 2 * x
+        return torch.cat([y, -y], dim=-1).softmax(-1)
+class T2RFeatureMap(nn.Module):
+    r"""
+    Simple linear mapping feature map as in
+    `Finetuning Pretrained Transformers into RNNs <https://arxiv.org/abs/2103.13076>`_
+    """
+    def __init__(
+        self,
+        head_dim: int,
+        dot_dim: int | None = None,
+        bias: bool | None = False,
+    ) -> None:
+        super().__init__()
+        # Trainable map
+        if dot_dim is None:
+            dot_dim = head_dim
+        self.head_dim = head_dim
+        self.dot_dim = dot_dim
+        self.bias = bias
+        self.layer = nn.Linear(head_dim, dot_dim, bias=bias)
+    def __repr__(self) -> str:
+        return f"{self.__class__.__name__}(head_dim={self.head_dim}, dot_dim={self.dot_dim}, bias={self.bias})"
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        return self.layer(x).relu()
+class DPFPFeatureMap(nn.Module):
+    r"""
+    Deterministic Parameter-Free Projection (DPFP) feature map in
+    `Linear Transformers Are Secretly Fast Weight Programmers <https://arxiv.org/abs/2102.11174>`_
+    """
+    def __init__(
+        self,
+        head_dim: int,
+        nu: int = 4,
+    ) -> None:
+        super().__init__()
+        self.nu = nu
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        relu = x.relu()
+        x = torch.cat([relu, -relu], dim=-1)
+        x_rolled = torch.cat([x.roll(shifts=j, dims=-1) for j in range(1, self.nu+1)], dim=-1)
+        x_repeat = torch.cat([x] * self.nu, dim=-1)
+        return x_repeat * x_rolled
+class HadamardFeatureMap(nn.Module):
+    def __init__(
+        self,
+        head_dim: int,
+    ) -> None:
+        super().__init__()
+        # Trainable map
+        self.layer1 = nn.Linear(head_dim, head_dim)
+        self.layer2 = nn.Linear(head_dim, head_dim)
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        return self.layer1(x) * self.layer2(x)
+class LearnableOuterProductFeatureMap(nn.Module):
+    def __init__(
+        self,
+        head_dim: int,
+        feature_dim: int,
+    ) -> None:
+        super().__init__()
+        # Trainable map
+        self.layer1 = nn.Linear(head_dim, feature_dim, bias=False)
+        self.layer2 = nn.Linear(head_dim, feature_dim, bias=False)
+        self.normalizer = feature_dim ** -0.5
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        return flatten_diag_outer_product(self.layer1(x), self.layer2(x))
+class LearnablePolySketchNonNegativeFeatureMap(nn.Module):
+    def __init__(
+        self,
+        head_dim: int,
+        sketch_size: int | None = None,
+        degree: int | None = 2,
+    ) -> None:
+        super().__init__()
+        assert is_power_of_2(degree) and degree >= 2, f"The degree {degree} must be a power of 2"
+        if sketch_size is None:
+            sketch_size = head_dim
+        self.head_dim = head_dim
+        self.sketch_size = sketch_size
+        self.degree = degree
+        self.gamma = nn.Parameter(torch.ones(head_dim))
+        self.beta = nn.Parameter(torch.zeros(head_dim))
+        # NOTE: the sketch layers defined here are quite different from the original paper
+        # currently we simply use linear layers without any non-linear activations
+        self.sketches1 = nn.ModuleList([
+            nn.Linear(head_dim, sketch_size, bias=False),
+            *[nn.Linear(sketch_size, sketch_size, bias=False) for _ in range(int(math.log2(self.degree)) - 2)],
+        ])
+        self.sketches2 = nn.ModuleList([
+            nn.Linear(head_dim, sketch_size, bias=False),
+            *[nn.Linear(sketch_size, sketch_size, bias=False) for _ in range(int(math.log2(self.degree)) - 2)],
+        ])
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        # Section 2.1
+        x = layer_norm(x, self.gamma, self.beta)
+        # first map the input to sketch size with learnable parameters
+        x = self.sketches1[0](x) * self.sketches2[0](x) * self.head_dim ** -0.5
+        for i in range(1, int(math.log2(self.degree)) - 1):
+            x = self.sketches1[i](x) * self.sketches2[i](x) * self.head_dim ** -0.5
+        # do sketch mapping for log2(p) - 1 times in total
+        # do p=2 mapping to ensure non-negativity
+        return flatten_diag_outer_product(x, x)
+class TaylorFeatureMap(nn.Module):
+    def __init__(
+        self,
+        head_dim: int,
+    ) -> None:
+        super().__init__()
+        self.head_dim = head_dim
+        self.r2 = math.sqrt(2)
+        self.rd = math.sqrt(self.head_dim)
+        self.rrd = math.sqrt(self.rd)
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        x2_1, x2_2 = flatten_diag_outer_product_off1(x, x)
+        return torch.cat([torch.ones_like(x[..., 0:1]), x / self.rrd, x2_2 / (self.rd * self.r2), x2_1 / self.rd], dim=-1)
+class RebasedFeatureMap(nn.Module):
+    def __init__(
+        self,
+        head_dim: int,
+        use_gamma: bool | None = True,
+        use_beta: bool | None = True,
+        normalize: bool | None = True,
+    ) -> None:
+        super().__init__()
+        self.head_dim = head_dim
+        self.use_gamma = use_gamma
+        self.use_beta = use_beta
+        self.normalize = normalize
+        self.gamma = None
+        self.beta = None
+        if use_gamma:
+            self.gamma = nn.Parameter(torch.ones(head_dim))
+        if use_beta:
+            self.beta = nn.Parameter(torch.zeros(head_dim))
+    def forward(self, x: torch.Tensor, flatten: bool | None = True) -> torch.Tensor:
+        if self.use_beta and self.use_gamma and self.normalize:
+            x = layer_norm(x, self.gamma, self.beta)
+        elif self.normalize:
+            x = F.layer_norm(x, (self.head_dim,), self.gamma, self.beta)
+        elif self.use_gamma and self.use_beta:
+            x = torch.addcmul(self.beta, x, self.gamma)
+        elif self.use_gamma:
+            x = x.mul(self.gamma)
+        else:
+            raise RuntimeError(f"Not supported combination of `use_gamma`, `use_beta` and `normalize`, "
+                               f"which is currently set as (`{self.use_gamma}`, `{self.use_beta}`, `{self.normalize}`)")
+        if not flatten:
+            return x
+        x2_1, x2_2 = flatten_diag_outer_product_off1(x, x)
+        # rebased use learnable parameters to approximate any quadratic function
+        return torch.cat([x2_2 * self.head_dim ** -0.5, x2_1 * (2 / self.head_dim) ** 0.5], dim=-1)
+class ReLUFeatureMap(nn.Module):
+    def __init__(
+        self,
+    ) -> None:
+        super().__init__()
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        return F.relu(x)
+class SquaredReLUFeatureMap(nn.Module):
+    def __init__(
+        self,
+    ) -> None:
+        super().__init__()
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        return sqrelu(x)
+class GELUFeatureMap(nn.Module):
+    def __init__(
+        self,
+    ) -> None:
+        super().__init__()
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        return fast_gelu_impl(x)
+class SwishFeatureMap(nn.Module):
+    def __init__(
+        self,
+    ) -> None:
+        super().__init__()
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        return swish(x)
+class SigmoidFeatureMap(nn.Module):
+    def __init__(
+        self,
+    ) -> None:
+        super().__init__()
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        return sigmoid(x)

build/torch-cuda/modules/fused_bitlinear.py ADDED Viewed

	@@ -0,0 +1,638 @@

+# Copyright (c) 2023-2026, Songlin Yang, Yu Zhang, Zhiyuan Li
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+# For a list of all contributors, visit:
+#   https://github.com/fla-org/flash-linear-attention/graphs/contributors
+# Implementations of BitLinear layer with fused LayerNorm and quantized Linear layer.
+# [The Era of 1-bit LLMs: All Large Language Models are in 1.58 Bits](https://arxiv.org/abs/2402.17764)
+# [Scalable MatMul-free Language Modeling](https://arxiv.org/abs/2406.02528)
+#
+# Code adapted from https://github.com/ridgerchu/matmulfreellm/
+from __future__ import annotations
+import math
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+import triton
+import triton.language as tl
+from ..modules.layernorm import RMSNorm
+from ..utils import IS_AMD, autotune_cache_kwargs, get_multiprocessor_count, input_guard, require_version
+NUM_WARPS_AUTOTUNE = [1, 2, 4, 8, 16] if IS_AMD else [1, 2, 4, 8, 16, 32]
+def activation_quant(x):
+    """
+    Per-token quantization to 8 bits. No grouping is needed for quantization.
+    Args:
+        x: An activation tensor with shape [n, d].
+    Returns:
+        A quantized activation tensor with shape [n, d].
+    """
+    # Compute the scale factor
+    scale = 127.0 / x.abs().max(dim=-1, keepdim=True).values.clamp_(min=1e-5)
+    # Quantize and then de-quantize the tensor
+    y = (x * scale).round().clamp_(-128, 127) / scale
+    return y
+def weight_quant(w):
+    """
+    Per-tensor quantization to 1.58 bits. No grouping is needed for quantization.
+    Args:
+        w: A weight tensor with shape [d, k].
+    Returns:
+        A quantized weight tensor with shape [d, k].
+    """
+    # Compute the scale factor
+    scale = 1.0 / w.abs().mean().clamp_(min=1e-5)
+    # Quantize and then de-quantize the tensor
+    u = (w * scale).round().clamp_(-1, 1) / scale
+    return u
+@triton.autotune(
+    configs=[
+        triton.Config({}, num_warps=num_warps)
+        for num_warps in NUM_WARPS_AUTOTUNE
+    ],
+    key=["N", "HAS_RESIDUAL", "STORE_RESIDUAL_OUT", "IS_RMS_NORM", "HAS_BIAS"],
+    **autotune_cache_kwargs,
+)
+@triton.jit
+def layer_norm_fwd_kernel_quant(
+    X,  # pointer to the input
+    Y,  # pointer to the output
+    W,  # pointer to the weights
+    B,  # pointer to the biases
+    RESIDUAL,  # pointer to the residual
+    RESIDUAL_OUT,  # pointer to the residual
+    Mean,  # pointer to the mean
+    Rstd,  # pointer to the 1/std
+    stride_x_row,  # how much to increase the pointer when moving by 1 row
+    stride_y_row,
+    stride_res_row,
+    stride_res_out_row,
+    N,  # number of columns in X
+    eps,  # epsilon to avoid division by zero
+    IS_RMS_NORM: tl.constexpr,
+    BLOCK_N: tl.constexpr,
+    HAS_RESIDUAL: tl.constexpr,
+    STORE_RESIDUAL_OUT: tl.constexpr,
+    HAS_WEIGHT: tl.constexpr,
+    HAS_BIAS: tl.constexpr,
+):
+    # Map the program id to the row of X and Y it should compute.
+    row = tl.program_id(0)
+    X += row * stride_x_row
+    Y += row * stride_y_row
+    if HAS_RESIDUAL:
+        RESIDUAL += row * stride_res_row
+    if STORE_RESIDUAL_OUT:
+        RESIDUAL_OUT += row * stride_res_out_row
+    # Compute mean and variance
+    cols = tl.arange(0, BLOCK_N)
+    x = tl.load(X + cols, mask=cols < N, other=0.0).to(tl.float32)
+    if HAS_RESIDUAL:
+        residual = tl.load(RESIDUAL + cols, mask=cols < N, other=0.0).to(tl.float32)
+        x += residual
+    if STORE_RESIDUAL_OUT:
+        tl.store(RESIDUAL_OUT + cols, x, mask=cols < N)
+    if not IS_RMS_NORM:
+        mean = tl.sum(x, axis=0) / N
+        tl.store(Mean + row, mean)
+        xbar = tl.where(cols < N, x - mean, 0.0)
+        var = tl.sum(xbar * xbar, axis=0) / N
+    else:
+        xbar = tl.where(cols < N, x, 0.0)
+        var = tl.sum(xbar * xbar, axis=0) / N
+    rstd = 1 / tl.sqrt(var + eps)
+    tl.store(Rstd + row, rstd)
+    # Normalize and apply linear transformation
+    mask = cols < N
+    if HAS_WEIGHT:
+        w = tl.load(W + cols, mask=mask).to(tl.float32)
+    if HAS_BIAS:
+        b = tl.load(B + cols, mask=mask).to(tl.float32)
+    x_hat = (x - mean) * rstd if not IS_RMS_NORM else x * rstd
+    y = x_hat * w if HAS_WEIGHT else x_hat
+    if HAS_BIAS:
+        y = y + b
+    # Aply quantization to the output
+    scale = 127.0 / tl.maximum(tl.max(tl.abs(y), 0), 1e-5)
+    # Quantize and then de-quantize the tensor
+    y = tl.extra.cuda.libdevice.round(y * scale)
+    y = tl.maximum(tl.minimum(y, 127), -128) / scale
+    # Write output
+    tl.store(Y + cols, y, mask=mask)
+def layer_norm_fwd_quant(
+    x: torch.Tensor,
+    weight: torch.Tensor,
+    bias: torch.Tensor,
+    eps: float,
+    residual: torch.Tensor = None,
+    out_dtype: torch.dtype = None,
+    residual_dtype: torch.dtype = None,
+    is_rms_norm: bool = False,
+):
+    if residual is not None:
+        residual_dtype = residual.dtype
+    M, N = x.shape
+    # allocate output
+    y = torch.empty_like(x, dtype=x.dtype if out_dtype is None else out_dtype)
+    if residual is not None or (residual_dtype is not None and residual_dtype != x.dtype):
+        residual_out = torch.empty(M, N, device=x.device, dtype=residual_dtype)
+    else:
+        residual_out = None
+    mean = torch.empty((M,), dtype=torch.float32, device=x.device) if not is_rms_norm else None
+    rstd = torch.empty((M,), dtype=torch.float32, device=x.device)
+    # Less than 64KB per feature: enqueue fused kernel
+    MAX_FUSED_SIZE = 65536 // x.element_size()
+    BLOCK_N = min(MAX_FUSED_SIZE, triton.next_power_of_2(N))
+    if N > BLOCK_N:
+        raise RuntimeError("This layer norm doesn't support feature dim >= 64KB.")
+    # heuristics for number of warps
+    layer_norm_fwd_kernel_quant[(M,)](
+        x,
+        y,
+        weight,
+        bias,
+        residual,
+        residual_out,
+        mean,
+        rstd,
+        x.stride(0),
+        y.stride(0),
+        residual.stride(0) if residual is not None else 0,
+        residual_out.stride(0) if residual_out is not None else 0,
+        N,
+        eps,
+        is_rms_norm,
+        BLOCK_N,
+        residual is not None,
+        residual_out is not None,
+        weight is not None,
+        bias is not None,
+    )
+    # residual_out is None if residual is None and residual_dtype == input_dtype
+    return y, mean, rstd, residual_out if residual_out is not None else x
+@triton.heuristics({
+    "RECOMPUTE_OUTPUT": lambda args: args["Y"] is not None,
+})
+@triton.autotune(
+    configs=[
+        triton.Config({}, num_warps=num_warps)
+        for num_warps in NUM_WARPS_AUTOTUNE
+    ],
+    key=["N", "HAS_DRESIDUAL", "STORE_DRESIDUAL", "IS_RMS_NORM", "HAS_BIAS"],
+    **autotune_cache_kwargs,
+)
+@triton.jit
+def layer_norm_bwd_kernel(
+    X,  # pointer to the input
+    W,  # pointer to the weights
+    B,  # pointer to the biases
+    Y,  # pointer to the output to be recomputed
+    DY,  # pointer to the output gradient
+    DX,  # pointer to the input gradient
+    DW,  # pointer to the partial sum of weights gradient
+    DB,  # pointer to the partial sum of biases gradient
+    DRESIDUAL,
+    DRESIDUAL_IN,
+    Mean,  # pointer to the mean
+    Rstd,  # pointer to the 1/std
+    stride_x_row,  # how much to increase the pointer when moving by 1 row
+    stride_y_row,
+    stride_dy_row,
+    stride_dx_row,
+    stride_dres_row,
+    stride_dres_in_row,
+    M,  # number of rows in X
+    N,  # number of columns in X
+    eps,  # epsilon to avoid division by zero
+    rows_per_program,
+    IS_RMS_NORM: tl.constexpr,
+    BLOCK_N: tl.constexpr,
+    HAS_DRESIDUAL: tl.constexpr,
+    STORE_DRESIDUAL: tl.constexpr,
+    HAS_WEIGHT: tl.constexpr,
+    HAS_BIAS: tl.constexpr,
+    RECOMPUTE_OUTPUT: tl.constexpr,
+):
+    # Map the program id to the elements of X, DX, and DY it should compute.
+    row_block_id = tl.program_id(0)
+    row_start = row_block_id * rows_per_program
+    cols = tl.arange(0, BLOCK_N)
+    mask = cols < N
+    X += row_start * stride_x_row
+    if HAS_DRESIDUAL:
+        DRESIDUAL += row_start * stride_dres_row
+    if STORE_DRESIDUAL:
+        DRESIDUAL_IN += row_start * stride_dres_in_row
+    DY += row_start * stride_dy_row
+    DX += row_start * stride_dx_row
+    if RECOMPUTE_OUTPUT:
+        Y += row_start * stride_y_row
+    if HAS_WEIGHT:
+        w = tl.load(W + cols, mask=mask).to(tl.float32)
+        dw = tl.zeros((BLOCK_N,), dtype=tl.float32)
+    if RECOMPUTE_OUTPUT and HAS_BIAS:
+        b = tl.load(B + cols, mask=mask, other=0.0).to(tl.float32)
+    if HAS_BIAS:
+        db = tl.zeros((BLOCK_N,), dtype=tl.float32)
+    row_end = min((row_block_id + 1) * rows_per_program, M)
+    for row in range(row_start, row_end):
+        # Load data to SRAM
+        x = tl.load(X + cols, mask=mask, other=0).to(tl.float32)
+        dy = tl.load(DY + cols, mask=mask, other=0).to(tl.float32)
+        if not IS_RMS_NORM:
+            mean = tl.load(Mean + row)
+        rstd = tl.load(Rstd + row)
+        # Compute dx
+        xhat = (x - mean) * rstd if not IS_RMS_NORM else x * rstd
+        xhat = tl.where(mask, xhat, 0.0)
+        if RECOMPUTE_OUTPUT:
+            y = xhat * w if HAS_WEIGHT else xhat
+            if HAS_BIAS:
+                y = y + b
+            # Aply quantization to the output
+            scale = 127.0 / tl.maximum(tl.max(tl.abs(y), 0), 1e-5)
+            # Quantize and then de-quantize the tensor
+            y = tl.extra.cuda.libdevice.round(y * scale)
+            y = tl.maximum(tl.minimum(y, 127), -128) / scale
+            tl.store(Y + cols, y, mask=mask)
+        wdy = dy
+        if HAS_WEIGHT:
+            wdy = dy * w
+            dw += dy * xhat
+        if HAS_BIAS:
+            db += dy
+        if not IS_RMS_NORM:
+            c1 = tl.sum(xhat * wdy, axis=0) / N
+            c2 = tl.sum(wdy, axis=0) / N
+            dx = (wdy - (xhat * c1 + c2)) * rstd
+        else:
+            c1 = tl.sum(xhat * wdy, axis=0) / N
+            dx = (wdy - xhat * c1) * rstd
+        if HAS_DRESIDUAL:
+            dres = tl.load(DRESIDUAL + cols, mask=mask, other=0).to(tl.float32)
+            dx += dres
+        # Write dx
+        if STORE_DRESIDUAL:
+            tl.store(DRESIDUAL_IN + cols, dx, mask=mask)
+        tl.store(DX + cols, dx, mask=mask)
+        X += stride_x_row
+        if HAS_DRESIDUAL:
+            DRESIDUAL += stride_dres_row
+        if STORE_DRESIDUAL:
+            DRESIDUAL_IN += stride_dres_in_row
+        if RECOMPUTE_OUTPUT:
+            Y += stride_y_row
+        DY += stride_dy_row
+        DX += stride_dx_row
+    if HAS_WEIGHT:
+        tl.store(DW + row_block_id * N + cols, dw, mask=mask)
+    if HAS_BIAS:
+        tl.store(DB + row_block_id * N + cols, db, mask=mask)
+def layer_norm_bwd(
+    dy: torch.Tensor,
+    x: torch.Tensor,
+    weight: torch.Tensor,
+    bias: torch.Tensor,
+    eps: float,
+    mean: torch.Tensor,
+    rstd: torch.Tensor,
+    dresidual: torch.Tensor = None,
+    has_residual: bool = False,
+    is_rms_norm: bool = False,
+    x_dtype: torch.dtype = None,
+    recompute_output: bool = False,
+):
+    M, N = x.shape
+    # allocate output
+    dx = torch.empty_like(x) if x_dtype is None else torch.empty(M, N, dtype=x_dtype, device=x.device)
+    dresidual_in = torch.empty_like(x) if has_residual and dx.dtype != x.dtype else None
+    y = torch.empty(M, N, dtype=dy.dtype, device=dy.device) if recompute_output else None
+    # Less than 64KB per feature: enqueue fused kernel
+    MAX_FUSED_SIZE = 65536 // x.element_size()
+    BLOCK_N = min(MAX_FUSED_SIZE, triton.next_power_of_2(N))
+    if N > BLOCK_N:
+        raise RuntimeError("This layer norm doesn't support feature dim >= 64KB.")
+    sm_count = get_multiprocessor_count(x.device.index)
+    _dw = torch.empty((sm_count, N), dtype=torch.float32, device=weight.device) if weight is not None else None
+    _db = torch.empty((sm_count, N), dtype=torch.float32, device=bias.device) if bias is not None else None
+    rows_per_program = math.ceil(M / sm_count)
+    grid = (sm_count,)
+    layer_norm_bwd_kernel[grid](
+        x,
+        weight,
+        bias,
+        y,
+        dy,
+        dx,
+        _dw,
+        _db,
+        dresidual,
+        dresidual_in,
+        mean,
+        rstd,
+        x.stride(0),
+        0 if not recompute_output else y.stride(0),
+        dy.stride(0),
+        dx.stride(0),
+        dresidual.stride(0) if dresidual is not None else 0,
+        dresidual_in.stride(0) if dresidual_in is not None else 0,
+        M,
+        N,
+        eps,
+        rows_per_program,
+        is_rms_norm,
+        BLOCK_N,
+        dresidual is not None,
+        dresidual_in is not None,
+        weight is not None,
+        bias is not None,
+    )
+    dw = _dw.sum(0).to(weight.dtype) if weight is not None else None
+    db = _db.sum(0).to(bias.dtype) if bias is not None else None
+    # Don't need to compute dresidual_in separately in this case
+    if has_residual and dx.dtype == x.dtype:
+        dresidual_in = dx
+    return (dx, dw, db, dresidual_in) if not recompute_output else (dx, dw, db, dresidual_in, y)
+class LayerNormLinearQuantFn(torch.autograd.Function):
+    @staticmethod
+    @input_guard
+    def forward(
+        ctx,
+        x,
+        norm_weight,
+        norm_bias,
+        linear_weight,
+        linear_bias,
+        residual=None,
+        eps=1e-6,
+        prenorm=False,
+        residual_in_fp32=False,
+        is_rms_norm=False,
+    ):
+        x_shape_og = x.shape
+        # reshape input data into 2D tensor
+        x = x.reshape(-1, x.shape[-1])
+        if residual is not None:
+            assert residual.shape == x_shape_og
+            residual = residual.reshape(-1, residual.shape[-1])
+        residual_dtype = residual.dtype if residual is not None else (torch.float32 if residual_in_fp32 else None)
+        y, mean, rstd, residual_out = layer_norm_fwd_quant(
+            x,
+            norm_weight,
+            norm_bias,
+            eps,
+            residual,
+            out_dtype=None if not torch.is_autocast_enabled() else torch.get_autocast_gpu_dtype(),
+            residual_dtype=residual_dtype,
+            is_rms_norm=is_rms_norm,
+        )
+        y = y.reshape(x_shape_og)
+        dtype = torch.get_autocast_gpu_dtype() if torch.is_autocast_enabled() else y.dtype
+        linear_weight = weight_quant(linear_weight).to(dtype)
+        linear_bias = linear_bias.to(dtype) if linear_bias is not None else None
+        out = F.linear(y.to(linear_weight.dtype), linear_weight, linear_bias)
+        # We don't store y, will be recomputed in the backward pass to save memory
+        ctx.save_for_backward(residual_out, norm_weight, norm_bias, linear_weight, mean, rstd)
+        ctx.x_shape_og = x_shape_og
+        ctx.eps = eps
+        ctx.is_rms_norm = is_rms_norm
+        ctx.has_residual = residual is not None
+        ctx.prenorm = prenorm
+        ctx.x_dtype = x.dtype
+        ctx.linear_bias_is_none = linear_bias is None
+        return out if not prenorm else (out, residual_out.reshape(x_shape_og))
+    @staticmethod
+    @input_guard
+    def backward(ctx, dout, *args):
+        x, norm_weight, norm_bias, linear_weight, mean, rstd = ctx.saved_tensors
+        dout = dout.reshape(-1, dout.shape[-1])
+        dy = F.linear(dout, linear_weight.t())
+        dlinear_bias = None if ctx.linear_bias_is_none else dout.sum(0)
+        assert dy.shape == x.shape
+        if ctx.prenorm:
+            dresidual = args[0]
+            dresidual = dresidual.reshape(-1, dresidual.shape[-1])
+            assert dresidual.shape == x.shape
+        else:
+            dresidual = None
+        dx, dnorm_weight, dnorm_bias, dresidual_in, y = layer_norm_bwd(
+            dy,
+            x,
+            norm_weight,
+            norm_bias,
+            ctx.eps,
+            mean,
+            rstd,
+            dresidual,
+            ctx.has_residual,
+            ctx.is_rms_norm,
+            x_dtype=ctx.x_dtype,
+            recompute_output=True,
+        )
+        dlinear_weight = torch.einsum("bo,bi->oi", dout, y)
+        return (
+            dx.reshape(ctx.x_shape_og),
+            dnorm_weight,
+            dnorm_bias,
+            dlinear_weight,
+            dlinear_bias,
+            dresidual_in.reshape(ctx.x_shape_og) if ctx.has_residual else None,
+            None,
+            None,
+            None,
+            None,
+        )
+def layer_norm_linear_quant_fn(
+    x,
+    norm_weight,
+    norm_bias,
+    linear_weight,
+    linear_bias,
+    residual=None,
+    eps=1e-6,
+    prenorm=False,
+    residual_in_fp32=False,
+    is_rms_norm=False,
+):
+    return LayerNormLinearQuantFn.apply(
+        x,
+        norm_weight,
+        norm_bias,
+        linear_weight,
+        linear_bias,
+        residual,
+        eps,
+        prenorm,
+        residual_in_fp32,
+        is_rms_norm,
+    )
+def rms_norm_linear_quant(
+    x: torch.Tensor,
+    norm_weight: torch.Tensor,
+    norm_bias: torch.Tensor,
+    linear_weight: torch.Tensor,
+    linear_bias: torch.Tensor,
+    residual: torch.Tensor = None,
+    eps: float = 1e-5,
+    prenorm: bool = False,
+    residual_in_fp32: bool = False,
+):
+    return layer_norm_linear_quant_fn(
+        x=x,
+        norm_weight=norm_weight,
+        norm_bias=norm_bias,
+        linear_weight=linear_weight,
+        linear_bias=linear_bias,
+        residual=residual,
+        eps=eps,
+        prenorm=prenorm,
+        residual_in_fp32=residual_in_fp32,
+        is_rms_norm=True,
+    )
+@require_version("triton>=3.0", "Triton >= 3.0 is required to do online quantization.")
+def bit_linear(x, weight, bias=None, norm_weight=None, norm_bias=None, eps=1e-8):
+    """
+    A functional version of BitLinear that applies quantization to activations and weights.
+    Args:
+        x: Input tensor with shape [n, d].
+        weight: Weight tensor with shape [out_features, in_features].
+        bias: Bias tensor with shape [out_features] (optional).
+        norm_weight: Weight tensor for RMS normalization with shape [in_features].
+        norm_bias: Bias tensor for RMS normalization with shape [in_features].
+        eps: A small constant for numerical stability in normalization.
+    Returns:
+        Output tensor with shape [n, out_features].
+    """
+    return layer_norm_linear_quant_fn(
+        x,
+        norm_weight,
+        norm_bias,
+        weight,
+        bias,
+        is_rms_norm=True,
+    )
+class BitLinear(nn.Linear):
+    """
+    A custom linear layer that applies quantization on both activations and weights.
+    This is primarily for training; kernel optimization is needed for efficiency in deployment.
+    """
+    def __init__(
+        self,
+        in_features: int,
+        out_features: int,
+        bias: bool = False,
+        norm_eps: float = 1e-8,
+    ):
+        """
+        Initializes the BitLinear layer.
+        Args:
+            in_features: Size of each input sample.
+            out_features: Size of each output sample.
+            bias: If set to False, the layer will not learn an additive bias. Default: True.
+        """
+        # Initialize the superclass nn.Linear with the given parameters
+        super().__init__(in_features, out_features, bias=bias)
+        self.norm = RMSNorm(in_features, eps=norm_eps, dtype=torch.float32)
+    def __repr__(self) -> str:
+        return f"{self.__class__.__name__}({super().extra_repr()}, norm_eps={self.norm.eps})"
+    def forward(self, x):
+        """
+        Overrides the forward pass to include quantization.
+        Args:
+            x: An input tensor with shape [n, d].
+        Returns:
+            An output tensor with shape [n, d].
+        """
+        # Weight tensor
+        w = self.weight
+        # Apply RMS normalization to the input
+        x_norm = self.norm(x)
+        # Apply quantization to both activations and weights
+        # Uses Straight-Through Estimator (STE) trick with .detach() for gradient flow
+        x_quant = x_norm + (activation_quant(x_norm) - x_norm).detach()
+        w_quant = w + (weight_quant(w) - w).detach()
+        # Perform linear operation with quantized values
+        y = F.linear(x_quant, w_quant)
+        return y
+class FusedBitLinear(BitLinear):
+    """
+    A custom linear layer that applies quantization on both activations and weights.
+    This is primarily for training; kernel optimization is needed for efficiency in deployment.
+    """
+    def __init__(self, in_features, out_features, bias=False):
+        """
+        Initializes the BitLinear layer.
+        Args:
+            in_features: Size of each input sample.
+            out_features: Size of each output sample.
+            bias: If set to False, the layer will not learn an additive bias. Default: True.
+        """
+        # Initialize the superclass nn.Linear with the given parameters
+        super().__init__(in_features, out_features, bias=bias)
+    def forward(self, x):
+        return layer_norm_linear_quant_fn(
+            x,
+            self.norm.weight,
+            self.norm.bias,
+            self.weight,
+            self.bias,
+            is_rms_norm=True,
+        )

build/torch-cuda/modules/fused_cross_entropy.py ADDED Viewed

	@@ -0,0 +1,459 @@

+# Copyright (c) 2023-2026, Songlin Yang, Yu Zhang, Zhiyuan Li
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+# For a list of all contributors, visit:
+#   https://github.com/fla-org/flash-linear-attention/graphs/contributors
+from typing import Any
+import torch
+import torch.nn as nn
+import triton
+import triton.language as tl
+from ..modules.backends import dispatch
+from ..ops.utils.op import exp, log, tanh
+from ..utils import input_guard
+# `all_gather_into_tensor` and `reduce_scatter_tensor` are new placeholders for
+# `_all_gather_base` and `_reduce_scatter_base`. They require the most recent
+# version of PyTorch. The following 2 lines are for backward compatibility with
+# older PyTorch.
+if "all_gather_into_tensor" not in dir(torch.distributed):
+    torch.distributed.all_gather_into_tensor = torch.distributed._all_gather_base
+@triton.heuristics({
+    "HAS_SMOOTHING": lambda args: args["label_smoothing"] > 0.0,
+    "HAS_SOFTCAPPING": lambda args: args["logit_softcapping"] is not None,
+})
+@triton.jit
+def cross_entropy_fwd_kernel(
+    loss_ptr,  # data ptrs
+    lse_ptr,
+    z_loss_ptr,
+    logits_ptr,
+    labels_ptr,
+    label_smoothing,
+    logit_scale,
+    lse_square_scale,
+    logit_softcapping,
+    ignore_index,
+    total_classes,
+    class_start_idx,  # Useful for tensor parallel when each rank only has a subset of classes
+    n_cols,  # shapes
+    n_rows,
+    logits_row_stride,  # strides
+    BLOCK_SIZE: tl.constexpr,
+    HAS_SMOOTHING: tl.constexpr,
+    HAS_SOFTCAPPING: tl.constexpr,
+    # if SPLIT (e.g. tensor parallel), don't include the LSE in the loss since it's not the final LSE
+    SPLIT: tl.constexpr,
+):
+    row_idx = tl.program_id(0)
+    col_block_idx = tl.program_id(1)
+    logits_ptr = logits_ptr + row_idx * logits_row_stride.to(tl.int64)
+    col_offsets = col_block_idx * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)
+    label_idx = tl.load(labels_ptr + row_idx)
+    logits = tl.load(logits_ptr + col_offsets, mask=col_offsets < n_cols, other=-float("inf"))
+    logits = logits.to(tl.float32) * logit_scale
+    if HAS_SOFTCAPPING:
+        logits = logit_softcapping * tanh(logits / logit_softcapping)
+    max_logits = tl.max(logits, 0)
+    if HAS_SMOOTHING:
+        sum_logits = tl.sum(tl.where(col_offsets < n_cols, logits, 0.0), 0)
+    lse = log(tl.sum(exp(logits - max_logits), 0)) + max_logits
+    tl.store(lse_ptr + col_block_idx * n_rows + row_idx, lse)
+    if label_idx == ignore_index:
+        loss = 0.0
+        z_loss = 0.0
+    else:
+        label_idx -= class_start_idx
+        if label_idx >= col_block_idx * BLOCK_SIZE and label_idx < min(
+            n_cols, (col_block_idx + 1) * BLOCK_SIZE,
+        ):
+            logits_label = tl.load(logits_ptr + label_idx).to(tl.float32) * logit_scale
+            if HAS_SOFTCAPPING:
+                logits_label = logit_softcapping * tanh(logits_label / logit_softcapping)
+            if HAS_SMOOTHING:
+                loss = (
+                    (lse if not SPLIT else 0.0)
+                    - label_smoothing * sum_logits / total_classes
+                    - (1 - label_smoothing) * logits_label
+                )
+            else:
+                loss = (lse if not SPLIT else 0.0) - logits_label
+        else:
+            # If label is out of bounds, we set the CE loss to 0.0. But we still want the label_smoothing loss
+            if HAS_SMOOTHING:
+                loss = label_smoothing * ((lse if not SPLIT else 0.0) - sum_logits / total_classes)
+            else:
+                loss = 0.0
+        if not SPLIT:
+            z_loss = lse_square_scale * lse * lse
+            loss += z_loss
+        else:
+            z_loss = 0.0
+    tl.store(loss_ptr + col_block_idx * n_rows + row_idx, loss)
+    if not SPLIT:
+        tl.store(z_loss_ptr + col_block_idx * n_rows + row_idx, z_loss)
+@triton.heuristics({
+    "HAS_SMOOTHING": lambda args: args["label_smoothing"] > 0.0,
+    "HAS_SOFTCAPPING": lambda args: args["logit_softcapping"] is not None,
+})
+@triton.jit
+def cross_entropy_bwd_kernel(
+    dlogits_ptr,  # data ptrs
+    dloss_ptr,
+    logits_ptr,
+    lse_ptr,
+    labels_ptr,
+    label_smoothing,
+    logit_scale,
+    lse_square_scale,
+    logit_softcapping,
+    ignore_index,
+    total_classes,
+    class_start_idx,  # Useful for tensor parallel when each rank only has a subset of classes
+    n_cols,  # shapes
+    logits_row_stride,  # strides
+    dlogits_row_stride,
+    dloss_row_stride,
+    BLOCK_SIZE: tl.constexpr,
+    HAS_SMOOTHING: tl.constexpr,
+    HAS_SOFTCAPPING: tl.constexpr,
+):
+    row_idx = tl.program_id(0)
+    col_block_idx = tl.program_id(1)
+    logits_ptr = logits_ptr + row_idx * logits_row_stride.to(tl.int64)
+    dlogits_ptr = dlogits_ptr + row_idx * dlogits_row_stride.to(tl.int64)
+    col_offsets = col_block_idx * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)
+    label_idx = tl.load(labels_ptr + row_idx)
+    if label_idx != ignore_index:
+        dloss = tl.load(dloss_ptr + row_idx * dloss_row_stride)
+    else:
+        dloss = 0.0
+    logits = tl.load(logits_ptr + col_offsets, mask=col_offsets < n_cols, other=-float("inf")).to(
+        tl.float32,
+    ) * logit_scale
+    if HAS_SOFTCAPPING:
+        t = tanh(logits / logit_softcapping)
+        logits = logit_softcapping * t
+    lse = tl.load(lse_ptr + row_idx)
+    probs = exp(logits - lse)
+    probs += 2.0 * lse_square_scale * lse * probs
+    label_idx -= class_start_idx
+    if HAS_SMOOTHING:
+        smooth_negative = label_smoothing / total_classes
+        probs = tl.where(col_offsets == label_idx, probs - (1 - label_smoothing), probs) - smooth_negative
+    else:
+        probs = tl.where(col_offsets == label_idx, probs - 1.0, probs)
+    # d(softcap * tanh(x/softcap))/dx = 1 - tanh(x/softcap)^2
+    if HAS_SOFTCAPPING:
+        probs = probs * (1.0 - t * t)
+    tl.store(dlogits_ptr + col_offsets, (dloss * logit_scale) * probs, mask=col_offsets < n_cols)
+def fused_cross_entropy_forward(
+    logits: torch.Tensor,
+    target: torch.Tensor,
+    label_smoothing: float = 0.0,
+    logit_scale: float = 1.0,
+    lse_square_scale: float = 0.0,
+    logit_softcapping: float = None,
+    ignore_index: int = -100,
+    process_group=None,
+):
+    n_rows, n_cols = logits.shape
+    assert target.shape == (n_rows,)
+    world_size = 1 if process_group is None else torch.distributed.get_world_size(process_group)
+    total_classes = world_size * n_cols
+    rank = 0 if process_group is None else torch.distributed.get_rank(process_group)
+    class_start_idx = rank * n_cols
+    if logits.stride(-1) != 1:
+        logits = logits.contiguous()
+    # Set these similar to https://github.com/triton-lang/triton/blob/main/python/tutorials/02-fused-softmax.py
+    MAX_BLOCK_SIZE = 64 * 1024
+    BLOCK_SIZE = min(triton.next_power_of_2(n_cols), MAX_BLOCK_SIZE)
+    num_warps = (
+        4
+        if BLOCK_SIZE < 2048
+        else (8 if BLOCK_SIZE < 8192 else (16 if BLOCK_SIZE < 128 * 1024 else 32))
+    )
+    # We may split the lse computation across multiple blocks, then do a reduction
+    # lse(local_lse) to get the final LSE. This is faster for large n_cols (e.g., > 64k)
+    # where having just one thread block processing more than 64k elements is slow.
+    split = world_size > 1 or n_cols > MAX_BLOCK_SIZE
+    n_splits = (n_cols + BLOCK_SIZE - 1) // BLOCK_SIZE
+    loss_shape = (n_splits, n_rows) if n_splits > 1 else (n_rows,)
+    losses = torch.empty(*loss_shape, dtype=torch.float, device=logits.device)
+    lse = torch.empty(*loss_shape, dtype=torch.float, device=logits.device)
+    z_losses = torch.empty(*loss_shape, dtype=torch.float, device=logits.device)
+    cross_entropy_fwd_kernel[(n_rows, n_splits)](
+        losses,  # data ptrs
+        lse,
+        z_losses,
+        logits,
+        target,
+        label_smoothing,
+        logit_scale,
+        lse_square_scale,
+        logit_softcapping,
+        ignore_index,
+        total_classes,
+        class_start_idx,
+        n_cols,  # shapes
+        n_rows,
+        logits.stride(0),  # strides
+        BLOCK_SIZE=BLOCK_SIZE,  # constants
+        num_warps=num_warps,
+        SPLIT=split,
+    )
+    if split:
+        # If there's no label_smoothing, if target are in the vocab of this partition, losses contains
+        # - predicted logit, and 0 otherwise.
+        # If there's label_smoothing=0.1, for target in the vocab of this partition, losses contains
+        # -0.9 * predicted logit - 0.1 * sum logit / total_classes.
+        # For target not in the vocab of this partition, losses contains
+        # -0.1 * sum logit / total_classes.
+        if n_splits > 1:
+            lse = torch.logsumexp(lse, dim=0)
+            losses = losses.sum(dim=0)
+        if world_size > 1:
+            lse_allgather = torch.empty(world_size, n_rows, dtype=lse.dtype, device=lse.device)
+            torch.distributed.all_gather_into_tensor(lse_allgather, lse, group=process_group)
+            handle_losses = torch.distributed.all_reduce(
+                losses, op=torch.distributed.ReduceOp.SUM, group=process_group, async_op=True,
+            )
+            lse = torch.logsumexp(lse_allgather, dim=0)
+            handle_losses.wait()
+        # After the allreduce, if there's no label_smoothing, the total losses are - predicted_logit,
+        # we just have to add the (global) lse.
+        # If there's label_smoothing=0.1, the total losses are
+        # -0.9 * predicted_logit - 0.1 * sum logit / total_classes.
+        # Again, we just have to add the (global) lse.
+        losses += lse
+        if lse_square_scale != 0.0:
+            z_losses = lse_square_scale * lse.square()
+            z_losses.masked_fill_(target == ignore_index, 0.0)
+            losses += z_losses
+        else:
+            z_losses = torch.zeros_like(losses)
+        losses.masked_fill_(target == ignore_index, 0.0)
+    return losses, z_losses, lse, total_classes, class_start_idx
+class CrossEntropyLossFunction(torch.autograd.Function):
+    @staticmethod
+    @input_guard
+    def forward(
+        ctx,
+        logits,
+        target,
+        label_smoothing=0.0,
+        logit_scale=1.0,
+        lse_square_scale=0.0,
+        logit_softcapping=None,
+        ignore_index=-100,
+        inplace_backward=False,
+        process_group=None,
+    ):
+        losses, z_losses, lse, total_classes, class_start_idx = fused_cross_entropy_forward(
+            logits,
+            target,
+            label_smoothing,
+            logit_scale,
+            lse_square_scale,
+            logit_softcapping,
+            ignore_index,
+            process_group,
+        )
+        ctx.save_for_backward(logits, lse, target)
+        ctx.mark_non_differentiable(z_losses)
+        ctx.label_smoothing = label_smoothing
+        ctx.logit_scale = logit_scale
+        ctx.lse_square_scale = lse_square_scale
+        ctx.logit_softcapping = logit_softcapping
+        ctx.ignore_index = ignore_index
+        ctx.total_classes = total_classes
+        ctx.class_start_idx = class_start_idx
+        ctx.inplace_backward = inplace_backward
+        return losses, z_losses
+    @staticmethod
+    @input_guard
+    def backward(ctx, grad_losses, grad_z_losses):
+        del grad_z_losses  # z_losses are only for logging.
+        logits, lse, target = ctx.saved_tensors
+        dlogits = logits if ctx.inplace_backward else torch.empty_like(logits)
+        n_rows, n_cols = logits.shape
+        BLOCK_SIZE = min(triton.next_power_of_2(n_cols), 4 * 1024)
+        num_warps = 4 if BLOCK_SIZE < 2048 else (8 if BLOCK_SIZE < 8192 else 16)
+        def grid(META): return (n_rows, triton.cdiv(n_cols, META["BLOCK_SIZE"]))  # noqa
+        cross_entropy_bwd_kernel[grid](
+            dlogits,  # data ptrs
+            grad_losses,
+            logits,
+            lse,
+            target,
+            ctx.label_smoothing,
+            ctx.logit_scale,
+            ctx.lse_square_scale,
+            ctx.logit_softcapping,
+            ctx.ignore_index,
+            ctx.total_classes,
+            ctx.class_start_idx,
+            n_cols,  # shapes
+            logits.stride(0),  # strides
+            dlogits.stride(0),
+            grad_losses.stride(0),
+            BLOCK_SIZE=BLOCK_SIZE,  # constants
+            num_warps=num_warps,
+        )
+        return dlogits, None, None, None, None, None, None, None, None, None
+@dispatch('modules')
+def cross_entropy_loss(
+    logits: torch.Tensor,
+    target: torch.Tensor,
+    label_smoothing: float = 0.0,
+    logit_scale: float = 1.0,
+    lse_square_scale: float = 0.0,
+    logit_softcapping: float = None,
+    ignore_index=-100,
+    inplace_backward: bool = False,
+    process_group=None,
+) -> tuple[torch.Tensor, torch.Tensor]:
+    """
+    Arguments:
+        logits: [batch, vocab_size]
+        target: [batch,]
+        label_smoothing: float
+        logit_scale: float.
+            Multiply logits by this scale before calculating the loss.
+        lse_square_scale: float.
+            If > 0, we add lse_square_scale * lse(logits) ^ 2 to the loss.
+            This is also referred to as "z-loss".
+        logit_softcapping: float.
+            If > 0, apply logit softcapping: logits = softcap * tanh(logits / softcap).
+            This prevents logit magnitudes from growing unboundedly.
+        ignore_index: int.
+            If target == ignore_index, the loss is set to 0.0.
+        inplace_backward: bool.
+            If True, we do the backward pass in-place by modifying the logits.
+            This saves memory.
+        process_group:
+            if not None, we're doing Tensor Parallel: each process is responsible for
+            one part of the vocab. The loss will be aggregated across processes.
+    Returns:
+        losses: [batch,], float
+        z_losses: [batch,], float
+    """
+    return CrossEntropyLossFunction.apply(
+        logits,
+        target,
+        label_smoothing,
+        logit_scale,
+        lse_square_scale,
+        logit_softcapping,
+        ignore_index,
+        inplace_backward,
+        process_group,
+    )
+class FusedCrossEntropyLoss(nn.Module):
+    def __init__(
+        self,
+        ignore_index: int = -100,
+        reduction: str = "mean",
+        label_smoothing: float = 0.0,
+        logit_scale: float = 1.0,
+        lse_square_scale: float = 0.0,
+        logit_softcapping: float = None,
+        inplace_backward: bool = False,
+        process_group: Any = None,
+        return_z_loss: bool = False,
+    ):
+        """
+        Arguments:
+            ignore_index: int. If target == ignore_index, the loss is set to 0.0.
+            label_smoothing: float
+            lse_square_scale: float. If > 0, we add lse_square_scale * lse(logits) ^ 2 to the loss.
+                This is also referred to as "z-loss".
+            logit_softcapping: float. If > 0, apply logit softcapping:
+                logits = softcap * tanh(logits / softcap).
+                This prevents logit magnitudes from growing unboundedly.
+            inplace_backward: bool. If True, we do the backward pass in-place by modifying the logits.
+                This saves memory.
+            process_group: if not None, we're doing Tensor Parallel: each process is responsible for
+                one part of the vocab. The loss will be aggregated across processes.
+            return_z_loss: bool. If True, we return the component of the loss contributed by
+                the lse_square_scale value. This value is only for logging and does not support
+                backprop.
+        """
+        super().__init__()
+        if reduction not in ["mean", "none", "sum"]:
+            raise NotImplementedError("Only support reduction = 'mean' or 'none' or 'sum'")
+        self.ignore_index = ignore_index
+        self.reduction = reduction
+        self.label_smoothing = label_smoothing
+        self.logit_scale = logit_scale
+        self.lse_square_scale = lse_square_scale
+        self.logit_softcapping = logit_softcapping
+        self.inplace_backward = inplace_backward
+        self.process_group = process_group
+        self.return_z_loss = return_z_loss
+    def forward(self, input, target):
+        """
+        Arguments:
+            input: (batch, vocab_size)
+            target: (batch,)
+        Returns:
+            losses: (batch,) if reduction is 'none', else (1,), dtype float
+            z_loss: (batch,) if reduction is 'none', else (1,), dtype float (if self.return_z_loss)
+        """
+        assert input.device.type in ('cuda', 'npu') and target.device.type in ('cuda', 'npu'), (
+            "Only support CUDA/NPU tensors"
+        )
+        loss, z_loss = cross_entropy_loss(
+            input,
+            target,
+            label_smoothing=self.label_smoothing,
+            logit_scale=self.logit_scale,
+            lse_square_scale=self.lse_square_scale,
+            logit_softcapping=self.logit_softcapping,
+            ignore_index=self.ignore_index,
+            inplace_backward=self.inplace_backward,
+            process_group=self.process_group,
+        )
+        if self.reduction == "mean":
+            loss = loss.sum() / (target != self.ignore_index).sum()
+        elif self.reduction == "sum":
+            loss = loss.sum()
+        else:
+            loss = loss
+        if not self.return_z_loss:
+            return loss
+        if self.reduction == "mean":
+            z_loss = z_loss.sum() / (target != self.ignore_index).sum()
+        elif self.reduction == "sum":
+            z_loss = z_loss.sum()
+        else:
+            z_loss = z_loss
+        return loss, z_loss

build/torch-cuda/modules/fused_kl_div.py ADDED Viewed

	@@ -0,0 +1,372 @@

+# Copyright (c) 2023-2026, Songlin Yang, Yu Zhang, Zhiyuan Li
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+# For a list of all contributors, visit:
+#   https://github.com/fla-org/flash-linear-attention/graphs/contributors
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+import triton
+import triton.language as tl
+from ..modules.backends import dispatch
+from ..ops.utils.op import exp, log
+from ..utils import IS_AMD, input_guard
+# The hard limit of TRITON_MAX_TENSOR_NUMEL is 1048576
+# https://github.com/triton-lang/triton/blob/ba42a5c68fd0505f8c42f4202d53be0f8d9a5fe0/python/triton/language/core.py#L19
+# However, setting limit as 65536 as in LayerNorm tutorial is faster because of less register spilling
+# The optimal maximum block size depends on your hardware, your kernel, and your dtype
+MAX_FUSED_SIZE = 65536 // 2
+STATIC_WARPS = 32 if not IS_AMD else 16
+@triton.jit
+def kl_div_kernel(
+    logits,
+    target_logits,
+    loss,
+    s_logits,
+    s_loss,
+    reduction: tl.constexpr,
+    N: tl.constexpr,
+    V: tl.constexpr,
+    BV: tl.constexpr,
+):
+    # https://github.com/triton-lang/triton/issues/1058
+    # If N*V is too large, i_n * stride will overflow out of int32, so we convert to int64
+    i_n = tl.program_id(0).to(tl.int64)
+    logits += i_n * s_logits
+    target_logits += i_n * s_logits
+    # m is the max value. use the notation from the paper
+    sm = float('-inf')
+    tm = float('-inf')
+    # d is the sum. use the notation from the paper
+    sd, td = 0.0, 0.0
+    NV = tl.cdiv(V, BV)
+    for iv in range(0, NV):
+        o_x = iv * BV + tl.arange(0, BV)
+        # for student
+        b_sl = tl.load(logits + o_x, mask=o_x < V, other=float('-inf'))
+        b_sm = tl.max(b_sl)
+        m_new = tl.maximum(sm, b_sm)
+        sd = sd * exp(sm - m_new) + tl.sum(exp(b_sl - m_new))
+        sm = m_new
+        # for teacher
+        b_tl = tl.load(target_logits + o_x, mask=o_x < V, other=float('-inf'))
+        b_tm = tl.max(b_tl)
+        m_new = tl.maximum(tm, b_tm)
+        td = td * exp(tm - m_new) + tl.sum(exp(b_tl - m_new))
+        tm = m_new
+    b_loss = 0.
+    # KL(y_true || y) = exp(y_true) * (log(y_true) - log(y))
+    for iv in range(0, NV):
+        o_x = iv * BV + tl.arange(0, BV)
+        b_sl = tl.load(logits + o_x, mask=o_x < V, other=float('-inf'))
+        b_tl = tl.load(target_logits + o_x, mask=o_x < V, other=float('-inf'))
+        b_sp_log = b_sl - sm - log(sd)
+        b_tp_log = b_tl - tm - log(td)
+        b_sp = exp(b_sp_log)
+        b_tp = exp(b_tp_log)
+        b_kl = tl.where(o_x < V, b_tp * (b_tp_log - b_sp_log), 0)
+        b_dl = -b_tp + b_sp
+        b_loss += tl.sum(b_kl)
+        if reduction == 'batchmean':
+            b_dl = b_dl / N
+        tl.store(logits + o_x, b_dl, mask=o_x < V)
+    # Normalize the loss by the number of elements if reduction is 'batchmean'
+    if reduction == 'batchmean':
+        b_loss = b_loss / N
+    tl.store(loss + i_n * s_loss, b_loss)
+@triton.jit
+def elementwise_mul_kernel(
+    x,
+    g,
+    N: tl.constexpr,
+    B: tl.constexpr,
+):
+    """
+    This function multiplies each element of the tensor pointed by x with the value pointed by g.
+    The multiplication is performed in-place on the tensor pointed by x.
+    Parameters:
+    x:
+        Pointer to the input tensor.
+    g:
+        Pointer to the gradient output value.
+    N (int):
+        The number of columns in the input tensor.
+    B (int):
+        The block size for Triton operations.
+    """
+    # Get the program ID and convert it to int64 to avoid overflow
+    i_x = tl.program_id(0).to(tl.int64)
+    o_x = i_x * B + tl.arange(0, B)
+    # Load the gradient output value
+    b_g = tl.load(g)
+    b_x = tl.load(x + o_x, mask=o_x < N)
+    tl.store(x + o_x, b_x * b_g, mask=o_x < N)
+@dispatch('modules')
+def fused_kl_div_forward(
+    x: torch.Tensor,
+    target_x: torch.Tensor,
+    weight: torch.Tensor,
+    target_weight: torch.Tensor,
+    reduction: str = 'batchmean',
+    accumulate_grad_in_fp32: bool = True,
+):
+    device = x.device
+    # ideally, we would like to achieve the same memory consumption as [N, H],
+    # so the expected chunk size should be:
+    # NC = ceil(V / H)
+    # C = ceil(N / NC)
+    # for ex: N = 4096*4, V = 32000, H = 4096 ==> NC = 8, C = ceil(N / NC) = 2048
+    N, H, V = *x.shape, weight.shape[0]
+    BV = min(MAX_FUSED_SIZE, triton.next_power_of_2(V))
+    # TODO: in real cases, we may need to limit the number of chunks NC to
+    # ensure the precisions of accumulated gradients
+    NC = min(8, triton.cdiv(V, H))
+    C = triton.next_power_of_2(triton.cdiv(N, NC))
+    NC = triton.cdiv(N, C)
+    grad_dtype = torch.float32 if accumulate_grad_in_fp32 else weight.dtype
+    dx = torch.zeros_like(x, device=device)
+    dw = torch.zeros_like(weight, device=device, dtype=grad_dtype) if weight is not None else None
+    # we use fp32 for loss accumulator
+    loss = torch.zeros(N, dtype=torch.float32, device=device)
+    for ic in range(NC):
+        start, end = ic * C, min((ic + 1) * C, N)
+        # [C, N]
+        c_sx = x[start:end]
+        c_tx = target_x[start:end]
+        # when doing matmul, use the original precision
+        # [C, V]
+        c_sl = F.linear(c_sx, weight)
+        c_tl = F.linear(c_tx, target_weight)
+        if weight is not None and c_sx.dtype != grad_dtype:
+            c_sx = c_sx.to(dtype=grad_dtype)
+        # unreduced loss
+        c_loss = loss[start:end]
+        # Here we calculate the gradient of c_sx in place so we can save memory.
+        kl_div_kernel[(c_sx.shape[0],)](
+            logits=c_sl,
+            target_logits=c_tl,
+            loss=c_loss,
+            s_logits=c_sl.stride(-2),
+            s_loss=c_loss.stride(-1),
+            reduction=reduction,
+            N=N,
+            V=V,
+            BV=BV,
+            num_warps=STATIC_WARPS,
+        )
+        # gradient of logits is computed in-place by the above triton kernel and is of shape: C x V
+        # thus dx[start: end] should be of shape: C x H
+        # additionally, since we are chunking the inputs, observe that the loss and gradients are calculated only
+        # on `n_non_ignore` tokens. However, the gradient of the input should be calculated for all tokens.
+        # Thus, we need an additional scaling factor of (n_non_ignore/total) to scale the gradients.
+        # [C, H]
+        dx[start:end] = torch.mm(c_sl, weight)
+        if weight is not None:
+            torch.addmm(
+                input=dw,
+                mat1=c_sl.t().to(dtype=grad_dtype),
+                mat2=c_sx,
+                out=dw,
+            )
+    loss = loss.sum()
+    if dw is not None:
+        dw = dw.to(weight)
+    return loss, dx, dw
+@dispatch('modules')
+def fused_kl_div_backward(
+    do: torch.Tensor,
+    dx: torch.Tensor,
+    dw: torch.Tensor,
+):
+    # If cross entropy is the last layer, do is 1.0. Skip the mul to save time
+    if torch.ne(do, torch.tensor(1.0, device=do.device)):
+        # We use a Triton kernel instead of a PyTorch operation because modifying inputs in-place
+        # for gradient storage and backward multiple times causes anomalies with PyTorch but not with Triton.
+        N, H = dx.shape
+        B = min(MAX_FUSED_SIZE, triton.next_power_of_2(H))
+        elementwise_mul_kernel[(triton.cdiv(N * H, B),)](
+            x=dx,
+            g=do,
+            N=N*H,
+            B=B,
+            num_warps=STATIC_WARPS,
+        )
+        # handle dw
+        if dw is not None:
+            V, H = dw.shape
+            elementwise_mul_kernel[(triton.cdiv(V * H, B),)](
+                x=dw,
+                g=do,
+                N=V*H,
+                B=B,
+                num_warps=STATIC_WARPS,
+            )
+    return dx, dw
+class FusedKLDivLossFunction(torch.autograd.Function):
+    @staticmethod
+    @input_guard
+    def forward(
+        ctx,
+        x: torch.Tensor,
+        target_x: torch.Tensor,
+        weight: torch.Tensor,
+        target_weight: torch.Tensor,
+        reduction: str,
+        accumulate_grad_in_fp32: bool,
+    ):
+        loss, dx, dw = fused_kl_div_forward(
+            x=x,
+            target_x=target_x,
+            weight=weight,
+            target_weight=target_weight,
+            reduction=reduction,
+            accumulate_grad_in_fp32=accumulate_grad_in_fp32,
+        )
+        ctx.save_for_backward(dx, dw)
+        return loss
+    @staticmethod
+    @input_guard
+    def backward(ctx, do):
+        dx, dw = ctx.saved_tensors
+        dx, dw = fused_kl_div_backward(do, dx, dw)
+        return dx, None, dw, None, None, None
+def fused_kl_div_loss(
+    x: torch.Tensor,
+    target_x: torch.Tensor,
+    weight: torch.Tensor,
+    target_weight: torch.Tensor,
+    reduction: str = 'batchmean',
+    accumulate_grad_in_fp32: bool = True,
+) -> tuple[torch.Tensor, torch.Tensor]:
+    """
+    Args:
+        x (`torch.Tensor`):
+            Tensor of shape `[batch_size * seq_len, hidden_size]`.
+        target_x (`torch.Tensor`):
+            Frozen teacher input tensor of shape `[batch_size * seq_len, hidden_size]`.
+            Must not require gradients.
+        weight (`torch.Tensor`):
+            Tensor of shape `[vocab_size, hidden_size]`.
+        target_weight (`torch.Tensor`):
+            Frozen teacher weight tensor of shape `[vocab_size, hidden_size]`.
+            Must not require gradients.
+        reduction (`str`):
+            Specifies the reduction to apply to the output: 'batchmean'. Default: 'batchmean'.
+        accumulate_grad_in_fp32 (`bool`):
+            Whether to accumulate the student weight gradient in fp32 before casting it back
+            to `weight.dtype`. Default: `True`.
+    Returns:
+        loss
+    """
+    if target_x.requires_grad or target_weight.requires_grad:
+        raise RuntimeError(
+            "FusedKLDivLoss treats target_x and target_weight as a frozen teacher and does not compute "
+            "gradients for them. Detach target_x/target_weight before calling FusedKLDivLoss, or use "
+            "torch.nn.functional.kl_div if teacher gradients are required."
+        )
+    return FusedKLDivLossFunction.apply(
+        x,
+        target_x,
+        weight,
+        target_weight,
+        reduction,
+        accumulate_grad_in_fp32,
+    )
+class FusedKLDivLoss(nn.Module):
+    def __init__(
+        self,
+        reduction: str = 'batchmean',
+        accumulate_grad_in_fp32: bool = True,
+    ):
+        """
+        Args:
+            reduction (`str`):
+                Specifies the reduction to apply to the output: 'batchmean'. Default: 'batchmean'.
+            accumulate_grad_in_fp32 (`bool`):
+                Whether to accumulate the student weight gradient in fp32 before casting it back
+                to `weight.dtype`. Default: `True`.
+        Note:
+            FusedKLDivLoss only computes gradients for `x` and `weight`; `target_x` and
+            `target_weight` are treated as frozen teacher tensors and must not require gradients.
+        """
+        super().__init__()
+        assert reduction in ['batchmean'], f"reduction: {reduction} is not supported"
+        self.reduction = reduction
+        self.accumulate_grad_in_fp32 = accumulate_grad_in_fp32
+    def forward(
+        self,
+        x: torch.Tensor,
+        target_x: torch.Tensor,
+        weight: torch.Tensor,
+        target_weight: torch.Tensor,
+    ):
+        """
+        Args:
+            x (`torch.Tensor`):
+                Tensor of shape `[batch_size * seq_len, hidden_size]`.
+            target_x (`torch.Tensor`):
+                Frozen teacher input tensor of shape `[batch_size * seq_len, hidden_size]`.
+                Must not require gradients.
+            weight (`torch.Tensor`):
+                Tensor of shape `[vocab_size, hidden_size]`.
+            target_weight (`torch.Tensor`):
+                Frozen teacher weight tensor of shape `[vocab_size, hidden_size]`.
+                Must not require gradients.
+        Returns:
+            loss
+        """
+        loss = fused_kl_div_loss(
+            x=x,
+            target_x=target_x,
+            weight=weight,
+            target_weight=target_weight,
+            reduction=self.reduction,
+            accumulate_grad_in_fp32=self.accumulate_grad_in_fp32,
+        )
+        return loss

build/torch-cuda/modules/fused_linear_cross_entropy.py ADDED Viewed

	@@ -0,0 +1,767 @@

+# Copyright (c) 2023-2026, Songlin Yang, Yu Zhang, Zhiyuan Li
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+# For a list of all contributors, visit:
+#   https://github.com/fla-org/flash-linear-attention/graphs/contributors
+# Code adapted from
+# https://github.com/linkedin/Liger-Kernel/blob/main/src/liger_kernel/ops/fused_linear_cross_entropy.py
+from functools import partial
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+import triton
+import triton.language as tl
+from torch.distributed import DeviceMesh
+from torch.distributed.tensor import Replicate, Shard, distribute_module
+from torch.distributed.tensor.parallel import ParallelStyle
+from ..modules.backends import dispatch
+from ..ops.utils.op import exp, log, tanh
+from ..utils import IS_AMD, input_guard
+try:
+    from torch.distributed.tensor import DTensor
+except (ImportError, AttributeError):
+    DTensor = None
+# The hard limit of TRITON_MAX_TENSOR_NUMEL is 1048576
+# https://github.com/triton-lang/triton/blob/ba42a5c68fd0505f8c42f4202d53be0f8d9a5fe0/python/triton/language/core.py#L19
+# However, setting limit as 65536 as in LayerNorm tutorial is faster because of less register spilling
+# The optimal maximum block size depends on your hardware, your kernel, and your dtype
+MAX_FUSED_SIZE = 65536 // 2
+STATIC_WARPS = 32 if not IS_AMD else 16
+@triton.heuristics({
+    'HAS_SCALE': lambda args: args['scale'] is not None,
+    'HAS_SOFTCAPPING': lambda args: args['softcapping'] is not None,
+})
+@triton.jit
+def logsumexp_fwd_kernel(
+    x,
+    z,
+    scale,
+    softcapping,
+    D: tl.constexpr,
+    B: tl.constexpr,
+    HAS_SCALE: tl.constexpr,
+    HAS_SOFTCAPPING: tl.constexpr,
+):
+    i_n, i_d = tl.program_id(0).to(tl.int64), tl.program_id(1).to(tl.int64)
+    o_d = i_d * B + tl.arange(0, B)
+    m_d = o_d < D
+    b_x = tl.load(x + i_n * D + o_d, mask=m_d, other=-float('inf'))
+    if HAS_SCALE:
+        b_x = b_x * scale
+    if HAS_SOFTCAPPING:
+        b_x = softcapping * tanh(b_x / softcapping)
+    b_m = tl.max(b_x, 0)
+    b_z = log(tl.sum(exp(b_x - b_m), 0)) + b_m
+    tl.store(z + i_n * tl.cdiv(D, B) + i_d, b_z)
+@dispatch('modules')
+def logsumexp_fwd(
+    x,
+    scale: float | None = None,
+    softcapping: float | None = None,
+    dtype: torch.dtype | None = None,
+):
+    shape = x.shape
+    x = x.view(-1, shape[-1])
+    N, D = x.shape
+    B = min(triton.next_power_of_2(D), 64 * 1024)
+    ND = triton.cdiv(D, B)
+    z = x.new_empty(N, ND, dtype=torch.float)
+    logsumexp_fwd_kernel[(N, ND)](
+        x=x,
+        z=z,
+        scale=scale,
+        softcapping=softcapping,
+        D=D,
+        B=B,
+    )
+    z = z.logsumexp(-1).view(*shape[:-1])
+    if dtype is not None and dtype != torch.float:
+        z = z.to(dtype)
+    return z
+@triton.jit
+def cross_entropy_kernel(
+    logits,
+    lse,
+    target,
+    loss,
+    total,
+    ignore_index,
+    label_smoothing: tl.constexpr,
+    logit_scale: tl.constexpr,
+    logit_softcapping: tl.constexpr,
+    reduction: tl.constexpr,
+    V: tl.constexpr,
+    BV: tl.constexpr,
+):
+    """
+    This kernel computes both cross entropy loss and the gradient of the input.
+    We only consider hard label + mean reduction for now.
+    Please refer to https://pytorch.org/docs/stable/generated/torch.nn.CrossEntropyLoss.html for the math.
+    Args:
+        logits:
+            Pointer to logits tensor.
+        lse:
+            Pointer to logsumexp tensor.
+        target: Pointer to target tensor.
+        loss:
+            Pointer to tensor to store the loss.
+        V (int):
+            The number of columns in the input tensor.
+        total (int):
+            The number of non-ignored classes.
+        ignore_index (int):
+            The index to ignore in the target.
+        label_smoothing (float):
+            The amount of smoothing when computing the loss, where 0.0 means no smoothing.
+        reduction (str):
+            The string for the reduction to apply
+        BV (int):
+            The block size for vocab.
+    """
+    # https://github.com/triton-lang/triton/issues/1058
+    # If B*T*V is too large, i_n * stride will overflow out of int32, so we convert to int64
+    i_n = tl.program_id(0).to(tl.int64)
+    NV = tl.cdiv(V, BV)
+    # 1. Load target first because if the target is ignore_index, we can return right away
+    b_y = tl.load(target + i_n)
+    # 2. locate the start index
+    logits += i_n * V
+    if b_y == ignore_index:
+        # set all x as 0
+        for i in range(0, V, BV):
+            o_v = i + tl.arange(0, BV)
+            tl.store(logits + o_v, 0.0, mask=o_v < V)
+        return
+    # Online softmax: 2 loads + 1 store (compared with 3 loads + 1 store for the safe softmax)
+    # Refer to Algorithm 3 in the paper: https://arxiv.org/pdf/1805.02867
+    # 3. [Online softmax] first pass: compute logsumexp
+    # we did this in another kernel
+    b_l = tl.load(logits + b_y).to(tl.float32) * logit_scale
+    if logit_softcapping is not None:
+        b_t_y = tanh(b_l / logit_softcapping)
+        b_l = logit_softcapping * b_t_y
+        # Save the softcap derivative for the target position for use in step 6
+        b_softcap_deriv_y = 1.0 - b_t_y * b_t_y
+    b_lse = tl.load(lse + i_n)
+    # 4. Calculate the loss
+    # loss = lse - logits_l
+    b_loss = b_lse - b_l
+    # Label smoothing is a general case of normal cross entropy
+    # See the full derivation at https://github.com/linkedin/Liger-Kernel/pull/198#issue-2503665310
+    b_z = 0.0
+    eps = label_smoothing / V
+    # We need tl.debug_barrier() as mentioned in
+    # https://github.com/triton-lang/triton/blob/ba42a5c68fd0505f8c42f4202d53be0f8d9a5fe0/python/triton/ops/cross_entropy.py#L34
+    tl.debug_barrier()
+    # 5. [Online Softmax] Second pass: compute gradients
+    # For 'mean' reduction, gradients are normalized by number of non-ignored elements
+    # dx_y = (softmax(x_y) - 1) / N
+    # dx_i = softmax(x_i) / N, i != y
+    # For label smoothing:
+    # dx_i = (softmax(x_y) - label_smoothing / V) / N, i != y
+    # dx_y = (softmax(x_y) - label_smoothing / V - (1 - label_smoothing)) / N
+    #      = dx_i - (1 - label_smoothing) / N
+    for iv in range(0, NV):
+        o_v = iv * BV + tl.arange(0, BV)
+        b_logits = tl.load(logits + o_v, mask=o_v < V, other=float('-inf')).to(tl.float32) * logit_scale
+        if logit_softcapping is not None:
+            b_t = tanh(b_logits / logit_softcapping)
+            b_capped = logit_softcapping * b_t
+        else:
+            b_capped = b_logits
+        if label_smoothing > 0:
+            # scale X beforehand to avoid overflow
+            b_z += tl.sum(tl.where(o_v < V, -eps * b_capped, 0.0))
+        b_p = (exp(b_capped - b_lse) - eps) * logit_scale
+        # d(softcap * tanh(x/softcap))/dx = 1 - tanh(x/softcap)^2
+        if logit_softcapping is not None:
+            b_p = b_p * (1.0 - b_t * b_t)
+        if reduction == "mean":
+            b_p = b_p / total
+        tl.store(logits + o_v, b_p, mask=o_v < V)
+        tl.debug_barrier()
+    # Original loss = H(q, p),  with label smoothing regularization = H(q', p) and (label_smoothing / V) = eps
+    # H(q', p) = (1 - label_smoothing) * H(q, p) + label_smoothing * H(u, p)
+    #          = (1 - label_smoothing) * H(q, p) + eps * sum(logsoftmax(x_i))
+    # By using m (global max of xi) and d (sum of e^(xi-m)), we can simplify as:
+    #          = (1 - label_smoothing) * H(q, p) + (-sum(x_i * eps) + label_smoothing * (m + logd))
+    # Refer to H(q', p) in section 7 of the paper:
+    # https://arxiv.org/pdf/1512.00567
+    # pytorch:
+    # https://github.com/pytorch/pytorch/blob/2981534f54d49fa3a9755c9b0855e7929c2527f0/aten/src/ATen/native/LossNLL.cpp#L516
+    # See full derivation at https://github.com/linkedin/Liger-Kernel/pull/198#issuecomment-2333753087
+    if label_smoothing > 0:
+        b_loss = b_loss * (1 - label_smoothing) + (b_z + label_smoothing * b_lse)
+    # 6. Specially handle the i==y case where `dx_y = (softmax(x_y) - (1 - label_smoothing) / N`
+    b_l = tl.load(logits + b_y)
+    # The correction term also needs the softcap chain rule factor
+    if logit_softcapping is not None:
+        b_sc_factor = b_softcap_deriv_y
+    else:
+        b_sc_factor = 1.0
+    # Normalize the loss by the number of non-ignored elements if reduction is "mean"
+    if reduction == 'mean':
+        b_loss = b_loss / total
+        b_l += (label_smoothing - 1) / total * logit_scale * b_sc_factor
+    else:
+        b_l += (label_smoothing - 1) * logit_scale * b_sc_factor
+    tl.store(loss + i_n, b_loss)
+    tl.store(logits + b_y, b_l)
+@triton.jit
+def elementwise_mul_kernel(
+    x,
+    g,
+    N: tl.constexpr,
+    B: tl.constexpr,
+):
+    """
+    This function multiplies each element of the tensor pointed by x with the value pointed by g.
+    The multiplication is performed in-place on the tensor pointed by x.
+    Parameters:
+    x:
+        Pointer to the input tensor.
+    g:
+        Pointer to the gradient output value.
+    N (int):
+        The number of columns in the input tensor.
+    B (int):
+        The block size for Triton operations.
+    """
+    # Get the program ID and convert it to int64 to avoid overflow
+    i_x = tl.program_id(0).to(tl.int64)
+    o_x = i_x * B + tl.arange(0, B)
+    # Load the gradient output value
+    b_g = tl.load(g)
+    b_x = tl.load(x + o_x, mask=o_x < N)
+    tl.store(x + o_x, b_x * b_g, mask=o_x < N)
+@dispatch('modules')
+def fused_linear_cross_entropy_forward(
+    x: torch.Tensor,
+    target: torch.LongTensor,
+    weight: torch.Tensor,
+    bias: torch.Tensor = None,
+    ignore_index: int = -100,
+    label_smoothing: float = 0.0,
+    logit_scale: float = 1.0,
+    logit_softcapping: float = None,
+    num_chunks: int = 8,
+    reduction: str = "mean",
+    use_l2warp: bool = False,
+    l2_penalty_factor: float = 1e-4,
+    accumulate_grad_in_fp32: bool = True,
+):
+    device = x.device
+    # inputs have shape: [N, H]
+    # materialized activations will have shape: [N, V]
+    # the increase in memory = [N, V]
+    # reduction can be achieved by partitioning the number of tokens N into smaller chunks.
+    # ideally, we would like to achieve the same memory consumption as [N, H],
+    # so the expected chunk size should be:
+    # NC = ceil(V / H)
+    # C = ceil(N / NC)
+    # for ex: N = 4096*4, V = 32000, H = 4096 ==> NC = 8, C = ceil(N / NC) = 2048
+    N, H, V = *x.shape, weight.shape[0]
+    BV = min(MAX_FUSED_SIZE, triton.next_power_of_2(V))
+    # TODO: in real cases, we may need to limit the number of chunks NC to
+    # ensure the precisions of accumulated gradients
+    NC = min(num_chunks, triton.cdiv(V, H))
+    C = triton.next_power_of_2(triton.cdiv(N, NC))
+    NC = triton.cdiv(N, C)
+    # [N, H]
+    dx = torch.zeros_like(x, device=device)
+    grad_dtype = torch.float32 if accumulate_grad_in_fp32 else weight.dtype
+    bias_grad_dtype = None
+    if bias is not None:
+        bias_grad_dtype = torch.float32 if accumulate_grad_in_fp32 else bias.dtype
+    # [V, H]
+    dw = torch.zeros_like(weight, device=device, dtype=grad_dtype) if weight is not None else None
+    # [V]
+    db = torch.zeros_like(bias, device=device, dtype=bias_grad_dtype) if bias is not None else None
+    # [N]
+    loss = torch.zeros(N, device=device, dtype=torch.float)
+    total = target.ne(ignore_index).sum().item()
+    for ic in range(NC):
+        start, end = ic * C, min((ic + 1) * C, N)
+        # [C, N]
+        c_x = x[start:end]
+        # when doing matmul, use the original precision
+        # [C, V]
+        c_logits = F.linear(c_x, weight, bias)
+        if weight is not None and c_x.dtype != grad_dtype:
+            c_x = c_x.to(dtype=grad_dtype)
+        c_target = target[start:end]
+        # [C]
+        # keep lse in fp32 to maintain precision
+        c_lse = logsumexp_fwd(c_logits, scale=logit_scale, softcapping=logit_softcapping, dtype=torch.float)
+        # unreduced loss
+        c_loss = loss[start:end]
+        if use_l2warp:
+            c_maxx, c_ids = torch.max(c_logits, -1, keepdim=True)
+        # Here we calculate the gradient of c_logits in place so we can save memory.
+        cross_entropy_kernel[(c_logits.shape[0],)](
+            logits=c_logits,
+            lse=c_lse,
+            target=c_target,
+            loss=c_loss,
+            total=total,
+            ignore_index=ignore_index,
+            label_smoothing=label_smoothing,
+            logit_scale=logit_scale,
+            logit_softcapping=logit_softcapping,
+            reduction=reduction,
+            V=V,
+            BV=BV,
+            num_warps=STATIC_WARPS,
+        )
+        if use_l2warp:
+            # a. Calculate the L2 gradient w.r.t logits (g_logits_l2)
+            g_logits_l2 = torch.zeros_like(c_logits)
+            # Match L2Wrap: normalize by the full number of input tokens, not by non-ignored labels.
+            l2_factor = l2_penalty_factor / N
+            penalty_grad = c_maxx * l2_factor
+            g_logits_l2.scatter_(-1, c_ids, penalty_grad)
+            # b. Backpropagate g_logits_l2 to get its effect on dx, dw, db
+            # and add it to the main gradients.
+            # Total_dx = CE_dx + L2_dx
+            # Total_dw = CE_dw + L2_dw
+            # Total_db = CE_db + L2_db
+            if weight is not None:
+                torch.addmm(
+                    input=dw,
+                    mat1=g_logits_l2.t().to(dtype=grad_dtype),
+                    mat2=c_x,
+                    out=dw,
+                )
+            if bias is not None:
+                torch.add(input=db, other=g_logits_l2.sum(0, dtype=bias_grad_dtype), out=db)
+            # The dx contribution must be added to the final dx calculation
+            dx_l2_contribution = torch.mm(g_logits_l2, weight)
+        else:
+            dx_l2_contribution = 0.0
+        # gradient of logits is computed in-place by the above triton kernel and is of shape: C x V
+        # thus dx should be of shape: C x H
+        dx[start:end] = torch.mm(c_logits, weight) + dx_l2_contribution
+        if weight is not None:
+            torch.addmm(
+                input=dw,
+                mat1=c_logits.t().to(dtype=grad_dtype),
+                mat2=c_x,
+                out=dw,
+            )
+        if bias is not None:
+            torch.add(input=db, other=c_logits.sum(0, dtype=bias_grad_dtype), out=db)
+    loss = loss.sum()
+    if dw is not None:
+        dw = dw.to(weight)
+    if db is not None:
+        db = db.to(bias)
+    return loss, dx, dw, db
+@dispatch('modules')
+def fused_linear_cross_entropy_backward(
+    do: torch.Tensor,
+    dx: torch.Tensor,
+    dw: torch.Tensor,
+    db: torch.Tensor,
+):
+    # If cross entropy is the last layer, do is 1.0. Skip the mul to save time
+    if torch.ne(do, torch.tensor(1.0, device=do.device)):
+        # We use a Triton kernel instead of a PyTorch operation because modifying inputs in-place
+        # for gradient storage and backward multiple times causes anomalies with PyTorch but not with Triton.
+        N, H = dx.shape
+        B = min(MAX_FUSED_SIZE, triton.next_power_of_2(H))
+        elementwise_mul_kernel[(triton.cdiv(N * H, B),)](
+            x=dx,
+            g=do,
+            N=N*H,
+            B=B,
+            num_warps=STATIC_WARPS,
+        )
+        # handle dw
+        if dw is not None:
+            V, H = dw.shape
+            elementwise_mul_kernel[(triton.cdiv(V * H, B),)](
+                x=dw,
+                g=do,
+                N=V*H,
+                B=B,
+                num_warps=STATIC_WARPS,
+            )
+        if db is not None:
+            V = db.shape[0]
+            elementwise_mul_kernel[(triton.cdiv(V, B),)](
+                x=db,
+                g=do,
+                N=V,
+                B=B,
+                num_warps=STATIC_WARPS,
+            )
+    return dx, dw, db
+class FusedLinearCrossEntropyFunction(torch.autograd.Function):
+    @staticmethod
+    @input_guard
+    def forward(
+        ctx,
+        x: torch.Tensor,
+        target: torch.LongTensor,
+        weight: torch.Tensor,
+        bias: torch.Tensor = None,
+        ignore_index: int = -100,
+        label_smoothing: float = 0.0,
+        logit_scale: float = 1.0,
+        logit_softcapping: float = None,
+        num_chunks: int = 8,
+        reduction: str = "mean",
+        use_l2warp: bool = False,
+        l2_penalty_factor: float = 1e-4,
+        accumulate_grad_in_fp32: bool = True,
+    ):
+        """
+        Fusing the last linear layer with cross-entropy loss
+            Reference: https://github.com/mgmalek/efficient_cross_entropy
+        Handle the forward and backward pass of the final linear layer via cross-entropy loss by avoiding
+        the materialization of the large logits tensor. Since Cross Entropy Loss is the last layer, we can
+        compute the gradient at the forward pass. By doing so, we don't have to store the x and target
+        for the backward pass.
+        x (torch.Tensor): [batch_size * seq_len, hidden_size]
+        target (torch.LongTensor): [batch_size * seq_len]
+            where each value is in [0, vocab_size).
+        weight (torch.Tensor): [vocab_size, hidden_size]
+            where `vocab_size` is the number of classes.
+        bias (Optional[torch.Tensor]): [vocab_size]
+            where `vocab_size` is the number of classes.
+        ignore_index:
+            the index to ignore in the target.
+        label_smoothing:
+            the amount of smoothing when computing the loss, where 0.0 means no smoothing.
+        logit_scale: float = 1.0,
+            A scaling factor applied to the logits. Default: 1.0
+        logit_softcapping: float = None,
+            If > 0, apply logit softcapping: logits = softcap * tanh(logits / softcap).
+            Default: 0.0
+        num_chunks: int
+            The number of chunks to split the input tensor into for processing.
+            This can help optimize memory usage and computation speed.
+            Default: 8
+        reduction:
+            Specifies the reduction to apply to the output: 'mean' | 'sum'.
+            'mean': the weighted mean of the output is taken,
+            'sum': the output will be summed.
+            Default: 'mean'.
+        use_l2warp: bool = False,
+            Whether to use L2 regularization on the logits to prevent overconfidence.
+            Default: False
+        l2_penalty_factor: float = 1e-4,
+            The L2Warp penalty factor. Default: 1e-4
+        accumulate_grad_in_fp32: bool = True,
+            Whether to accumulate weight and bias gradients in fp32 before casting them
+            back to the parameter dtype. Default: True
+        """
+        loss, dx, dw, db = fused_linear_cross_entropy_forward(
+            x,
+            target,
+            weight,
+            bias,
+            ignore_index,
+            label_smoothing,
+            logit_scale,
+            logit_softcapping,
+            num_chunks,
+            reduction,
+            use_l2warp,
+            l2_penalty_factor,
+            accumulate_grad_in_fp32,
+        )
+        # downcast to dtype and store for backward
+        ctx.save_for_backward(
+            dx.detach(),
+            dw.detach() if weight is not None else None,
+            db.detach() if bias is not None else None,
+        )
+        return loss
+    @staticmethod
+    @input_guard
+    def backward(ctx, do):
+        dx, dw, db = ctx.saved_tensors
+        dx, dw, db = fused_linear_cross_entropy_backward(do, dx, dw, db)
+        return dx, None, dw, db, None, None, None, None, None, None, None, None, None
+def fused_linear_cross_entropy_loss(
+    x: torch.Tensor,
+    target: torch.LongTensor,
+    weight: torch.Tensor,
+    bias: torch.Tensor = None,
+    ignore_index: int = -100,
+    label_smoothing: float = 0.0,
+    logit_scale: float = 1.0,
+    logit_softcapping: float = None,
+    num_chunks: int = 8,
+    reduction: str = "mean",
+    use_l2warp: bool = False,
+    l2_penalty_factor: float = 1e-4,
+    accumulate_grad_in_fp32: bool = True,
+) -> tuple[torch.Tensor, torch.Tensor]:
+    """
+    Args:
+        x (torch.Tensor): [batch_size * seq_len, hidden_size]
+        target (torch.LongTensor): [batch_size * seq_len]
+            where each value is in [0, vocab_size).
+        weight (torch.Tensor): [vocab_size, hidden_size]
+            where `vocab_size` is the number of classes.
+        bias (Optional[torch.Tensor]): [vocab_size]
+            where `vocab_size` is the number of classes.
+        ignore_index: int.
+            If target == ignore_index, the loss is set to 0.0.
+        label_smoothing: float
+        logit_scale: float
+            A scaling factor applied to the logits. Default: 1.0
+        logit_softcapping: float
+            If > 0, apply logit softcapping: logits = softcap * tanh(logits / softcap).
+            Default: 0.0
+        num_chunks: int
+            The number of chunks to split the input tensor into for processing.
+            This can help optimize memory usage and computation speed.
+            Default: 8
+        reduction:
+            Specifies the reduction to apply to the output: 'mean' | 'sum'.
+            'mean': the weighted mean of the output is taken,
+            'sum': the output will be summed.
+            Default: 'mean'.
+        use_l2warp:
+            Whether to add the L2Warp logit regularization gradient. The penalty is normalized by
+            the full number of input tokens, matching `fla.modules.l2warp.l2_warp`.
+            Default: `False`.
+        l2_penalty_factor:
+            The L2Warp penalty factor. Default: `1e-4`.
+        accumulate_grad_in_fp32:
+            Whether to accumulate weight and bias gradients in fp32 before casting them
+            back to the parameter dtype. Default: `True`.
+    Returns:
+        losses: [batch,], float
+    """
+    return FusedLinearCrossEntropyFunction.apply(
+        x,
+        target,
+        weight,
+        bias,
+        ignore_index,
+        label_smoothing,
+        logit_scale,
+        logit_softcapping,
+        num_chunks,
+        reduction,
+        use_l2warp,
+        l2_penalty_factor,
+        accumulate_grad_in_fp32,
+    )
+class FusedLinearCrossEntropyLoss(nn.Module):
+    def __init__(
+        self,
+        ignore_index: int = -100,
+        label_smoothing: float = 0.0,
+        logit_scale: float = 1.0,
+        logit_softcapping: float = None,
+        num_chunks: int = 8,
+        reduction: str = "mean",
+        use_l2warp: bool = False,
+        l2_penalty_factor: float = 1e-4,
+        accumulate_grad_in_fp32: bool = True,
+    ):
+        """
+        Args:
+            ignore_index: int.
+                If target == ignore_index, the loss is set to 0.0.
+            label_smoothing: float
+            logit_scale: float
+                A scaling factor applied to the logits. Default: 1.0
+            logit_softcapping: float
+                If > 0, apply logit softcapping: logits = softcap * tanh(logits / softcap).
+                Default: 0.0
+            num_chunks: int
+                The number of chunks to split the input tensor into for processing.
+                This can help optimize memory usage and computation speed.
+                Default: 8
+            reduction:
+                Specifies the reduction to apply to the output: 'mean' | 'sum'.
+                'mean': the weighted mean of the output is taken,
+                'sum': the output will be summed.
+                Default: 'mean'.
+            use_l2warp:
+                Whether to add the L2Warp logit regularization gradient. The penalty is normalized by
+                the full number of input tokens, matching `fla.modules.l2warp.l2_warp`.
+                Default: `False`.
+            l2_penalty_factor:
+                The L2Warp penalty factor. Default: `1e-4`.
+            accumulate_grad_in_fp32:
+                Whether to accumulate weight and bias gradients in fp32 before casting them
+                back to the parameter dtype. Default: `True`.
+        """
+        super().__init__()
+        assert reduction in ["mean", "sum"], f"reduction: {reduction} is not supported"
+        self.ignore_index = ignore_index
+        self.label_smoothing = label_smoothing
+        self.logit_scale = logit_scale
+        self.logit_softcapping = logit_softcapping
+        self.num_chunks = num_chunks
+        self.reduction = reduction
+        self.use_l2warp = use_l2warp
+        self.l2_penalty_factor = l2_penalty_factor
+        self.accumulate_grad_in_fp32 = accumulate_grad_in_fp32
+    @torch.compiler.disable
+    def forward(
+        self,
+        x: torch.Tensor,
+        target: torch.LongTensor,
+        weight: torch.Tensor,
+        bias: torch.Tensor | None = None,
+    ):
+        """
+        Args:
+            x (torch.Tensor): [batch_size, seq_len, hidden_size]
+            target (torch.LongTensor): [batch_size, seq_len]
+                where each value is in [0, V).
+            weight (torch.Tensor): [vocab_size, hidden_size]
+                where `vocab_size` is the number of classes.
+            bias (Optional[torch.Tensor]): [vocab_size]
+                where `vocab_size` is the number of classes.
+        Returns:
+            loss
+        """
+        loss = fused_linear_cross_entropy_loss(
+            x.view(-1, x.shape[-1]),
+            target.view(-1),
+            weight=weight,
+            bias=bias,
+            ignore_index=self.ignore_index,
+            label_smoothing=self.label_smoothing,
+            logit_scale=self.logit_scale,
+            logit_softcapping=self.logit_softcapping,
+            num_chunks=self.num_chunks,
+            reduction=self.reduction,
+            use_l2warp=self.use_l2warp,
+            l2_penalty_factor=self.l2_penalty_factor,
+            accumulate_grad_in_fp32=self.accumulate_grad_in_fp32,
+        )
+        return loss
+class LinearLossParallel(ParallelStyle):
+    def __init__(
+        self,
+        *,
+        sequence_dim: int = 1,
+        use_local_output: bool = False,
+    ):
+        super().__init__()
+        self.sequence_sharding = (Shard(sequence_dim),)
+        self.use_local_output = use_local_output
+    @staticmethod
+    def _prepare_input_fn(sequence_sharding, mod, inputs, device_mesh):
+        x, target, weight, bias = inputs
+        if not isinstance(x, DTensor):
+            # assume the input passed in already sharded on the sequence dim and create the DTensor
+            x = DTensor.from_local(x, device_mesh, sequence_sharding)
+        if x.placements != sequence_sharding:
+            x = x.redistribute(placements=sequence_sharding, async_op=True)
+        if not isinstance(target, DTensor):
+            target = DTensor.from_local(target, device_mesh, [Replicate()])
+        if target.placements != sequence_sharding:
+            target = target.redistribute(placements=sequence_sharding, async_op=True)
+        if not isinstance(weight, DTensor):
+            weight = DTensor.from_local(weight, device_mesh, [Replicate()])
+        if weight.placements != [Replicate()]:
+            # we replicate the weight/bias in FLCE
+            weight = weight.redistribute(placements=[Replicate()], async_op=True)
+        if bias is not None and not isinstance(bias, DTensor):
+            bias = DTensor.from_local(bias, device_mesh, [Replicate()])
+        if bias is not None and bias.placements != [Replicate()]:
+            bias = bias.redistribute(placements=[Replicate()], async_op=True)
+        return x.to_local(), target.to_local(), weight.to_local(), bias.to_local() if bias is not None else bias
+    @staticmethod
+    def _prepare_output_fn(use_local_output, mod, outputs, device_mesh):
+        return outputs.to_local() if use_local_output else outputs
+    def _apply(self, module: nn.Module, device_mesh: DeviceMesh) -> nn.Module:
+        return distribute_module(
+            module,
+            device_mesh,
+            partition_fn=None,
+            input_fn=partial(self._prepare_input_fn, self.sequence_sharding),
+            output_fn=partial(self._prepare_output_fn, self.use_local_output),
+        )

build/torch-cuda/modules/fused_norm_gate.py ADDED Viewed

	@@ -0,0 +1,1245 @@

+# Copyright (c) 2023-2026, Songlin Yang, Yu Zhang, Zhiyuan Li
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+# For a list of all contributors, visit:
+#   https://github.com/fla-org/flash-linear-attention/graphs/contributors
+from __future__ import annotations
+import math
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+import triton
+import triton.language as tl
+from ..utils import autotune_cache_kwargs, get_multiprocessor_count, input_guard
+@triton.heuristics(
+    {
+        "STORE_RESIDUAL_OUT": lambda args: args["residual_out"] is not None,
+        "HAS_RESIDUAL": lambda args: args["residual"] is not None,
+        "HAS_WEIGHT": lambda args: args["w"] is not None,
+        "HAS_BIAS": lambda args: args["b"] is not None,
+    }
+)
+@triton.autotune(
+    configs=[triton.Config({"BT": BT}, num_warps=num_warps) for BT in [16, 32, 64] for num_warps in [4, 8, 16]],
+    key=["D", "NB", "IS_RMS_NORM", "STORE_RESIDUAL_OUT", "HAS_RESIDUAL", "HAS_WEIGHT"],
+    **autotune_cache_kwargs,
+)
+@triton.jit
+def layer_norm_gated_fwd_kernel(
+    x,  # pointer to the input
+    g,  # pointer to the gate
+    y,  # pointer to the output
+    w,  # pointer to the weights
+    b,  # pointer to the biases
+    residual,  # pointer to the residual
+    residual_out,  # pointer to the residual
+    mean,  # pointer to the mean
+    rstd,  # pointer to the 1/std
+    eps,  # epsilon to avoid division by zero
+    T,  # number of rows in x
+    D: tl.constexpr,  # number of columns in x
+    BT: tl.constexpr,
+    BD: tl.constexpr,
+    NB: tl.constexpr,
+    ACTIVATION: tl.constexpr,
+    IS_RMS_NORM: tl.constexpr,
+    STORE_RESIDUAL_OUT: tl.constexpr,
+    HAS_RESIDUAL: tl.constexpr,
+    HAS_WEIGHT: tl.constexpr,
+    HAS_BIAS: tl.constexpr,
+):
+    i_t = tl.program_id(0)
+    o_d = tl.arange(0, BD)
+    m_d = o_d < D
+    p_x = tl.make_block_ptr(x, (T, D), (D, 1), (i_t * BT, 0), (BT, BD), (1, 0))
+    b_x = tl.load(p_x, boundary_check=(0, 1)).to(tl.float32)
+    if HAS_RESIDUAL:
+        p_res = tl.make_block_ptr(residual, (T, D), (D, 1), (i_t * BT, 0), (BT, BD), (1, 0))
+        b_x += tl.load(p_res, boundary_check=(0, 1)).to(tl.float32)
+    if STORE_RESIDUAL_OUT:
+        p_res_out = tl.make_block_ptr(residual_out, (T, D), (D, 1), (i_t * BT, 0), (BT, BD), (1, 0))
+        tl.store(p_res_out, b_x.to(p_res_out.dtype.element_ty), boundary_check=(0, 1))
+    if not IS_RMS_NORM:
+        b_mean = tl.sum(b_x, axis=1) / D
+        p_mean = tl.make_block_ptr(mean, (T,), (1,), (i_t * BT,), (BT,), (0,))
+        tl.store(p_mean, b_mean.to(p_mean.dtype.element_ty), boundary_check=(0,))
+        b_xbar = tl.where(m_d[None, :], b_x - b_mean[:, None], 0.0)
+        b_var = tl.sum(b_xbar * b_xbar, axis=1) / D
+    else:
+        b_xbar = tl.where(m_d[None, :], b_x, 0.0)
+        b_var = tl.sum(b_xbar * b_xbar, axis=1) / D
+    b_rstd = 1 / tl.sqrt(b_var + eps)
+    p_rstd = tl.make_block_ptr(rstd, (T,), (1,), (i_t * BT,), (BT,), (0,))
+    tl.store(p_rstd, b_rstd.to(p_rstd.dtype.element_ty), boundary_check=(0,))
+    if HAS_WEIGHT:
+        b_w = tl.load(w + o_d, mask=m_d).to(tl.float32)
+    if HAS_BIAS:
+        b_b = tl.load(b + o_d, mask=m_d).to(tl.float32)
+    b_x_hat = (b_x - b_mean[:, None]) * b_rstd[:, None] if not IS_RMS_NORM else b_x * b_rstd[:, None]
+    b_y = b_x_hat * b_w[None, :] if HAS_WEIGHT else b_x_hat
+    if HAS_BIAS:
+        b_y = b_y + b_b[None, :]
+    # swish/sigmoid output gate
+    p_g = tl.make_block_ptr(g, (T, D), (D, 1), (i_t * BT, 0), (BT, BD), (1, 0))
+    b_g = tl.load(p_g, boundary_check=(0, 1)).to(tl.float32)
+    if ACTIVATION == "swish" or ACTIVATION == "silu":
+        b_y = b_y * b_g * tl.sigmoid(b_g)
+    elif ACTIVATION == "sigmoid":
+        b_y = b_y * tl.sigmoid(b_g)
+    # Write output
+    p_y = tl.make_block_ptr(y, (T, D), (D, 1), (i_t * BT, 0), (BT, BD), (1, 0))
+    tl.store(p_y, b_y.to(p_y.dtype.element_ty), boundary_check=(0, 1))
+@triton.heuristics(
+    {
+        "STORE_RESIDUAL_OUT": lambda args: args["residual_out"] is not None,
+        "HAS_RESIDUAL": lambda args: args["residual"] is not None,
+        "HAS_WEIGHT": lambda args: args["w"] is not None,
+        "HAS_BIAS": lambda args: args["b"] is not None,
+    }
+)
+@triton.autotune(
+    configs=[triton.Config({}, num_warps=num_warps) for num_warps in [2, 4, 8, 16]],
+    key=["D", "IS_RMS_NORM", "STORE_RESIDUAL_OUT", "HAS_RESIDUAL", "HAS_WEIGHT"],
+    **autotune_cache_kwargs,
+)
+@triton.jit
+def layer_norm_gated_fwd_kernel1(
+    x,  # pointer to the input
+    g,  # pointer to the gate
+    y,  # pointer to the output
+    w,  # pointer to the weights
+    b,  # pointer to the biases
+    residual,  # pointer to the residual
+    residual_out,  # pointer to the residual
+    mean,  # pointer to the mean
+    rstd,  # pointer to the 1/std
+    eps,  # epsilon to avoid division by zero
+    D: tl.constexpr,  # number of columns in x
+    BD: tl.constexpr,
+    ACTIVATION: tl.constexpr,
+    IS_RMS_NORM: tl.constexpr,
+    STORE_RESIDUAL_OUT: tl.constexpr,
+    HAS_RESIDUAL: tl.constexpr,
+    HAS_WEIGHT: tl.constexpr,
+    HAS_BIAS: tl.constexpr,
+):
+    i_t = tl.program_id(0)
+    x += i_t * D
+    y += i_t * D
+    g += i_t * D
+    if HAS_RESIDUAL:
+        residual += i_t * D
+    if STORE_RESIDUAL_OUT:
+        residual_out += i_t * D
+    o_d = tl.arange(0, BD)
+    m_d = o_d < D
+    b_x = tl.load(x + o_d, mask=m_d, other=0.0).to(tl.float32)
+    if HAS_RESIDUAL:
+        b_x += tl.load(residual + o_d, mask=m_d, other=0.0).to(tl.float32)
+    if STORE_RESIDUAL_OUT:
+        tl.store(residual_out + o_d, b_x, mask=m_d)
+    if not IS_RMS_NORM:
+        b_mean = tl.sum(b_x, axis=0) / D
+        tl.store(mean + i_t, b_mean)
+        b_xbar = tl.where(m_d, b_x - b_mean, 0.0)
+        b_var = tl.sum(b_xbar * b_xbar, axis=0) / D
+    else:
+        b_xbar = tl.where(m_d, b_x, 0.0)
+        b_var = tl.sum(b_xbar * b_xbar, axis=0) / D
+    b_rstd = 1 / tl.sqrt(b_var + eps)
+    tl.store(rstd + i_t, b_rstd)
+    if HAS_WEIGHT:
+        b_w = tl.load(w + o_d, mask=m_d).to(tl.float32)
+    if HAS_BIAS:
+        b_b = tl.load(b + o_d, mask=m_d).to(tl.float32)
+    b_x_hat = (b_x - b_mean) * b_rstd if not IS_RMS_NORM else b_x * b_rstd
+    b_y = b_x_hat * b_w if HAS_WEIGHT else b_x_hat
+    if HAS_BIAS:
+        b_y = b_y + b_b
+    # swish/sigmoid output gate
+    b_g = tl.load(g + o_d, mask=m_d, other=0.0).to(tl.float32)
+    if ACTIVATION == "swish" or ACTIVATION == "silu":
+        b_y = b_y * b_g * tl.sigmoid(b_g)
+    elif ACTIVATION == "sigmoid":
+        b_y = b_y * tl.sigmoid(b_g)
+    # Write output
+    tl.store(y + o_d, b_y, mask=m_d)
+@triton.heuristics(
+    {
+        "HAS_DRESIDUAL": lambda args: args["dresidual"] is not None,
+        "HAS_WEIGHT": lambda args: args["w"] is not None,
+        "HAS_BIAS": lambda args: args["b"] is not None,
+        "RECOMPUTE_OUTPUT": lambda args: args["y"] is not None,
+    }
+)
+@triton.autotune(
+    configs=[triton.Config({"BT": BT}, num_warps=num_warps) for BT in [16, 32, 64] for num_warps in [4, 8, 16]],
+    key=["D", "NB", "IS_RMS_NORM", "HAS_DRESIDUAL", "HAS_WEIGHT"],
+    **autotune_cache_kwargs,
+)
+@triton.jit
+def layer_norm_gated_bwd_kernel(
+    x,  # pointer to the input
+    g,  # pointer to the gate
+    w,  # pointer to the weights
+    b,  # pointer to the biases
+    y,  # pointer to the output to be recomputed
+    dy,  # pointer to the output gradient
+    dx,  # pointer to the input gradient
+    dg,  # pointer to the gate gradient
+    dw,  # pointer to the partial sum of weights gradient
+    db,  # pointer to the partial sum of biases gradient
+    dresidual,
+    dresidual_in,
+    mean,
+    rstd,
+    T,
+    BS,
+    D: tl.constexpr,
+    BT: tl.constexpr,
+    BD: tl.constexpr,
+    NB: tl.constexpr,
+    ACTIVATION: tl.constexpr,
+    IS_RMS_NORM: tl.constexpr,
+    STORE_DRESIDUAL: tl.constexpr,
+    HAS_DRESIDUAL: tl.constexpr,
+    HAS_WEIGHT: tl.constexpr,
+    HAS_BIAS: tl.constexpr,
+    RECOMPUTE_OUTPUT: tl.constexpr,
+):
+    i_s = tl.program_id(0)
+    o_d = tl.arange(0, BD)
+    m_d = o_d < D
+    if HAS_WEIGHT:
+        b_w = tl.load(w + o_d, mask=m_d).to(tl.float32)
+        b_dw = tl.zeros((BT, BD), dtype=tl.float32)
+    if HAS_BIAS:
+        b_b = tl.load(b + o_d, mask=m_d, other=0.0).to(tl.float32)
+        b_db = tl.zeros((BT, BD), dtype=tl.float32)
+    # the caller guarantees NS = min(SM, T), so every program has at least one token.
+    # the last program's range may slightly exceed T (since BS = ceil(T/NS));
+    # make_block_ptr uses the true tensor shape (T, D), so boundary_check
+    # handles the partial tail tile by zero-padding loads and skipping stores.
+    # the m_t mask below further ensures dw/db only accumulate valid rows (< T).
+    for i_t in range(i_s * BS, i_s * BS + BS, BT):
+        p_x = tl.make_block_ptr(x, (T, D), (D, 1), (i_t, 0), (BT, BD), (1, 0))
+        p_g = tl.make_block_ptr(g, (T, D), (D, 1), (i_t, 0), (BT, BD), (1, 0))
+        p_dy = tl.make_block_ptr(dy, (T, D), (D, 1), (i_t, 0), (BT, BD), (1, 0))
+        p_dx = tl.make_block_ptr(dx, (T, D), (D, 1), (i_t, 0), (BT, BD), (1, 0))
+        p_dg = tl.make_block_ptr(dg, (T, D), (D, 1), (i_t, 0), (BT, BD), (1, 0))
+        # [BT, BD]
+        b_x = tl.load(p_x, boundary_check=(0, 1)).to(tl.float32)
+        b_g = tl.load(p_g, boundary_check=(0, 1)).to(tl.float32)
+        b_dy = tl.load(p_dy, boundary_check=(0, 1)).to(tl.float32)
+        if not IS_RMS_NORM:
+            p_mean = tl.make_block_ptr(mean, (T,), (1,), (i_t,), (BT,), (0,))
+            b_mean = tl.load(p_mean, boundary_check=(0,))
+        p_rstd = tl.make_block_ptr(rstd, (T,), (1,), (i_t,), (BT,), (0,))
+        b_rstd = tl.load(p_rstd, boundary_check=(0,))
+        # Compute dx
+        b_xhat = (b_x - b_mean[:, None]) * b_rstd[:, None] if not IS_RMS_NORM else b_x * b_rstd[:, None]
+        b_xhat = tl.where(m_d[None, :], b_xhat, 0.0)
+        b_y = b_xhat * b_w[None, :] if HAS_WEIGHT else b_xhat
+        if HAS_BIAS:
+            b_y = b_y + b_b[None, :]
+        if RECOMPUTE_OUTPUT:
+            p_y = tl.make_block_ptr(y, (T, D), (D, 1), (i_t, 0), (BT, BD), (1, 0))
+            tl.store(p_y, b_y.to(p_y.dtype.element_ty), boundary_check=(0, 1))
+        b_sigmoid_g = tl.sigmoid(b_g)
+        if ACTIVATION == "swish" or ACTIVATION == "silu":
+            b_dg = b_dy * b_y * (b_sigmoid_g + b_g * b_sigmoid_g * (1 - b_sigmoid_g))
+            b_dy = b_dy * b_g * b_sigmoid_g
+        elif ACTIVATION == "sigmoid":
+            b_dg = b_dy * b_y * b_sigmoid_g * (1 - b_sigmoid_g)
+            b_dy = b_dy * b_sigmoid_g
+        b_wdy = b_dy
+        if HAS_WEIGHT or HAS_BIAS:
+            # when BT > BS, a tile may span into the next program's range;
+            # mask to this program's upper bound to avoid double-counting dw/db.
+            m_t = (i_t + tl.arange(0, BT)) < min(i_s * BS + BS, T)
+        if HAS_WEIGHT:
+            b_wdy = b_dy * b_w
+            b_dw += tl.where(m_t[:, None], b_dy * b_xhat, 0.0)
+        if HAS_BIAS:
+            b_db += tl.where(m_t[:, None], b_dy, 0.0)
+        if not IS_RMS_NORM:
+            b_c1 = tl.sum(b_xhat * b_wdy, axis=1) / D
+            b_c2 = tl.sum(b_wdy, axis=1) / D
+            b_dx = (b_wdy - (b_xhat * b_c1[:, None] + b_c2[:, None])) * b_rstd[:, None]
+        else:
+            b_c1 = tl.sum(b_xhat * b_wdy, axis=1) / D
+            b_dx = (b_wdy - b_xhat * b_c1[:, None]) * b_rstd[:, None]
+        if HAS_DRESIDUAL:
+            p_dres = tl.make_block_ptr(dresidual, (T, D), (D, 1), (i_t, 0), (BT, BD), (1, 0))
+            b_dres = tl.load(p_dres, boundary_check=(0, 1)).to(tl.float32)
+            b_dx += b_dres
+        # Write dx
+        if STORE_DRESIDUAL:
+            p_dres_in = tl.make_block_ptr(dresidual_in, (T, D), (D, 1), (i_t, 0), (BT, BD), (1, 0))
+            tl.store(p_dres_in, b_dx.to(p_dres_in.dtype.element_ty), boundary_check=(0, 1))
+        tl.store(p_dx, b_dx.to(p_dx.dtype.element_ty), boundary_check=(0, 1))
+        tl.store(p_dg, b_dg.to(p_dg.dtype.element_ty), boundary_check=(0, 1))
+    if HAS_WEIGHT:
+        tl.store(dw + i_s * D + o_d, tl.sum(b_dw, axis=0), mask=m_d)
+    if HAS_BIAS:
+        tl.store(db + i_s * D + o_d, tl.sum(b_db, axis=0), mask=m_d)
+@triton.heuristics(
+    {
+        "HAS_DRESIDUAL": lambda args: args["dresidual"] is not None,
+        "HAS_WEIGHT": lambda args: args["w"] is not None,
+        "HAS_BIAS": lambda args: args["b"] is not None,
+        "RECOMPUTE_OUTPUT": lambda args: args["y"] is not None,
+    }
+)
+@triton.autotune(
+    configs=[triton.Config({}, num_warps=num_warps) for num_warps in [2, 4, 8, 16]],
+    key=["D", "IS_RMS_NORM", "STORE_DRESIDUAL", "HAS_DRESIDUAL", "HAS_WEIGHT"],
+    **autotune_cache_kwargs,
+)
+@triton.jit
+def layer_norm_gated_bwd_kernel1(
+    x,  # pointer to the input
+    g,  # pointer to the gate
+    w,  # pointer to the weights
+    b,  # pointer to the biases
+    y,  # pointer to the output to be recomputed
+    dy,  # pointer to the output gradient
+    dx,  # pointer to the input gradient
+    dg,  # pointer to the gate gradient
+    dw,  # pointer to the partial sum of weights gradient
+    db,  # pointer to the partial sum of biases gradient
+    dresidual,
+    dresidual_in,
+    mean,
+    rstd,
+    T,
+    BS,
+    D: tl.constexpr,
+    BD: tl.constexpr,
+    ACTIVATION: tl.constexpr,
+    IS_RMS_NORM: tl.constexpr,
+    STORE_DRESIDUAL: tl.constexpr,
+    HAS_DRESIDUAL: tl.constexpr,
+    HAS_WEIGHT: tl.constexpr,
+    HAS_BIAS: tl.constexpr,
+    RECOMPUTE_OUTPUT: tl.constexpr,
+):
+    i_s = tl.program_id(0)
+    o_d = tl.arange(0, BD)
+    mask = o_d < D
+    x += i_s * BS * D
+    g += i_s * BS * D
+    if HAS_DRESIDUAL:
+        dresidual += i_s * BS * D
+    if STORE_DRESIDUAL:
+        dresidual_in += i_s * BS * D
+    dy += i_s * BS * D
+    dx += i_s * BS * D
+    dg += i_s * BS * D
+    if RECOMPUTE_OUTPUT:
+        y += i_s * BS * D
+    if HAS_WEIGHT:
+        b_w = tl.load(w + o_d, mask=mask).to(tl.float32)
+        b_dw = tl.zeros((BD,), dtype=tl.float32)
+    if HAS_BIAS:
+        b_b = tl.load(b + o_d, mask=mask, other=0.0).to(tl.float32)
+        b_db = tl.zeros((BD,), dtype=tl.float32)
+    for i_t in range(i_s * BS, min(i_s * BS + BS, T)):
+        # Load data to SRAM
+        b_x = tl.load(x + o_d, mask=mask, other=0).to(tl.float32)
+        b_g = tl.load(g + o_d, mask=mask, other=0).to(tl.float32)
+        b_dy = tl.load(dy + o_d, mask=mask, other=0).to(tl.float32)
+        if not IS_RMS_NORM:
+            b_mean = tl.load(mean + i_t)
+        b_rstd = tl.load(rstd + i_t)
+        # Compute dx
+        b_xhat = (b_x - b_mean) * b_rstd if not IS_RMS_NORM else b_x * b_rstd
+        b_xhat = tl.where(mask, b_xhat, 0.0)
+        b_y = b_xhat * b_w if HAS_WEIGHT else b_xhat
+        if HAS_BIAS:
+            b_y = b_y + b_b
+        if RECOMPUTE_OUTPUT:
+            tl.store(y + o_d, b_y, mask=mask)
+        b_sigmoid_g = tl.sigmoid(b_g)
+        if ACTIVATION == "swish" or ACTIVATION == "silu":
+            b_dg = b_dy * b_y * (b_sigmoid_g + b_g * b_sigmoid_g * (1 - b_sigmoid_g))
+            b_dy = b_dy * b_g * b_sigmoid_g
+        elif ACTIVATION == "sigmoid":
+            b_dg = b_dy * b_y * b_sigmoid_g * (1 - b_sigmoid_g)
+            b_dy = b_dy * b_sigmoid_g
+        b_wdy = b_dy
+        if HAS_WEIGHT:
+            b_wdy = b_dy * b_w
+            b_dw += b_dy * b_xhat
+        if HAS_BIAS:
+            b_db += b_dy
+        if not IS_RMS_NORM:
+            b_c1 = tl.sum(b_xhat * b_wdy, axis=0) / D
+            b_c2 = tl.sum(b_wdy, axis=0) / D
+            b_dx = (b_wdy - (b_xhat * b_c1 + b_c2)) * b_rstd
+        else:
+            b_c1 = tl.sum(b_xhat * b_wdy, axis=0) / D
+            b_dx = (b_wdy - b_xhat * b_c1) * b_rstd
+        if HAS_DRESIDUAL:
+            b_dres = tl.load(dresidual + o_d, mask=mask, other=0).to(tl.float32)
+            b_dx += b_dres
+        # Write dx
+        if STORE_DRESIDUAL:
+            tl.store(dresidual_in + o_d, b_dx, mask=mask)
+        tl.store(dx + o_d, b_dx, mask=mask)
+        tl.store(dg + o_d, b_dg, mask=mask)
+        x += D
+        g += D
+        if HAS_DRESIDUAL:
+            dresidual += D
+        if STORE_DRESIDUAL:
+            dresidual_in += D
+        if RECOMPUTE_OUTPUT:
+            y += D
+        dy += D
+        dx += D
+        dg += D
+    if HAS_WEIGHT:
+        tl.store(dw + i_s * D + o_d, b_dw, mask=mask)
+    if HAS_BIAS:
+        tl.store(db + i_s * D + o_d, b_db, mask=mask)
+def layer_norm_gated_fwd(
+    x: torch.Tensor,
+    g: torch.Tensor,
+    weight: torch.Tensor,
+    bias: torch.Tensor,
+    activation: str = "swish",
+    eps: float = 1e-5,
+    residual: torch.Tensor = None,
+    out_dtype: torch.dtype = None,
+    residual_dtype: torch.dtype = None,
+    is_rms_norm: bool = False,
+):
+    if residual is not None:
+        residual_dtype = residual.dtype
+    T, D = x.shape
+    if residual is not None:
+        assert residual.shape == (T, D)
+    if weight is not None:
+        assert weight.shape == (D,)
+    if bias is not None:
+        assert bias.shape == (D,)
+    # allocate output
+    y = torch.empty_like(x, dtype=x.dtype if out_dtype is None else out_dtype)
+    if residual is not None or (residual_dtype is not None and residual_dtype != x.dtype):
+        residual_out = torch.empty(T, D, device=x.device, dtype=residual_dtype)
+    else:
+        residual_out = None
+    mean = torch.empty((T,), dtype=torch.float, device=x.device) if not is_rms_norm else None
+    rstd = torch.empty((T,), dtype=torch.float, device=x.device)
+    # Less than 64KB per feature: enqueue fused kernel
+    MAX_FUSED_SIZE = 65536 // x.element_size()
+    BD = min(MAX_FUSED_SIZE, triton.next_power_of_2(D))
+    if D > BD:
+        raise RuntimeError("This layer norm doesn't support feature dim >= 64KB.")
+    # heuristics for number of warps
+    if D <= 512:
+        # NOTE(tylerr): Avoid excessive recompilation and autotuning by tolerating a larger range
+        # of T before recompiling the kernel.
+        # NB = triton.cdiv(T, 2048)
+        NB = triton.cdiv(T, 2048 * 32)
+        def grid(meta):
+            return (triton.cdiv(T, meta["BT"]),)
+        layer_norm_gated_fwd_kernel[grid](
+            x=x,
+            g=g,
+            y=y,
+            w=weight,
+            b=bias,
+            residual=residual,
+            residual_out=residual_out,
+            mean=mean,
+            rstd=rstd,
+            eps=eps,
+            T=T,
+            D=D,
+            BD=BD,
+            NB=NB,
+            ACTIVATION=activation,
+            IS_RMS_NORM=is_rms_norm,
+        )
+    else:
+        layer_norm_gated_fwd_kernel1[(T,)](
+            x=x,
+            g=g,
+            y=y,
+            w=weight,
+            b=bias,
+            residual=residual,
+            residual_out=residual_out,
+            mean=mean,
+            rstd=rstd,
+            eps=eps,
+            D=D,
+            BD=BD,
+            ACTIVATION=activation,
+            IS_RMS_NORM=is_rms_norm,
+        )
+    # residual_out is None if residual is None and residual_dtype == input_dtype
+    return y, mean, rstd, residual_out if residual_out is not None else x
+def layer_norm_gated_bwd(
+    dy: torch.Tensor,
+    x: torch.Tensor,
+    g: torch.Tensor,
+    weight: torch.Tensor,
+    bias: torch.Tensor,
+    activation: str = "swish",
+    eps: float = 1e-5,
+    mean: torch.Tensor = None,
+    rstd: torch.Tensor = None,
+    dresidual: torch.Tensor = None,
+    has_residual: bool = False,
+    is_rms_norm: bool = False,
+    x_dtype: torch.dtype = None,
+    recompute_output: bool = False,
+):
+    T, D = x.shape
+    assert dy.shape == (T, D)
+    if dresidual is not None:
+        assert dresidual.shape == (T, D)
+    if weight is not None:
+        assert weight.shape == (D,)
+    if bias is not None:
+        assert bias.shape == (D,)
+    # allocate output
+    dx = torch.empty_like(x) if x_dtype is None else torch.empty(T, D, dtype=x_dtype, device=x.device)
+    dg = torch.empty_like(g) if x_dtype is None else torch.empty(T, D, dtype=x_dtype, device=x.device)
+    dresidual_in = torch.empty_like(x) if has_residual and dx.dtype != x.dtype else None
+    y = torch.empty(T, D, dtype=dy.dtype, device=dy.device) if recompute_output else None
+    # Less than 64KB per feature: enqueue fused kernel
+    MAX_FUSED_SIZE = 65536 // x.element_size()
+    BD = min(MAX_FUSED_SIZE, triton.next_power_of_2(D))
+    if D > BD:
+        raise RuntimeError("This layer norm doesn't support feature dim >= 64KB.")
+    # cap program count to T so no program is completely idle.
+    # without this, high-SM GPUs (e.g. B200, 160 SMs) with small T would
+    # launch idle programs whose make_block_ptr offsets exceed the tensor shape.
+    NS = min(get_multiprocessor_count(x.device.index), T)
+    BS = math.ceil(T / NS)
+    dw = torch.empty((NS, D), dtype=torch.float, device=weight.device) if weight is not None else None
+    db = torch.empty((NS, D), dtype=torch.float, device=bias.device) if bias is not None else None
+    grid = (NS,)
+    if D <= 512:
+        # NOTE(tylerr): Avoid excessive recompilation and autotuning by tolerating a larger range
+        # of T before recompiling the kernel.
+        # NB = triton.cdiv(T, 2048)
+        NB = triton.cdiv(T, 2048 * 32)
+        layer_norm_gated_bwd_kernel[grid](
+            x=x,
+            g=g,
+            w=weight,
+            b=bias,
+            y=y,
+            dy=dy,
+            dx=dx,
+            dg=dg,
+            dw=dw,
+            db=db,
+            dresidual=dresidual,
+            dresidual_in=dresidual_in,
+            mean=mean,
+            rstd=rstd,
+            T=T,
+            D=D,
+            BS=BS,
+            BD=BD,
+            NB=NB,
+            ACTIVATION=activation,
+            IS_RMS_NORM=is_rms_norm,
+            STORE_DRESIDUAL=dresidual_in is not None,
+        )
+    else:
+        layer_norm_gated_bwd_kernel1[grid](
+            x=x,
+            g=g,
+            w=weight,
+            b=bias,
+            y=y,
+            dy=dy,
+            dx=dx,
+            dg=dg,
+            dw=dw,
+            db=db,
+            dresidual=dresidual,
+            dresidual_in=dresidual_in,
+            mean=mean,
+            rstd=rstd,
+            T=T,
+            D=D,
+            BS=BS,
+            BD=BD,
+            ACTIVATION=activation,
+            IS_RMS_NORM=is_rms_norm,
+            STORE_DRESIDUAL=dresidual_in is not None,
+        )
+    dw = dw.sum(0).to(weight.dtype) if weight is not None else None
+    db = db.sum(0).to(bias.dtype) if bias is not None else None
+    # Don't need to compute dresidual_in separately in this case
+    if has_residual and dx.dtype == x.dtype:
+        dresidual_in = dx
+    return (dx, dg, dw, db, dresidual_in) if not recompute_output else (dx, dg, dw, db, dresidual_in, y)
+class LayerNormGatedFunction(torch.autograd.Function):
+    @staticmethod
+    @input_guard
+    def forward(
+        ctx,
+        x: torch.Tensor,
+        g: torch.Tensor,
+        weight: torch.Tensor,
+        bias: torch.Tensor,
+        activation: str,
+        residual: torch.Tensor | None = None,
+        eps: float = 1e-6,
+        prenorm: bool = False,
+        residual_in_fp32: bool = False,
+        is_rms_norm: bool = False,
+    ):
+        x_shape_og = x.shape
+        g_shape_og = g.shape
+        # reshape input data into 2D tensor
+        x = x.reshape(-1, x.shape[-1])
+        g = g.reshape(-1, g.shape[-1])
+        if residual is not None:
+            assert residual.shape == x_shape_og
+            residual = residual.reshape(-1, residual.shape[-1])
+        residual_dtype = residual.dtype if residual is not None else (torch.float if residual_in_fp32 else None)
+        y, mean, rstd, residual_out = layer_norm_gated_fwd(
+            x=x,
+            g=g,
+            weight=weight,
+            bias=bias,
+            activation=activation,
+            eps=eps,
+            residual=residual,
+            residual_dtype=residual_dtype,
+            is_rms_norm=is_rms_norm,
+        )
+        ctx.save_for_backward(residual_out, g, weight, bias, mean, rstd)
+        ctx.x_shape_og = x_shape_og
+        ctx.g_shape_og = g_shape_og
+        ctx.activation = activation
+        ctx.eps = eps
+        ctx.is_rms_norm = is_rms_norm
+        ctx.has_residual = residual is not None
+        ctx.prenorm = prenorm
+        ctx.x_dtype = x.dtype
+        y = y.reshape(x_shape_og)
+        return y if not prenorm else (y, residual_out.reshape(x_shape_og))
+    @staticmethod
+    @input_guard
+    def backward(ctx, dy, *args):
+        x, g, weight, bias, mean, rstd = ctx.saved_tensors
+        dy = dy.reshape(-1, dy.shape[-1])
+        assert dy.shape == x.shape
+        if ctx.prenorm:
+            dresidual = args[0]
+            dresidual = dresidual.reshape(-1, dresidual.shape[-1])
+            assert dresidual.shape == x.shape
+        else:
+            dresidual = None
+        dx, dg, dw, db, dres_in = layer_norm_gated_bwd(
+            dy=dy,
+            x=x,
+            g=g,
+            weight=weight,
+            bias=bias,
+            activation=ctx.activation,
+            eps=ctx.eps,
+            mean=mean,
+            rstd=rstd,
+            dresidual=dresidual,
+            has_residual=ctx.has_residual,
+            is_rms_norm=ctx.is_rms_norm,
+            x_dtype=ctx.x_dtype,
+        )
+        return (
+            dx.reshape(ctx.x_shape_og),
+            dg.reshape(ctx.g_shape_og),
+            dw,
+            db,
+            None,
+            dres_in.reshape(ctx.x_shape_og) if ctx.has_residual else None,
+            None,
+            None,
+            None,
+            None,
+        )
+class LayerNormGatedLinearFunction(torch.autograd.Function):
+    @staticmethod
+    @input_guard
+    def forward(
+        ctx,
+        x: torch.Tensor,
+        g: torch.Tensor,
+        norm_weight: torch.Tensor,
+        norm_bias: torch.Tensor,
+        linear_weight: torch.Tensor,
+        linear_bias: torch.Tensor,
+        residual: torch.Tensor | None = None,
+        eps: float = 1e-6,
+        prenorm: bool = False,
+        residual_in_fp32: bool = False,
+        is_rms_norm: bool = False,
+    ):
+        x_shape_og = x.shape
+        g_shape_og = g.shape
+        # reshape input data into 2D tensor
+        x = x.reshape(-1, x.shape[-1])
+        g = g.reshape(-1, g.shape[-1])
+        if residual is not None:
+            assert residual.shape == x_shape_og
+            residual = residual.reshape(-1, residual.shape[-1])
+        residual_dtype = residual.dtype if residual is not None else (torch.float if residual_in_fp32 else None)
+        y, mean, rstd, residual_out = layer_norm_gated_fwd(
+            x=x,
+            g=g,
+            weight=norm_weight,
+            bias=norm_bias,
+            eps=eps,
+            residual=residual,
+            residual_dtype=residual_dtype,
+            is_rms_norm=is_rms_norm,
+        )
+        y = y.reshape(x_shape_og)
+        dtype = torch.get_autocast_gpu_dtype() if torch.is_autocast_enabled() else y.dtype
+        linear_weight = linear_weight.to(dtype)
+        linear_bias = linear_bias.to(dtype) if linear_bias is not None else None
+        out = F.linear(y.to(linear_weight.dtype), linear_weight, linear_bias)
+        # We don't store y, will be recomputed in the backward pass to save memory
+        ctx.save_for_backward(residual_out, g, norm_weight, norm_bias, linear_weight, mean, rstd)
+        ctx.x_shape_og = x_shape_og
+        ctx.g_shape_og = g_shape_og
+        ctx.eps = eps
+        ctx.is_rms_norm = is_rms_norm
+        ctx.has_residual = residual is not None
+        ctx.prenorm = prenorm
+        ctx.x_dtype = x.dtype
+        ctx.linear_bias_is_none = linear_bias is None
+        return out if not prenorm else (out, residual_out.reshape(x_shape_og))
+    @staticmethod
+    @input_guard
+    def backward(ctx, dout, *args):
+        x, g, norm_weight, norm_bias, linear_weight, mean, rstd = ctx.saved_tensors
+        dout = dout.reshape(-1, dout.shape[-1])
+        dy = F.linear(dout, linear_weight.t())
+        dlinear_bias = None if ctx.linear_bias_is_none else dout.sum(0)
+        assert dy.shape == x.shape
+        if ctx.prenorm:
+            dresidual = args[0]
+            dresidual = dresidual.reshape(-1, dresidual.shape[-1])
+            assert dresidual.shape == x.shape
+        else:
+            dresidual = None
+        dx, dg, dnorm_weight, dnorm_bias, dres_in, y = layer_norm_gated_bwd(
+            dy=dy,
+            x=x,
+            g=g,
+            weight=norm_weight,
+            bias=norm_bias,
+            eps=ctx.eps,
+            mean=mean,
+            rstd=rstd,
+            dresidual=dresidual,
+            has_residual=ctx.has_residual,
+            is_rms_norm=ctx.is_rms_norm,
+            x_dtype=ctx.x_dtype,
+            recompute_output=True,
+        )
+        dlinear_weight = torch.einsum("bo,bi->oi", dout, y)
+        return (
+            dx.reshape(ctx.x_shape_og),
+            dg.reshape(ctx.g_shape_og),
+            dnorm_weight,
+            dnorm_bias,
+            dlinear_weight,
+            dlinear_bias,
+            dres_in.reshape(ctx.x_shape_og) if ctx.has_residual else None,
+            None,
+            None,
+            None,
+            None,
+        )
+def layer_norm_gated(
+    x: torch.Tensor,
+    g: torch.Tensor,
+    weight: torch.Tensor,
+    bias: torch.Tensor,
+    activation: str = "swish",
+    residual: torch.Tensor | None = None,
+    prenorm: bool = False,
+    residual_in_fp32: bool = False,
+    eps: float = 1e-6,
+):
+    return LayerNormGatedFunction.apply(
+        x,
+        g,
+        weight,
+        bias,
+        activation,
+        residual,
+        eps,
+        prenorm,
+        residual_in_fp32,
+        False,
+    )
+def rms_norm_gated(
+    x: torch.Tensor,
+    g: torch.Tensor,
+    weight: torch.Tensor,
+    bias: torch.Tensor,
+    activation: str = "swish",
+    residual: torch.Tensor | None = None,
+    prenorm: bool = False,
+    residual_in_fp32: bool = False,
+    eps: float = 1e-6,
+):
+    return LayerNormGatedFunction.apply(
+        x,
+        g,
+        weight,
+        bias,
+        activation,
+        residual,
+        eps,
+        prenorm,
+        residual_in_fp32,
+        True,
+    )
+def layer_norm_swish_gate_linear(
+    x: torch.Tensor,
+    g: torch.Tensor,
+    norm_weight: torch.Tensor,
+    norm_bias: torch.Tensor,
+    linear_weight: torch.Tensor,
+    linear_bias: torch.Tensor,
+    residual: torch.Tensor | None = None,
+    prenorm: bool = False,
+    residual_in_fp32: bool = False,
+    eps: float = 1e-6,
+):
+    return LayerNormGatedLinearFunction.apply(
+        x,
+        g,
+        norm_weight,
+        norm_bias,
+        linear_weight,
+        linear_bias,
+        residual,
+        eps,
+        prenorm,
+        residual_in_fp32,
+        False,
+    )
+def rms_norm_swish_gate_linear(
+    x,
+    g: torch.Tensor,
+    norm_weight: torch.Tensor,
+    norm_bias: torch.Tensor,
+    linear_weight: torch.Tensor,
+    linear_bias: torch.Tensor,
+    residual: torch.Tensor | None = None,
+    prenorm: bool = False,
+    residual_in_fp32: bool = False,
+    eps: float = 1e-6,
+):
+    return LayerNormGatedLinearFunction.apply(
+        x,
+        g,
+        norm_weight,
+        norm_bias,
+        linear_weight,
+        linear_bias,
+        residual,
+        eps,
+        prenorm,
+        residual_in_fp32,
+        True,
+    )
+class FusedLayerNormGated(nn.Module):
+    def __init__(
+        self,
+        hidden_size: int,
+        elementwise_affine: bool = True,
+        bias: bool = False,
+        activation: str = "swish",
+        eps: float = 1e-5,
+        device: torch.device | None = None,
+        dtype: torch.dtype | None = None,
+    ) -> FusedLayerNormGated:
+        factory_kwargs = {"device": device, "dtype": dtype}
+        super().__init__()
+        self.hidden_size = hidden_size
+        self.elementwise_affine = elementwise_affine
+        self.eps = eps
+        self.activation = activation
+        if self.activation not in ["swish", "silu", "sigmoid"]:
+            raise ValueError(f"Unsupported activation: {self.activation}")
+        self.register_parameter("weight", None)
+        self.register_parameter("bias", None)
+        if elementwise_affine:
+            self.weight = nn.Parameter(torch.empty(hidden_size, **factory_kwargs))
+            if bias:
+                self.bias = nn.Parameter(torch.empty(hidden_size, **factory_kwargs))
+        self.reset_parameters()
+    def reset_parameters(self):
+        if self.elementwise_affine:
+            nn.init.ones_(self.weight)
+            if self.bias is not None:
+                nn.init.zeros_(self.bias)
+    def __repr__(self) -> str:
+        s = f"{self.__class__.__name__}({self.hidden_size}"
+        if not self.elementwise_affine:
+            s += f", elementwise_affine={self.elementwise_affine}"
+        s += f", eps={self.eps}"
+        s += f", activation={self.activation}"
+        s += ")"
+        return s
+    def forward(
+        self,
+        x: torch.Tensor,
+        g: torch.Tensor,
+        residual: torch.Tensor | None = None,
+        prenorm: bool = False,
+        residual_in_fp32: bool = False,
+    ) -> torch.Tensor:
+        return layer_norm_gated(
+            x,
+            g,
+            self.weight,
+            self.bias,
+            self.activation,
+            residual=residual,
+            eps=self.eps,
+            prenorm=prenorm,
+            residual_in_fp32=residual_in_fp32,
+        )
+class FusedRMSNormGated(nn.Module):
+    def __init__(
+        self,
+        hidden_size: int,
+        elementwise_affine: bool = True,
+        eps: float = 1e-5,
+        activation: str = "swish",
+        device: torch.device | None = None,
+        dtype: torch.dtype | None = None,
+    ) -> FusedRMSNormGated:
+        factory_kwargs = {"device": device, "dtype": dtype}
+        super().__init__()
+        self.hidden_size = hidden_size
+        self.elementwise_affine = elementwise_affine
+        self.eps = eps
+        self.activation = activation
+        if self.activation not in ["swish", "silu", "sigmoid"]:
+            raise ValueError(f"Unsupported activation: {self.activation}")
+        if elementwise_affine:
+            self.weight = nn.Parameter(torch.empty(hidden_size, **factory_kwargs))
+        else:
+            self.register_parameter("weight", None)
+        self.register_parameter("bias", None)
+        self.reset_parameters()
+    def reset_parameters(self):
+        if self.elementwise_affine:
+            nn.init.ones_(self.weight)
+    def __repr__(self) -> str:
+        s = f"{self.__class__.__name__}({self.hidden_size}"
+        if not self.elementwise_affine:
+            s += f", elementwise_affine={self.elementwise_affine}"
+        s += f", eps={self.eps}"
+        s += f", activation={self.activation}"
+        s += ")"
+        return s
+    def forward(
+        self,
+        x: torch.Tensor,
+        g: torch.Tensor,
+        residual: torch.Tensor | None = None,
+        prenorm: bool = False,
+        residual_in_fp32: bool = False,
+    ) -> torch.Tensor:
+        return rms_norm_gated(
+            x,
+            g,
+            self.weight,
+            self.bias,
+            self.activation,
+            residual=residual,
+            eps=self.eps,
+            prenorm=prenorm,
+            residual_in_fp32=residual_in_fp32,
+        )
+class FusedLayerNormSwishGate(FusedLayerNormGated):
+    def __init__(
+        self,
+        hidden_size: int,
+        elementwise_affine: bool = True,
+        bias: bool = False,
+        eps: float = 1e-5,
+        device: torch.device | None = None,
+        dtype: torch.dtype | None = None,
+    ) -> FusedLayerNormSwishGate:
+        super().__init__(
+            hidden_size=hidden_size,
+            elementwise_affine=elementwise_affine,
+            bias=bias,
+            eps=eps,
+            device=device,
+            dtype=dtype,
+        )
+class FusedRMSNormSwishGate(FusedRMSNormGated):
+    def __init__(
+        self,
+        hidden_size: int,
+        elementwise_affine: bool = True,
+        eps: float = 1e-5,
+        device: torch.device | None = None,
+        dtype: torch.dtype | None = None,
+    ) -> FusedRMSNormSwishGate:
+        super().__init__(
+            hidden_size=hidden_size,
+            elementwise_affine=elementwise_affine,
+            eps=eps,
+            device=device,
+            dtype=dtype,
+        )
+class FusedLayerNormGatedLinear(nn.Module):
+    def __init__(
+        self,
+        hidden_size: int,
+        elementwise_affine: bool = True,
+        eps: float = 1e-5,
+        device: torch.device | None = None,
+        dtype: torch.dtype | None = None,
+    ) -> FusedLayerNormGatedLinear:
+        factory_kwargs = {"device": device, "dtype": dtype}
+        super().__init__()
+        self.hidden_size = hidden_size
+        self.elementwise_affine = elementwise_affine
+        self.eps = eps
+        if elementwise_affine:
+            self.weight = nn.Parameter(torch.empty(hidden_size, **factory_kwargs))
+        else:
+            self.register_parameter("weight", None)
+        self.register_parameter("bias", None)
+        self.reset_parameters()
+    def reset_parameters(self):
+        if self.elementwise_affine:
+            nn.init.ones_(self.weight)
+    def __repr__(self) -> str:
+        s = f"{self.__class__.__name__}({self.hidden_size}"
+        if not self.elementwise_affine:
+            s += f", elementwise_affine={self.elementwise_affine}"
+        s += f", eps={self.eps}"
+        s += ")"
+        return s
+    def forward(
+        self,
+        x: torch.Tensor,
+        g: torch.Tensor,
+        weight: torch.Tensor | None = None,
+        bias: torch.Tensor | None = None,
+        residual: torch.Tensor | None = None,
+        prenorm: bool = False,
+        residual_in_fp32: bool = False,
+    ) -> torch.Tensor:
+        return layer_norm_swish_gate_linear(
+            x,
+            g,
+            self.weight,
+            self.bias,
+            weight,
+            bias,
+            residual=residual,
+            eps=self.eps,
+            prenorm=prenorm,
+            residual_in_fp32=residual_in_fp32,
+        )
+class FusedLayerNormSwishGateLinear(FusedLayerNormGatedLinear):
+    def __init__(
+        self,
+        hidden_size: int,
+        elementwise_affine: bool = True,
+        eps: float = 1e-5,
+        device: torch.device | None = None,
+        dtype: torch.dtype | None = None,
+    ) -> FusedLayerNormSwishGateLinear:
+        super().__init__(
+            hidden_size=hidden_size,
+            elementwise_affine=elementwise_affine,
+            eps=eps,
+            device=device,
+            dtype=dtype,
+        )
+class FusedRMSNormGatedLinear(nn.Module):
+    def __init__(
+        self,
+        hidden_size,
+        elementwise_affine: bool = True,
+        eps: float = 1e-5,
+        device: torch.device | None = None,
+        dtype: torch.dtype | None = None,
+    ) -> FusedRMSNormGatedLinear:
+        factory_kwargs = {"device": device, "dtype": dtype}
+        super().__init__()
+        self.hidden_size = hidden_size
+        self.elementwise_affine = elementwise_affine
+        self.eps = eps
+        self.register_parameter("weight", None)
+        self.register_parameter("bias", None)
+        if elementwise_affine:
+            self.weight = nn.Parameter(torch.empty(hidden_size, **factory_kwargs))
+        self.reset_parameters()
+    def reset_parameters(self):
+        if self.elementwise_affine:
+            nn.init.ones_(self.weight)
+    def __repr__(self) -> str:
+        s = f"{self.__class__.__name__}({self.hidden_size}"
+        if not self.elementwise_affine:
+            s += f", elementwise_affine={self.elementwise_affine}"
+        s += f", eps={self.eps}"
+        s += ")"
+        return s
+    def forward(
+        self,
+        x: torch.Tensor,
+        g: torch.Tensor,
+        weight: torch.Tensor | None = None,
+        bias: torch.Tensor | None = None,
+        residual: torch.Tensor | None = None,
+        prenorm: bool = False,
+        residual_in_fp32: bool = False,
+    ) -> torch.Tensor:
+        return rms_norm_swish_gate_linear(
+            x,
+            g,
+            self.weight,
+            self.bias,
+            weight,
+            bias,
+            residual=residual,
+            eps=self.eps,
+            prenorm=prenorm,
+            residual_in_fp32=residual_in_fp32,
+        )
+class FusedRMSNormSwishGateLinear(FusedRMSNormGatedLinear):
+    def __init__(
+        self,
+        hidden_size: int,
+        elementwise_affine: bool = True,
+        eps: float = 1e-5,
+        device: torch.device | None = None,
+        dtype: torch.dtype | None = None,
+    ) -> FusedRMSNormSwishGateLinear:
+        super().__init__(
+            hidden_size=hidden_size,
+            elementwise_affine=elementwise_affine,
+            eps=eps,
+            device=device,
+            dtype=dtype,
+        )

build/torch-cuda/modules/grpo.py ADDED Viewed

	@@ -0,0 +1,421 @@

+# Copyright (c) 2023-2026, Songlin Yang, Yu Zhang, Zhiyuan Li
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+# For a list of all contributors, visit:
+#   https://github.com/fla-org/flash-linear-attention/graphs/contributors
+# modified from https://github.com/mdy666/mdy_triton/blob/e0a856347bd988e05e0152332bba35f1d33c5b1f/others/grpo/grpo_loss.ipynb
+# XHS ID: blueeeee
+# https://github.com/huggingface/trl/blob/main/trl/trainer/grpo_trainer.py
+"""
+# Get the per-token log probabilities for the completions for the model and the reference model
+    def _get_per_token_logps(self, model, input_ids, attention_mask, logits_to_keep):
+        # We add 1 to `logits_to_keep` because the last logits of the sequence is later excluded
+        logits = model(input_ids=input_ids, attention_mask=attention_mask, logits_to_keep=logits_to_keep + 1).logits
+        logits = logits[:, :-1, :]  # (B, L-1, V), exclude the last logit: it corresponds to the next token pred
+        input_ids = input_ids[:, -logits_to_keep:]
+        # For transformers<=4.48, logits_to_keep argument isn't supported, so here we drop logits ourselves.
+        # See https://github.com/huggingface/trl/issues/2770
+        logits = logits[:, -logits_to_keep:]
+        return selective_log_softmax(logits, input_ids)  #  compute logprobs for the input tokens
+    def compute_loss(self, model, inputs, return_outputs=False, num_items_in_batch=None):
+        if return_outputs:
+            raise ValueError("The GRPOTrainer does not support returning outputs")
+        # Compute the per-token log probabilities for the model
+        prompt_ids, prompt_mask = inputs["prompt_ids"], inputs["prompt_mask"]
+        completion_ids, completion_mask = inputs["completion_ids"], inputs["completion_mask"]
+        input_ids = torch.cat([prompt_ids, completion_ids], dim=1)
+        attention_mask = torch.cat([prompt_mask, completion_mask], dim=1)
+        logits_to_keep = completion_ids.size(1)  # we only need to compute the logits for the completion tokens
+        per_token_logps = self._get_per_token_logps(model, input_ids, attention_mask, logits_to_keep)
+        # Compute the KL divergence between the model and the reference model
+        ref_per_token_logps = inputs["ref_per_token_logps"]
+        per_token_kl = torch.exp(ref_per_token_logps - per_token_logps) - (ref_per_token_logps - per_token_logps) - 1
+        # x - x.detach() allows for preserving gradients from x
+        advantages = inputs["advantages"]
+        per_token_loss = torch.exp(per_token_logps - per_token_logps.detach()) * advantages.unsqueeze(1)
+        per_token_loss = -(per_token_loss - self.beta * per_token_kl)
+        loss = ((per_token_loss * completion_mask).sum(dim=1) / completion_mask.sum(dim=1)).mean()
+        # Log the metrics
+        completion_length = self.accelerator.gather_for_metrics(completion_mask.sum(1)).float().mean().item()
+        self._metrics["completion_length"].append(completion_length)
+        mean_kl = ((per_token_kl * completion_mask).sum(dim=1) / completion_mask.sum(dim=1)).mean()
+        self._metrics["kl"].append(self.accelerator.gather_for_metrics(mean_kl).mean().item())
+        return loss
+"""
+import torch
+import triton
+import triton.language as tl
+from ..modules.backends import dispatch
+from ..ops.utils.op import exp, log
+from ..utils import IS_AMD, autotune_cache_kwargs, input_guard
+NUM_WARPS_AUTOTUNE = [4, 8, 16] if IS_AMD else [4, 8, 16, 32]
+@triton.autotune(
+    configs=[
+        triton.Config({'BLOCK_SIZE': BLOCK_SIZE},  num_warps=NUM_WARPS, num_stages=NUM_STAGES)
+        for BLOCK_SIZE in [1024, 2048, 4096, 8192]
+        for NUM_WARPS in NUM_WARPS_AUTOTUNE
+        for NUM_STAGES in [1, 2, 4]
+    ],
+    key=['B', 'N'],
+    **autotune_cache_kwargs,
+)
+@triton.jit
+def grpo_fwd_kernel(
+    logits_ptr,
+    ref_logp_ptr,
+    input_ids_ptr,
+    advantages_ptr,
+    completion_mask_ptr,
+    loss_ptr,
+    lse_ptr,
+    beta,
+    save_kl: tl.constexpr,
+    B,
+    M,
+    N,
+    L,
+    start_idx,
+    BLOCK_SIZE: tl.constexpr,
+):
+    row_idx = tl.program_id(0)
+    off_b = row_idx // L
+    N = tl.cast(N, tl.int64)
+    loss_ptr += row_idx
+    completion_mask_ptr += row_idx
+    not_skip = tl.load(completion_mask_ptr).to(tl.int1)
+    if not_skip == 1:
+        ref_logp_ptr += row_idx
+        lse_ptr += row_idx
+        advantages_ptr += off_b
+        logits_ptr += N * (row_idx + off_b)
+        input_ids_ptr += row_idx + (off_b+1) * start_idx
+        base_cols = tl.arange(0, BLOCK_SIZE)
+        m_i = -float("inf")
+        l_i = 0.0
+        for start_n in tl.range(0, N, BLOCK_SIZE):
+            cols = start_n + base_cols
+            mask = cols < N
+            logits = tl.load(logits_ptr+cols, mask=mask, other=-float('inf')).to(tl.float32)
+            m_ij = tl.max(logits)
+            new_m_i = tl.maximum(m_i, m_ij)
+            l_i = l_i * exp(m_i - new_m_i) + tl.sum(exp(logits - new_m_i))
+            m_i = new_m_i
+        lse = log(l_i) + m_i
+        idx = tl.load(input_ids_ptr)
+        x = tl.load(logits_ptr+idx).to(tl.float32)
+        advantage = tl.load(advantages_ptr).to(tl.float32)
+        ref_logp = tl.load(ref_logp_ptr)
+        logp = x - lse
+        diff = ref_logp - logp
+        kl = exp(diff) - diff - 1
+        loss = kl * beta - advantage
+        tl.store(loss_ptr, loss.to(loss_ptr.dtype.element_ty))
+        tl.store(lse_ptr, lse.to(lse_ptr.dtype.element_ty))
+        if save_kl:
+            tl.store(loss_ptr+M, kl.to(loss_ptr.dtype.element_ty))
+    else:
+        # store 0
+        tl.store(loss_ptr, 0.0)
+        if save_kl:
+            tl.store(loss_ptr+M, 0.0)
+@triton.autotune(
+    configs=[
+        triton.Config({},  num_warps=NUM_WARPS, num_stages=NUM_STAGES)
+        for NUM_WARPS in [32]
+        for NUM_STAGES in [4]
+    ],
+    key=['B', 'N'],
+    **autotune_cache_kwargs,
+)
+@triton.jit
+def grpo_bwd_kernel(
+    dloss_ptr,
+    dlogits_ptr,
+    logits_ptr,
+    ref_logp_ptr,
+    input_ids_ptr,
+    advantages_ptr,
+    completion_mask_ptr,
+    lse_ptr,
+    beta,
+    B,
+    N,
+    L,
+    start_idx,
+    BLOCK_SIZE: tl.constexpr,
+):
+    row_idx = tl.program_id(0)  # B*L
+    off_b = row_idx // L
+    N = tl.cast(N, tl.int64)
+    dlogits_ptr += N * (row_idx + off_b)
+    base_cols = tl.arange(0, BLOCK_SIZE)
+    completion_mask_ptr += row_idx
+    not_skip = tl.load(completion_mask_ptr).to(tl.int1)
+    if not_skip == 1:
+        lse_ptr += row_idx
+        dloss_ptr += row_idx
+        advantages_ptr += off_b
+        ref_logp_ptr += row_idx
+        logits_ptr += N * (row_idx + off_b)
+        input_ids_ptr += row_idx + (off_b+1) * start_idx
+        dloss = tl.load(dloss_ptr).to(tl.float32)
+        lse = tl.load(lse_ptr).to(tl.float32)
+        idx = tl.load(input_ids_ptr)
+        x = tl.load(logits_ptr+idx).to(tl.float32)
+        advantage = tl.load(advantages_ptr).to(tl.float32)
+        ref_logp = tl.load(ref_logp_ptr)
+        # Need for in-place grad.
+        tl.debug_barrier()
+        logp = x - lse
+        dlogp = (beta * (-1.0 * exp(ref_logp - logp) + 1)
+                 - advantage) * dloss
+        for start_n in tl.range(0, N, BLOCK_SIZE):
+            cols = start_n + base_cols
+            mask = cols < N
+            logits = tl.load(logits_ptr+cols, mask=mask, other=-float('inf')).to(tl.float32)
+            probs = exp(logits - lse)
+            dlogits = tl.where(cols == idx, 1-probs, -probs) * dlogp
+            tl.store(dlogits_ptr+cols, dlogits.to(dlogits_ptr.dtype.element_ty), mask=mask)
+    else:
+        dlogits = tl.zeros((BLOCK_SIZE,), dtype=tl.float32)
+        for start_n in tl.range(0, N, BLOCK_SIZE):
+            cols = start_n + base_cols
+            mask = cols < N
+            tl.store(dlogits_ptr+cols, dlogits.to(dlogits_ptr.dtype.element_ty), mask=mask)
+class GrpoLoss(torch.autograd.Function):
+    @input_guard
+    @staticmethod
+    def forward(ctx, logits, ref_logp, input_ids, advantages, beta, completion_mask, save_kl, inplace=True):
+        ctx.input_shape = logits.shape
+        B, L_ADD_1, N = ctx.input_shape
+        L = L_ADD_1 - 1
+        M = B * L
+        input_ids_start_index = input_ids.size(1) - L
+        if not save_kl:
+            loss = torch.empty(B, L, device=logits.device, dtype=torch.float32)
+        else:
+            loss = torch.empty(B*2, L, device=logits.device, dtype=torch.float32)
+        lse = torch.empty(B, L, device=logits.device, dtype=torch.float32)
+        if completion_mask is None:
+            completion_mask = torch.ones(B, L, device=logits.device, dtype=torch.int32)
+        else:
+            loss[:B].masked_fill_(completion_mask.logical_not(), 0.0)
+        grpo_fwd_kernel[(M,)](
+            logits_ptr=logits,
+            ref_logp_ptr=ref_logp,
+            input_ids_ptr=input_ids,
+            advantages_ptr=advantages,
+            completion_mask_ptr=completion_mask,
+            loss_ptr=loss,
+            lse_ptr=lse,
+            beta=beta,
+            save_kl=save_kl,
+            B=B, M=M, N=N, L=L,
+            start_idx=input_ids_start_index,
+        )
+        ctx.beta = beta
+        ctx.save_for_backward(lse, logits, input_ids, advantages, completion_mask)
+        ctx.ref_logp = ref_logp
+        ctx.inplace = inplace
+        return loss
+    @input_guard
+    @staticmethod
+    def backward(ctx, dloss):
+        # The grad of logits comes from two parts, the reward part and the kl part
+        lse, logits, input_ids, advantages, completion_mask = ctx.saved_tensors
+        inplace = ctx.inplace
+        B, L_ADD_1, N = ctx.input_shape
+        L = L_ADD_1 - 1
+        M = B * L
+        input_ids_start_index = input_ids.size(1) - L
+        # B, L_ADD_1, N
+        dlogits = logits if inplace else torch.empty_like(logits)
+        BN = min(65536, triton.next_power_of_2(N))
+        grpo_bwd_kernel[(M,)](
+            dloss_ptr=dloss,
+            dlogits_ptr=dlogits,
+            logits_ptr=logits,
+            ref_logp_ptr=ctx.ref_logp,
+            input_ids_ptr=input_ids,
+            advantages_ptr=advantages,
+            completion_mask_ptr=completion_mask,
+            lse_ptr=lse,
+            beta=ctx.beta,
+            B=B, N=N, L=L,
+            BLOCK_SIZE=BN,
+            start_idx=input_ids_start_index,
+        )
+        # The last token in the completion is not used in the loss computation
+        # and therefore its gradient should be set to 0
+        dlogits[:, -1, :].fill_(0.0)
+        return dlogits.view(*ctx.input_shape), None, None, None, None, None, None, None
+@dispatch('modules')
+def fused_grpo_loss(logits, ref_logp, input_ids, advantages,
+                    beta=0.1, completion_mask=None, save_kl=False, inplace=False) -> torch.Tensor:
+    '''
+    compute grpo loss, save memory(no addition usage) and fast speed(6X for A800)
+    Args:
+        logtits: Tensor, [B, L+1, vocab_size], the origin output of model, it's not logits[:, :-1]
+        ref_logp: Tensor, [B, L], the origin output of model, it's not ref_logits[:, :-1]
+        input_ids: Tensor, [B, K+L], it's prompt_completion_id, it contains the prompt ids and output ids
+        advantages: Tensor, [B], the advantages of each prompt
+        beta: float, the weight of kl loss
+        completion_mask: Tensor, loss mask
+        save_kl: bool, if true will save kl
+    Retutn:
+        loss: Tensor, [B, L], the loss of grpo, it contains the advantage part and kl part
+    NOTE: logits(ref_logits) is computed by these steps
+        logits_to_keep = completion_ids.size(1)
+        def get_per_token_logits(model, input_ids, attention_mask, logits_to_keep):
+            # We add 1 to `logits_to_keep` because the last logits of the sequence is later excluded
+            logits = model(
+                input_ids=input_ids, attention_mask=attention_mask, logits_to_keep=logits_to_keep + 1
+            ).logits
+            return logits
+        logits = get_per_token_logits(model, prompt_completion_ids, attention_mask, logits_to_keep)
+    '''
+    out = GrpoLoss.apply(logits, ref_logp, input_ids, advantages, beta, completion_mask, save_kl, inplace)
+    if not save_kl:
+        return out
+    else:
+        return out.chunk(2, axis=0)
+def grpo_loss_torch(logits, ref_logp, input_ids, advantages, beta=0.1, completion_mask=None, save_kl=False):
+    def get_log_probs(logits, input_ids):
+        per_token_logps = []
+        for logits_row, input_ids_row in zip(logits, input_ids[:, -logits.size(1):], strict=False):
+            log_probs = logits_row.log_softmax(dim=-1)
+            token_log_prob = torch.gather(log_probs, dim=1, index=input_ids_row.unsqueeze(1)).squeeze(1)
+            per_token_logps.append(token_log_prob)
+        return torch.stack(per_token_logps)
+    logits = logits[:, :-1]
+    per_token_logps = get_log_probs(logits, input_ids)
+    ref_per_token_logps = ref_logp
+    per_token_kl = torch.exp(ref_per_token_logps - per_token_logps) - (ref_per_token_logps - per_token_logps) - 1
+    per_token_loss = torch.exp(per_token_logps - per_token_logps.detach()) * advantages.unsqueeze(1)
+    per_token_loss = -(per_token_loss - beta * per_token_kl)
+    if completion_mask is not None:
+        per_token_loss *= completion_mask
+        if save_kl:
+            per_token_kl *= completion_mask
+    return per_token_loss if not save_kl else (per_token_loss, per_token_kl)
+@torch.compile(fullgraph=True)
+def grpo_loss_with_old_logps(
+    logps: torch.Tensor,
+    ref_logps: torch.Tensor,
+    old_logps: torch.Tensor,
+    pad_mask: torch.Tensor,
+    logits_to_keep: int,
+    rewards: torch.Tensor,
+    beta: float = 0.2,
+    epsilon: float = 0.2,
+):
+    """
+    Compute the GRPO (Group Relative Policy Optimization) loss.
+    Args:
+        logps (torch.Tensor): [Batch, Token_length] Log probabilities of the current policy.
+        ref_logps (torch.Tensor):[Batch, Token_length]  Log probabilities of the reference policy.
+        old_logps (torch.Tensor): [Batch, Token_length] Log probabilities of the old policy.
+        completion_ids (torch.Tensor): [Batch, Token_length] Completion token IDs (bool).
+        pad_token_id: Pad token ID.
+        logits_to_keep (int): Number of logits to keep for masking.
+        rewards (torch.Tensor): [Batch] Rewards for each generation.
+        beta (float) = 0.2: A hyperparameter for weighting the KL divergence term.
+        epsilon (float) = 0.2: An float hyperparameter for clipping the importance weights.
+    Returns:
+        torch.Tensor: The computed GRPO loss.
+    """
+    B = logps.shape[0]
+    assert B > 1, "Batch * Num generations should be greater than 1"
+    rewards_shaped = rewards.view(-1, B)  # B,num_generations
+    advantages = (rewards_shaped - rewards_shaped.mean(dim=1, keepdim=True)) / \
+        (rewards_shaped.std(dim=1, keepdim=True) + 1e-8)
+    advantages = advantages.view(-1)  # B*num_generations
+    # Calculate the per - token KL divergence
+    per_token_kl = torch.exp(ref_logps - logps) - (ref_logps - logps) - 1
+    # Calculate the ratio of probabilities (importance weights)
+    # Importance weights are calculated as exp(log_pi_theta - log_pi_theta_old)
+    importance_weights = torch.exp(logps - old_logps)
+    # Clip the importance weights to the range [1 - epsilon, 1 + epsilon]
+    importance_weights_clipped = torch.clamp(importance_weights, 1 - epsilon, 1 + epsilon)
+    # Create a completion mask. It checks which positions are valid based on logits_to_keep
+    completion_mask = torch.arange(logits_to_keep, device=logps.device)[None, :] >= 0
+    # Combine the completion mask and padding mask
+    completion_mask = completion_mask & pad_mask  # Ensure matching shape
+    # Add an extra dimension to advantages to match the shape for element - wise multiplication
+    advantages = advantages.unsqueeze(1)
+    # Calculate the per - token loss. It takes the minimum of the unclipped and clipped importance weights
+    # and subtracts the KL divergence term weighted by beta, then multiplies by the completion mask
+    token_loss = -(torch.min(advantages * importance_weights, advantages *
+                   importance_weights_clipped) - beta * per_token_kl) * completion_mask
+    # Calculate the final loss by summing the token losses and normalizing by the number of valid tokens
+    loss = -token_loss.sum() / completion_mask.sum()
+    return loss

build/torch-cuda/modules/l2norm.py ADDED Viewed

	@@ -0,0 +1,288 @@

+# Copyright (c) 2023-2026, Songlin Yang, Yu Zhang, Zhiyuan Li
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+# For a list of all contributors, visit:
+#   https://github.com/fla-org/flash-linear-attention/graphs/contributors
+import torch
+import torch.nn as nn
+import triton
+import triton.language as tl
+from ..ops.utils.cache import fla_cache_autotune
+from ..utils import IS_AMD, autotune_cache_kwargs, input_guard
+BT_LIST = [8, 16, 32, 64, 128]
+NUM_WARPS_AUTOTUNE = [1, 2, 4, 8, 16] if IS_AMD else [1, 2, 4, 8, 16, 32]
+@triton.autotune(
+    configs=[triton.Config({}, num_warps=num_warps) for num_warps in NUM_WARPS_AUTOTUNE],
+    key=["D"],
+    **autotune_cache_kwargs,
+)
+@triton.jit
+def l2norm_fwd_kernel1(
+    x,
+    y,
+    rstd,
+    eps,
+    D,
+    BD: tl.constexpr,
+):
+    i_t = tl.program_id(0)
+    x += i_t * D
+    y += i_t * D
+    # Compute mean and variance
+    cols = tl.arange(0, BD)
+    mask = cols < D
+    b_x = tl.load(x + cols, mask=mask, other=0.0).to(tl.float32)
+    b_rstd = 1 / tl.sqrt(tl.sum(b_x * b_x) + eps)
+    b_y = b_x * b_rstd
+    tl.store(y + cols, b_y, mask=mask)
+    tl.store(rstd + i_t, b_rstd)
+@triton.autotune(
+    configs=[triton.Config({}, num_warps=num_warps) for num_warps in NUM_WARPS_AUTOTUNE],
+    key=["D"],
+    **autotune_cache_kwargs,
+)
+@triton.jit
+def l2norm_bwd_kernel1(
+    y,
+    rstd,
+    dy,
+    dx,
+    eps,
+    D,
+    BD: tl.constexpr,
+):
+    i_t = tl.program_id(0)
+    y += i_t * D
+    dx += i_t * D
+    dy += i_t * D
+    cols = tl.arange(0, BD)
+    mask = cols < D
+    b_y = tl.load(y + cols, mask=mask, other=0.0).to(tl.float32)
+    b_rstd = tl.load(rstd + i_t).to(tl.float32)
+    b_dy = tl.load(dy + cols, mask=mask, other=0.0).to(tl.float32)
+    b_dx = b_dy * b_rstd - tl.sum(b_dy * b_y) * b_y * b_rstd
+    tl.store(dx + cols, b_dx, mask=mask)
+@fla_cache_autotune(
+    configs=[triton.Config({"BT": BT}, num_warps=num_warps) for num_warps in [1, 2, 4, 8, 16] for BT in BT_LIST],
+    key=["D", "NB"],
+    **autotune_cache_kwargs,
+)
+@triton.jit(do_not_specialize=["T"])
+def l2norm_fwd_kernel(
+    x,
+    y,
+    rstd,
+    eps,
+    T,
+    D: tl.constexpr,
+    BD: tl.constexpr,
+    NB: tl.constexpr,
+    BT: tl.constexpr,
+):
+    i_t = tl.program_id(0)
+    p_x = tl.make_block_ptr(x, (T, D), (D, 1), (i_t * BT, 0), (BT, BD), (1, 0))
+    p_y = tl.make_block_ptr(y, (T, D), (D, 1), (i_t * BT, 0), (BT, BD), (1, 0))
+    p_rstd = tl.make_block_ptr(rstd, (T,), (1,), (i_t * BT,), (BT,), (0,))
+    b_x = tl.load(p_x, boundary_check=(0, 1)).to(tl.float32)
+    b_rstd = 1 / tl.sqrt(tl.sum(b_x * b_x, 1) + eps)
+    b_y = b_x * b_rstd[:, None]
+    tl.store(p_y, b_y.to(p_y.dtype.element_ty), boundary_check=(0, 1))
+    tl.store(p_rstd, b_rstd.to(p_rstd.dtype.element_ty), boundary_check=(0,))
+@fla_cache_autotune(
+    configs=[triton.Config({"BT": BT}, num_warps=num_warps) for num_warps in [1, 2, 4, 8, 16] for BT in BT_LIST],
+    key=["D", "NB"],
+    **autotune_cache_kwargs,
+)
+@triton.jit(do_not_specialize=["T"])
+def l2norm_bwd_kernel(
+    y,
+    rstd,
+    dy,
+    dx,
+    eps,
+    T,
+    D: tl.constexpr,
+    BD: tl.constexpr,
+    NB: tl.constexpr,
+    BT: tl.constexpr,
+):
+    i_t = tl.program_id(0)
+    p_y = tl.make_block_ptr(y, (T, D), (D, 1), (i_t * BT, 0), (BT, BD), (1, 0))
+    p_rstd = tl.make_block_ptr(rstd, (T,), (1,), (i_t * BT,), (BT,), (0,))
+    p_dy = tl.make_block_ptr(dy, (T, D), (D, 1), (i_t * BT, 0), (BT, BD), (1, 0))
+    p_dx = tl.make_block_ptr(dx, (T, D), (D, 1), (i_t * BT, 0), (BT, BD), (1, 0))
+    b_y = tl.load(p_y, boundary_check=(0, 1)).to(tl.float32)
+    b_rstd = tl.load(p_rstd, boundary_check=(0,)).to(tl.float32)
+    b_dy = tl.load(p_dy, boundary_check=(0, 1)).to(tl.float32)
+    b_dx = b_dy * b_rstd[:, None] - tl.sum(b_dy * b_y, 1)[:, None] * b_y * b_rstd[:, None]
+    tl.store(p_dx, b_dx.to(p_dx.dtype.element_ty), boundary_check=(0, 1))
+def l2norm_fwd(
+    x: torch.Tensor,
+    eps: float = 1e-6,
+    output_dtype: torch.dtype | None = None,
+):
+    x_shape_og = x.shape
+    x = x.view(-1, x.shape[-1])
+    # allocate output
+    if output_dtype is None:
+        y = torch.empty_like(x)
+    else:
+        y = torch.empty_like(x, dtype=output_dtype)
+    assert y.stride(-1) == 1
+    T, D = x.shape[0], x.shape[-1]
+    # Less than 64KB per feature: enqueue fused kernel
+    MAX_FUSED_SIZE = 65536 // x.element_size()
+    BD = min(MAX_FUSED_SIZE, triton.next_power_of_2(D))
+    if D > BD:
+        raise RuntimeError("This layer doesn't support feature dim >= 64KB.")
+    rstd = torch.empty((T,), dtype=torch.float32, device=x.device)
+    if D <= 512:
+        # NOTE(tylerr): Avoid excessive recompilation and autotuning by tolerating a larger range
+        # of T before recompiling the kernel.
+        # NB = triton.cdiv(T, 2048)
+        NB = triton.cdiv(T, 2048 * 32)
+        def grid(meta):
+            return (triton.cdiv(T, meta["BT"]),)
+        l2norm_fwd_kernel[grid](
+            x=x,
+            y=y,
+            rstd=rstd,
+            eps=eps,
+            T=T,
+            D=D,
+            BD=BD,
+            NB=NB,
+        )
+    else:
+        l2norm_fwd_kernel1[(T,)](
+            x=x,
+            y=y,
+            rstd=rstd,
+            eps=eps,
+            D=D,
+            BD=BD,
+        )
+    return y.view(x_shape_og), rstd.view(x_shape_og[:-1])
+def l2norm_bwd(
+    y: torch.Tensor,
+    rstd: torch.Tensor,
+    dy: torch.Tensor,
+    eps: float = 1e-6,
+):
+    y_shape_og = y.shape
+    y = y.view(-1, dy.shape[-1])
+    dy = dy.view(-1, dy.shape[-1])
+    assert dy.shape == y.shape
+    # allocate output
+    dx = torch.empty_like(y)
+    T, D = y.shape[0], y.shape[-1]
+    # Less than 64KB per feature: enqueue fused kernel
+    MAX_FUSED_SIZE = 65536 // y.element_size()
+    BD = min(MAX_FUSED_SIZE, triton.next_power_of_2(D))
+    if D > BD:
+        raise RuntimeError("This layer norm doesn't support feature dim >= 64KB.")
+    if D <= 512:
+        # NOTE(tylerr): Avoid excessive recompilation and autotuning by tolerating a larger range
+        # of T before recompiling the kernel.
+        # NB = triton.cdiv(T, 2048)
+        NB = triton.cdiv(T, 2048 * 32)
+        def grid(meta):
+            return (triton.cdiv(T, meta["BT"]),)
+        l2norm_bwd_kernel[grid](
+            y=y,
+            rstd=rstd,
+            dy=dy,
+            dx=dx,
+            eps=eps,
+            T=T,
+            D=D,
+            BD=BD,
+            NB=NB,
+        )
+    else:
+        l2norm_bwd_kernel1[(T,)](
+            y=y,
+            rstd=rstd,
+            dy=dy,
+            dx=dx,
+            eps=eps,
+            D=D,
+            BD=BD,
+        )
+    return dx.view(y_shape_og)
+class L2NormFunction(torch.autograd.Function):
+    @staticmethod
+    @input_guard
+    def forward(
+        ctx,
+        x,
+        eps=1e-6,
+        output_dtype=None,
+    ):
+        y, rstd = l2norm_fwd(x, eps, output_dtype)
+        ctx.eps = eps
+        ctx.x_dtype = x.dtype
+        ctx.save_for_backward(y, rstd)
+        return y
+    @staticmethod
+    @input_guard
+    def backward(ctx, dy):
+        y, rstd = ctx.saved_tensors
+        dx = l2norm_bwd(y, rstd, dy, ctx.eps)
+        return dx, None, None
+def l2norm(
+    x: torch.Tensor,
+    eps: float = 1e-6,
+    output_dtype: torch.dtype | None = None,
+) -> torch.Tensor:
+    return L2NormFunction.apply(x, eps, output_dtype)
+l2_norm = l2norm
+class L2Norm(nn.Module):
+    def __init__(
+        self,
+        eps: float = 1e-6,
+        output_dtype: torch.dtype | None = None,
+    ):
+        super().__init__()
+        self.eps = eps
+        self.output_dtype = output_dtype
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        return l2norm(x, self.eps, self.output_dtype)

build/torch-cuda/modules/l2warp.py ADDED Viewed

	@@ -0,0 +1,51 @@

+# Copyright (c) 2023-2026, Songlin Yang, Yu Zhang, Zhiyuan Li
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+# For a list of all contributors, visit:
+#   https://github.com/fla-org/flash-linear-attention/graphs/contributors
+import torch
+class L2Wrap(torch.autograd.Function):
+    r"""
+    This class of penalty prevents the model from becoming overconfident,
+    thereby mitigating precision loss in BF16.
+    This version is memory-optimized by not storing the full logits tensor.
+    """
+    @staticmethod
+    def forward(
+        ctx,
+        loss: torch.Tensor,
+        logits: torch.Tensor,
+        l2_penalty_factor: float = 1e-4,
+    ) -> torch.Tensor:
+        """
+        Args:
+            loss (torch.Tensor):
+                The already-reduced (scalar) loss to wrap.
+            logits (torch.Tensor):
+                The logits of shape `[B, T, V]`.
+            l2_penalty_factor (float, Optional):
+                The strength of the L2 penalty on the max logit. Default: 1e-4.
+        """
+        maxx, ids = torch.max(logits, dim=-1, keepdim=True)
+        ctx.logits_shape = logits.shape
+        factor = l2_penalty_factor / (logits.shape[0] * logits.shape[1])
+        maxx = maxx * factor
+        ctx.save_for_backward(maxx, ids)
+        return loss
+    @staticmethod
+    def backward(ctx, grad_output: torch.Tensor):
+        maxx, ids = ctx.saved_tensors
+        glogits = torch.zeros(ctx.logits_shape, device=grad_output.device, dtype=grad_output.dtype)
+        # an autograd.Function must scale its input gradients by the upstream gradient; fold the
+        # scalar grad_output into the sparse maxx to avoid a second full-size logits allocation
+        glogits.scatter_(-1, ids, maxx * grad_output)
+        return grad_output, glogits, None
+l2_warp = L2Wrap.apply

build/torch-cuda/modules/layernorm.py ADDED Viewed

	@@ -0,0 +1,1472 @@

+# Copyright (c) 2023-2026, Songlin Yang, Yu Zhang, Zhiyuan Li
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+# For a list of all contributors, visit:
+#   https://github.com/fla-org/flash-linear-attention/graphs/contributors
+#
+# Copyright (c) 2023, Tri Dao
+# https://github.com/state-spaces/mamba/blob/fb7b5310fa865dbd62aa059b1e26f2b431363e2a/mamba_ssm/ops/triton/layernorm.py
+# Implement residual + layer_norm / rms_norm.
+# Based on the Triton LayerNorm tutorial: https://triton-lang.org/main/getting-started/tutorials/05-layer-norm.html
+# For the backward pass, we keep weight_grad and bias_grad in registers and accumulate.
+# This is faster for dimensions up to 8k, but after that it's much slower due to register spilling.
+# The models we train have hidden dim up to 8k anyway (e.g. Llama 70B), so this is fine.
+from __future__ import annotations
+from functools import partial
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+import triton
+import triton.language as tl
+from einops import rearrange
+from torch.distributed import DeviceMesh
+from torch.distributed.tensor import Replicate, Shard, distribute_module
+from torch.distributed.tensor.parallel import ParallelStyle
+from ..modules.backends import dispatch
+from ..utils import autotune_cache_kwargs, get_multiprocessor_count, input_guard
+try:
+    from torch.distributed.tensor import DTensor
+except (ImportError, AttributeError):
+    DTensor = None
+def layer_norm_ref(
+    x: torch.Tensor,
+    weight: torch.Tensor,
+    bias: torch.Tensor,
+    residual: torch.Tensor = None,
+    eps: float = 1e-5,
+    prenorm: bool = False,
+    upcast: bool = False,
+):
+    dtype = x.dtype
+    if upcast:
+        weight = weight.float()
+        bias = bias.float() if bias is not None else None
+    if upcast:
+        x = x.float()
+        residual = residual.float() if residual is not None else residual
+    if residual is not None:
+        x = (x + residual).to(x.dtype)
+    out = F.layer_norm(x.to(weight.dtype), x.shape[-1:], weight=weight, bias=bias, eps=eps).to(
+        dtype,
+    )
+    return out if not prenorm else (out, x)
+def rms_norm_ref(
+    x: torch.Tensor,
+    weight: torch.Tensor,
+    bias: torch.Tensor,
+    residual: torch.Tensor = None,
+    eps: float = 1e-5,
+    prenorm: bool = False,
+    upcast: bool = False,
+):
+    dtype = x.dtype
+    if upcast:
+        weight = weight.float()
+        bias = bias.float() if bias is not None else None
+    if upcast:
+        x = x.float()
+        residual = residual.float() if residual is not None else residual
+    if residual is not None:
+        x = (x + residual).to(x.dtype)
+    rstd = 1 / torch.sqrt((x.square()).mean(dim=-1, keepdim=True) + eps)
+    out = (x * rstd * weight) + bias if bias is not None else (x * rstd * weight)
+    out = out.to(dtype)
+    return out if not prenorm else (out, x)
+def group_norm_ref(
+    x: torch.Tensor,
+    weight: torch.Tensor,
+    bias: torch.Tensor,
+    num_groups: int,
+    residual: torch.Tensor = None,
+    eps: float = 1e-5,
+    is_rms_norm: bool = False,
+    prenorm: bool = False,
+    upcast: bool = False,
+):
+    dtype = x.dtype
+    if upcast:
+        weight = weight.float()
+        bias = bias.float() if bias is not None else None
+    if upcast:
+        x = x.float()
+        residual = residual.float() if residual is not None else residual
+    if residual is not None:
+        x = (x + residual).to(x.dtype)
+    residual = x
+    x, weight = [
+        rearrange(data, "... (g d) -> ... g d", g=num_groups) for data in (x, weight)
+    ]
+    if bias is not None:
+        bias = rearrange(bias, '... (g d) -> ... g d', g=num_groups)
+    if not is_rms_norm:
+        mean = x.mean(dim=-1, keepdim=True)
+        x = x - mean
+    rstd = 1 / torch.sqrt((x.square()).mean(dim=-1, keepdim=True) + eps)
+    out = (x * rstd * weight) + bias if bias is not None else (x * rstd * weight)
+    out = rearrange(out, "... g d -> ... (g d)")
+    out = out.to(dtype)
+    return out if not prenorm else (out, residual)
+class GroupNormRef(nn.Module):
+    def __init__(
+        self,
+        num_groups: int,
+        hidden_size: int,
+        elementwise_affine: bool = True,
+        bias: bool = False,
+        eps: float = 1e-5,
+        is_rms_norm: bool = False,
+    ) -> GroupNormRef:
+        super().__init__()
+        if hidden_size % num_groups != 0:
+            raise ValueError('num_channels must be divisible by num_groups')
+        self.num_groups = num_groups
+        self.hidden_size = hidden_size
+        self.elementwise_affine = elementwise_affine
+        self.eps = eps
+        self.is_rms_norm = is_rms_norm
+        self.register_parameter("weight", None)
+        self.register_parameter("bias", None)
+        if elementwise_affine:
+            self.weight = nn.Parameter(torch.empty(hidden_size))
+            if bias:
+                self.bias = nn.Parameter(torch.empty(hidden_size))
+        self.reset_parameters()
+    def reset_parameters(self):
+        if self.elementwise_affine:
+            nn.init.ones_(self.weight)
+            if self.bias is not None:
+                nn.init.zeros_(self.bias)
+    def __repr__(self) -> str:
+        s = f"{self.__class__.__name__}({self.num_groups}, {self.hidden_size}"
+        if not self.elementwise_affine:
+            s += f", elementwise_affine={self.elementwise_affine}"
+        if self.is_rms_norm:
+            s += f", is_rms_norm={self.is_rms_norm}"
+        s += f", eps={self.eps}"
+        s += ")"
+        return s
+    def forward(self, x, residual=None, prenorm=False):
+        return group_norm_ref(
+            x,
+            self.weight,
+            self.bias,
+            num_groups=self.num_groups,
+            residual=residual,
+            eps=self.eps,
+            is_rms_norm=self.is_rms_norm,
+            prenorm=prenorm,
+            upcast=True,
+        )
+@triton.autotune(
+    configs=[
+        triton.Config({'BT': BT}, num_warps=num_warps)
+        for BT in [32, 64, 128]
+        for num_warps in [2, 4, 8]
+    ],
+    key=['D', 'NB', 'HAS_RESIDUAL', 'STORE_RESIDUAL_OUT', 'IS_RMS_NORM'],
+    **autotune_cache_kwargs,
+)
+@triton.jit
+def layer_norm_fwd_kernel(
+    x,  # pointer to the input
+    y,  # pointer to the output
+    w,  # pointer to the weights
+    b,  # pointer to the biases
+    res,  # pointer to the res
+    res_out,  # pointer to the res
+    mean,  # pointer to the mean
+    rstd,  # pointer to the 1/std
+    eps,  # epsilon to avoid division by zero
+    T,
+    G: tl.constexpr,
+    D: tl.constexpr,
+    BT: tl.constexpr,
+    BD: tl.constexpr,
+    NB: tl.constexpr,
+    IS_RMS_NORM: tl.constexpr,
+    HAS_RESIDUAL: tl.constexpr,
+    STORE_RESIDUAL_OUT: tl.constexpr,
+    HAS_WEIGHT: tl.constexpr,
+    HAS_BIAS: tl.constexpr,
+):
+    i_t = tl.program_id(0)
+    o_t = i_t * BT + tl.arange(0, BT)
+    o_g = o_t % G
+    o_d = tl.arange(0, BD)
+    m_d = o_d < D
+    p_x = tl.make_block_ptr(x, (T, D), (D, 1), (i_t * BT, 0), (BT, BD), (1, 0))
+    b_x = tl.load(p_x, boundary_check=(0, 1)).to(tl.float32)
+    if HAS_RESIDUAL:
+        p_res = tl.make_block_ptr(res, (T, D), (D, 1), (i_t * BT, 0), (BT, BD), (1, 0))
+        b_x += tl.load(p_res, boundary_check=(0, 1)).to(tl.float32)
+    if STORE_RESIDUAL_OUT:
+        p_res_out = tl.make_block_ptr(res_out, (T, D), (D, 1), (i_t * BT, 0), (BT, BD), (1, 0))
+        tl.store(p_res_out, b_x.to(p_res_out.dtype.element_ty), boundary_check=(0, 1))
+    if not IS_RMS_NORM:
+        b_mean = tl.sum(b_x, axis=1) / D
+        p_mean = tl.make_block_ptr(mean, (T,), (1,), (i_t * BT,), (BT,), (0,))
+        tl.store(p_mean, b_mean.to(p_mean.dtype.element_ty), boundary_check=(0,))
+        b_xbar = tl.where(m_d[None, :], b_x - b_mean[:, None], 0.0)
+        b_var = tl.sum(b_xbar * b_xbar, axis=1) / D
+    else:
+        b_xbar = tl.where(m_d[None, :], b_x, 0.0)
+        b_var = tl.sum(b_xbar * b_xbar, axis=1) / D
+    b_rstd = 1 / tl.sqrt(b_var + eps)
+    p_rstd = tl.make_block_ptr(rstd, (T,), (1,), (i_t * BT,), (BT,), (0,))
+    tl.store(p_rstd, b_rstd.to(p_rstd.dtype.element_ty), boundary_check=(0,))
+    if HAS_WEIGHT:
+        b_w = tl.load(w + o_g[:, None] * D + o_d[None, :], mask=m_d[None, :]).to(tl.float32)
+    if HAS_BIAS:
+        b_b = tl.load(b + o_g[:, None] * D + o_d[None, :], mask=m_d[None, :]).to(tl.float32)
+    b_x_hat = (b_x - b_mean[:, None]) * b_rstd[:, None] if not IS_RMS_NORM else b_x * b_rstd[:, None]
+    b_y = b_x_hat * b_w if HAS_WEIGHT else b_x_hat
+    if HAS_BIAS:
+        b_y = b_y + b_b
+    # Write output
+    p_y = tl.make_block_ptr(y, (T, D), (D, 1), (i_t * BT, 0), (BT, BD), (1, 0))
+    tl.store(p_y, b_y.to(p_y.dtype.element_ty), boundary_check=(0, 1))
+@triton.autotune(
+    configs=[
+        triton.Config({}, num_warps=num_warps)
+        for num_warps in [2, 4, 8, 16]
+    ],
+    key=['D', 'HAS_RESIDUAL', 'STORE_RESIDUAL_OUT', 'IS_RMS_NORM'],
+    **autotune_cache_kwargs,
+)
+@triton.jit
+def layer_norm_fwd_kernel1(
+    x,  # pointer to the input
+    y,  # pointer to the output
+    w,  # pointer to the weights
+    b,  # pointer to the biases
+    res,  # pointer to the res
+    res_out,  # pointer to the res
+    mean,  # pointer to the mean
+    rstd,  # pointer to the 1/std
+    eps,  # epsilon to avoid division by zero
+    G: tl.constexpr,
+    D: tl.constexpr,
+    BD: tl.constexpr,
+    IS_RMS_NORM: tl.constexpr,
+    HAS_RESIDUAL: tl.constexpr,
+    STORE_RESIDUAL_OUT: tl.constexpr,
+    HAS_WEIGHT: tl.constexpr,
+    HAS_BIAS: tl.constexpr,
+):
+    i_t = tl.program_id(0)
+    i_g = i_t % G
+    x += i_t * D
+    y += i_t * D
+    if HAS_RESIDUAL:
+        res += i_t * D
+    if STORE_RESIDUAL_OUT:
+        res_out += i_t * D
+    o_d = tl.arange(0, BD)
+    m_d = o_d < D
+    b_x = tl.load(x + o_d, mask=m_d, other=0.0).to(tl.float32)
+    if HAS_RESIDUAL:
+        b_x += tl.load(res + o_d, mask=m_d, other=0.0).to(tl.float32)
+    if STORE_RESIDUAL_OUT:
+        tl.store(res_out + o_d, b_x, mask=m_d)
+    if not IS_RMS_NORM:
+        b_mean = tl.sum(b_x, axis=0) / D
+        tl.store(mean + i_t, b_mean)
+        b_xbar = tl.where(m_d, b_x - b_mean, 0.0)
+        b_var = tl.sum(b_xbar * b_xbar, axis=0) / D
+    else:
+        b_xbar = tl.where(m_d, b_x, 0.0)
+        b_var = tl.sum(b_xbar * b_xbar, axis=0) / D
+    b_rstd = 1 / tl.sqrt(b_var + eps)
+    tl.store(rstd + i_t, b_rstd)
+    if HAS_WEIGHT:
+        b_w = tl.load(w + i_g * D + o_d, mask=m_d).to(tl.float32)
+    if HAS_BIAS:
+        b_b = tl.load(b + i_g * D + o_d, mask=m_d).to(tl.float32)
+    b_x_hat = (b_x - b_mean) * b_rstd if not IS_RMS_NORM else b_x * b_rstd
+    b_y = b_x_hat * b_w if HAS_WEIGHT else b_x_hat
+    if HAS_BIAS:
+        b_y = b_y + b_b
+    # Write output
+    tl.store(y + o_d, b_y, mask=m_d)
+@triton.heuristics({
+    'RECOMPUTE_OUTPUT': lambda args: args['y'] is not None,
+})
+@triton.autotune(
+    configs=[
+        triton.Config({'BT': BT}, num_warps=num_warps)
+        for BT in [32, 64]
+        for num_warps in [2, 4, 8]
+    ],
+    key=['D', 'NB', 'HAS_DRESIDUAL', 'STORE_DRESIDUAL', 'IS_RMS_NORM'],
+    **autotune_cache_kwargs,
+)
+@triton.jit
+def layer_norm_bwd_kernel(
+    x,  # pointer to the input
+    w,  # pointer to the weights
+    b,  # pointer to the biases
+    y,  # pointer to the output to be recomputed
+    dy,  # pointer to the output gradient
+    dx,  # pointer to the input gradient
+    dw,  # pointer to the partial sum of weights gradient
+    db,  # pointer to the partial sum of biases gradient
+    dres,
+    dres_in,
+    mean,
+    rstd,
+    T,
+    G: tl.constexpr,
+    D: tl.constexpr,
+    BS: tl.constexpr,
+    BT: tl.constexpr,
+    BD: tl.constexpr,
+    NB: tl.constexpr,
+    GS: tl.constexpr,
+    IS_RMS_NORM: tl.constexpr,
+    HAS_DRESIDUAL: tl.constexpr,
+    STORE_DRESIDUAL: tl.constexpr,
+    HAS_WEIGHT: tl.constexpr,
+    HAS_BIAS: tl.constexpr,
+    RECOMPUTE_OUTPUT: tl.constexpr,
+):
+    i_s = tl.program_id(0)
+    i_g, i_sg = i_s // GS, i_s % GS
+    o_d = tl.arange(0, BD)
+    m_d = o_d < D
+    if HAS_WEIGHT:
+        b_w = tl.load(w + i_g * D + o_d, mask=m_d).to(tl.float32)
+        b_dw = tl.zeros((BT, BD), dtype=tl.float32)
+    if HAS_BIAS:
+        b_b = tl.load(b + i_g * D + o_d, mask=m_d, other=0.0).to(tl.float32)
+        b_db = tl.zeros((BT, BD), dtype=tl.float32)
+    # Tg: number of tokens per group, used as the logical shape for make_block_ptr.
+    # for mean/rstd with shape (T,) and stride (G,), the strided view has Tg elements per group.
+    # the caller guarantees NS capped so every program has work.
+    # the last program's range may slightly exceed Tg (since BS = cdiv(T, NS));
+    # boundary_check handles the partial tail tile, m_t < Tg masks dw/db accumulation.
+    Tg = T // G
+    for i_t in range(i_sg * BS, i_sg * BS + BS, BT):
+        p_x = tl.make_block_ptr(x + i_g * D, (Tg, D), (G*D, 1), (i_t, 0), (BT, BD), (1, 0))
+        p_dy = tl.make_block_ptr(dy + i_g * D, (Tg, D), (G*D, 1), (i_t, 0), (BT, BD), (1, 0))
+        p_dx = tl.make_block_ptr(dx + i_g * D, (Tg, D), (G*D, 1), (i_t, 0), (BT, BD), (1, 0))
+        # [BT, BD]
+        b_x = tl.load(p_x, boundary_check=(0, 1)).to(tl.float32)
+        b_dy = tl.load(p_dy, boundary_check=(0, 1)).to(tl.float32)
+        if not IS_RMS_NORM:
+            p_mean = tl.make_block_ptr(mean + i_g, (Tg,), (G,), (i_t,), (BT,), (0,))
+            b_mean = tl.load(p_mean, boundary_check=(0,))
+        p_rstd = tl.make_block_ptr(rstd + i_g, (Tg,), (G,), (i_t,), (BT,), (0,))
+        b_rstd = tl.load(p_rstd, boundary_check=(0,))
+        # Compute dx
+        b_xhat = (b_x - b_mean[:, None]) * b_rstd[:, None] if not IS_RMS_NORM else b_x * b_rstd[:, None]
+        b_xhat = tl.where(m_d[None, :], b_xhat, 0.0)
+        b_y = b_xhat * b_w[None, :] if HAS_WEIGHT else b_xhat
+        if HAS_BIAS:
+            b_y = b_y + b_b[None, :]
+        if RECOMPUTE_OUTPUT:
+            p_y = tl.make_block_ptr(y + i_g * D, (Tg, D), (G*D, 1), (i_t, 0), (BT, BD), (1, 0))
+            tl.store(p_y, b_y.to(p_y.dtype.element_ty), boundary_check=(0, 1))
+        b_wdy = b_dy
+        if HAS_WEIGHT or HAS_BIAS:
+            # when BT > BS, a tile may span into the next program's range;
+            # mask to this program's upper bound to avoid double-counting dw/db.
+            m_t = (i_t + tl.arange(0, BT)) < min(i_sg * BS + BS, Tg)
+        if HAS_WEIGHT:
+            b_wdy = b_dy * b_w
+            b_dw += tl.where(m_t[:, None], b_dy * b_xhat, 0.0)
+        if HAS_BIAS:
+            b_db += tl.where(m_t[:, None], b_dy, 0.0)
+        if not IS_RMS_NORM:
+            b_c1 = tl.sum(b_xhat * b_wdy, axis=1) / D
+            b_c2 = tl.sum(b_wdy, axis=1) / D
+            b_dx = (b_wdy - (b_xhat * b_c1[:, None] + b_c2[:, None])) * b_rstd[:, None]
+        else:
+            b_c1 = tl.sum(b_xhat * b_wdy, axis=1) / D
+            b_dx = (b_wdy - b_xhat * b_c1[:, None]) * b_rstd[:, None]
+        if HAS_DRESIDUAL:
+            p_dres = tl.make_block_ptr(dres + i_g * D, (Tg, D), (G*D, 1), (i_t, 0), (BT, BD), (1, 0))
+            b_dres = tl.load(p_dres, boundary_check=(0, 1)).to(tl.float32)
+            b_dx += b_dres
+        # Write dx
+        if STORE_DRESIDUAL:
+            p_dres_in = tl.make_block_ptr(dres_in + i_g * D, (Tg, D), (G*D, 1), (i_t, 0), (BT, BD), (1, 0))
+            tl.store(p_dres_in, b_dx.to(p_dres_in.dtype.element_ty), boundary_check=(0, 1))
+        tl.store(p_dx, b_dx.to(p_dx.dtype.element_ty), boundary_check=(0, 1))
+    if HAS_WEIGHT:
+        tl.store(dw + i_s * D + o_d, tl.sum(b_dw, axis=0), mask=m_d)
+    if HAS_BIAS:
+        tl.store(db + i_s * D + o_d, tl.sum(b_db, axis=0), mask=m_d)
+@triton.heuristics({
+    'RECOMPUTE_OUTPUT': lambda args: args['y'] is not None,
+})
+@triton.autotune(
+    configs=[
+        triton.Config({}, num_warps=num_warps)
+        for num_warps in [2, 4, 8]
+    ],
+    key=['D', 'HAS_DRESIDUAL', 'STORE_DRESIDUAL', 'IS_RMS_NORM'],
+    **autotune_cache_kwargs,
+)
+@triton.jit
+def layer_norm_bwd_kernel1(
+    x,  # pointer to the input
+    w,  # pointer to the weights
+    b,  # pointer to the biases
+    y,  # pointer to the output to be recomputed
+    dy,  # pointer to the output gradient
+    dx,  # pointer to the input gradient
+    dw,  # pointer to the partial sum of weights gradient
+    db,  # pointer to the partial sum of biases gradient
+    dres,
+    dres_in,
+    mean,
+    rstd,
+    T,
+    G: tl.constexpr,
+    D: tl.constexpr,
+    BS: tl.constexpr,
+    BD: tl.constexpr,
+    GS: tl.constexpr,
+    IS_RMS_NORM: tl.constexpr,
+    HAS_DRESIDUAL: tl.constexpr,
+    STORE_DRESIDUAL: tl.constexpr,
+    HAS_WEIGHT: tl.constexpr,
+    HAS_BIAS: tl.constexpr,
+    RECOMPUTE_OUTPUT: tl.constexpr,
+):
+    i_s = tl.program_id(0)
+    i_g, i_sg = i_s // GS, i_s % GS
+    o_d = tl.arange(0, BD)
+    mask = o_d < D
+    if HAS_WEIGHT:
+        b_w = tl.load(w + i_g * D + o_d, mask=mask).to(tl.float32)
+        b_dw = tl.zeros((BD,), dtype=tl.float32)
+    if RECOMPUTE_OUTPUT and HAS_BIAS:
+        b_b = tl.load(b + i_g * D + o_d, mask=mask, other=0.0).to(tl.float32)
+    if HAS_BIAS:
+        b_db = tl.zeros((BD,), dtype=tl.float32)
+    for i_t in range(i_sg * BS * G + i_g, min((i_sg * BS + BS) * G + i_g, T), G):
+        b_x = tl.load(x + i_t * D + o_d, mask=mask, other=0).to(tl.float32)
+        b_dy = tl.load(dy + i_t * D + o_d, mask=mask, other=0).to(tl.float32)
+        if not IS_RMS_NORM:
+            b_mean = tl.load(mean + i_t)
+        b_rstd = tl.load(rstd + i_t)
+        # Compute dx
+        b_xhat = (b_x - b_mean) * b_rstd if not IS_RMS_NORM else b_x * b_rstd
+        b_xhat = tl.where(mask, b_xhat, 0.0)
+        if RECOMPUTE_OUTPUT:
+            b_y = b_xhat * b_w if HAS_WEIGHT else b_xhat
+            if HAS_BIAS:
+                b_y = b_y + b_b
+            tl.store(y + i_t * D + o_d, b_y, mask=mask)
+        b_wdy = b_dy
+        if HAS_WEIGHT:
+            b_wdy = b_dy * b_w
+            b_dw += b_dy * b_xhat
+        if HAS_BIAS:
+            b_db += b_dy
+        if not IS_RMS_NORM:
+            b_c1 = tl.sum(b_xhat * b_wdy, axis=0) / D
+            b_c2 = tl.sum(b_wdy, axis=0) / D
+            b_dx = (b_wdy - (b_xhat * b_c1 + b_c2)) * b_rstd
+        else:
+            b_c1 = tl.sum(b_xhat * b_wdy, axis=0) / D
+            b_dx = (b_wdy - b_xhat * b_c1) * b_rstd
+        if HAS_DRESIDUAL:
+            b_dres = tl.load(dres + i_t * D + o_d, mask=mask, other=0).to(tl.float32)
+            b_dx += b_dres
+        # Write dx
+        b_dx = tl.cast(b_dx, dtype=dx.dtype.element_ty, fp_downcast_rounding='rtne')
+        if STORE_DRESIDUAL:
+            tl.store(dres_in + i_t * D + o_d, b_dx, mask=mask)
+        tl.store(dx + i_t * D + o_d, b_dx, mask=mask)
+    if HAS_WEIGHT:
+        tl.store(dw + i_s * D + o_d, b_dw, mask=mask)
+    if HAS_BIAS:
+        tl.store(db + i_s * D + o_d, b_db, mask=mask)
+@dispatch('modules')
+def layer_norm_fwd(
+    x: torch.Tensor,
+    weight: torch.Tensor,
+    bias: torch.Tensor,
+    eps: float = 1e-5,
+    residual: torch.Tensor = None,
+    out_dtype: torch.dtype = None,
+    residual_dtype: torch.dtype = None,
+    is_rms_norm: bool = False,
+    num_groups: int = 1,
+):
+    if residual is not None:
+        residual_dtype = residual.dtype
+    T, D, G = *x.shape, num_groups
+    if residual is not None:
+        assert residual.shape == (T, D)
+    if weight is not None:
+        assert weight.shape == (G * D,)
+    if bias is not None:
+        assert bias.shape == (G * D,)
+    # allocate output
+    y = torch.empty_like(x, dtype=x.dtype if out_dtype is None else out_dtype)
+    if residual is not None or (residual_dtype is not None and residual_dtype != x.dtype):
+        res_out = torch.empty(T, D, device=x.device, dtype=residual_dtype)
+    else:
+        res_out = None
+    mean = torch.empty((T,), dtype=torch.float, device=x.device) if not is_rms_norm else None
+    rstd = torch.empty((T,), dtype=torch.float, device=x.device)
+    # Less than 64KB per feature: enqueue fused kernel
+    MAX_FUSED_SIZE = 65536 // x.element_size()
+    BD = min(MAX_FUSED_SIZE, triton.next_power_of_2(D))
+    if D > BD:
+        raise RuntimeError("This layer norm doesn't support feature dim >= 64KB.")
+    # heuristics for number of warps
+    if D <= 512:
+        NB = triton.cdiv(T, 2048)
+        def grid(meta): return (triton.cdiv(T, meta['BT']), )
+        layer_norm_fwd_kernel[grid](
+            x,
+            y,
+            weight,
+            bias,
+            residual,
+            res_out,
+            mean,
+            rstd,
+            eps,
+            T=T,
+            G=G,
+            D=D,
+            BD=BD,
+            NB=NB,
+            IS_RMS_NORM=is_rms_norm,
+            HAS_RESIDUAL=residual is not None,
+            STORE_RESIDUAL_OUT=res_out is not None,
+            HAS_WEIGHT=weight is not None,
+            HAS_BIAS=bias is not None,
+        )
+    else:
+        layer_norm_fwd_kernel1[(T,)](
+            x,
+            y,
+            weight,
+            bias,
+            residual,
+            res_out,
+            mean,
+            rstd,
+            eps,
+            G=G,
+            D=D,
+            BD=BD,
+            IS_RMS_NORM=is_rms_norm,
+            HAS_RESIDUAL=residual is not None,
+            STORE_RESIDUAL_OUT=res_out is not None,
+            HAS_WEIGHT=weight is not None,
+            HAS_BIAS=bias is not None,
+        )
+    # res_out is None if residual is None and residual_dtype == input_dtype
+    return y, mean, rstd, res_out if res_out is not None else x
+@dispatch('modules')
+def layer_norm_bwd(
+    dy: torch.Tensor,
+    x: torch.Tensor,
+    weight: torch.Tensor,
+    bias: torch.Tensor,
+    mean: torch.Tensor = None,
+    rstd: torch.Tensor = None,
+    dres: torch.Tensor = None,
+    has_residual: bool = False,
+    is_rms_norm: bool = False,
+    x_dtype: torch.dtype = None,
+    recompute_output: bool = False,
+    num_groups: int = 1,
+):
+    T, D, G = *x.shape, num_groups
+    assert dy.shape == (T, D)
+    if dres is not None:
+        assert dres.shape == (T, D)
+    if weight is not None:
+        assert weight.shape == (G * D,)
+    if bias is not None:
+        assert bias.shape == (G * D,)
+    # allocate output
+    dx = torch.empty_like(x) if x_dtype is None else torch.empty(T, D, dtype=x_dtype, device=x.device)
+    dres_in = torch.empty_like(x) if has_residual and dx.dtype != x.dtype else None
+    y = torch.empty(T, D, dtype=dy.dtype, device=dy.device) if recompute_output else None
+    # Less than 64KB per feature: enqueue fused kernel
+    MAX_FUSED_SIZE = 65536 // x.element_size()
+    BD = min(MAX_FUSED_SIZE, triton.next_power_of_2(D))
+    if D > BD:
+        raise RuntimeError("This layer norm doesn't support feature dim >= 64KB.")
+    # each program handles one group only.
+    # cap per-group program count to T // G so no program is completely idle.
+    # without this, high-SM GPUs (e.g. B200, 160 SMs) with small T would
+    # launch idle programs whose make_block_ptr offsets exceed the tensor shape.
+    NS = min(triton.cdiv(get_multiprocessor_count(x.device.index), G), T // G) * G
+    BS = triton.cdiv(T, NS)
+    GS = NS // G
+    dw = torch.empty((NS, D), dtype=torch.float, device=weight.device) if weight is not None else None
+    db = torch.empty((NS, D), dtype=torch.float, device=bias.device) if bias is not None else None
+    grid = (NS,)
+    if D <= 512:
+        NB = triton.cdiv(T, 2048)
+        layer_norm_bwd_kernel[grid](
+            x,
+            weight,
+            bias,
+            y,
+            dy,
+            dx,
+            dw,
+            db,
+            dres,
+            dres_in,
+            mean,
+            rstd,
+            T=T,
+            G=G,
+            D=D,
+            BS=BS,
+            BD=BD,
+            NB=NB,
+            GS=GS,
+            IS_RMS_NORM=is_rms_norm,
+            HAS_DRESIDUAL=dres is not None,
+            STORE_DRESIDUAL=dres_in is not None,
+            HAS_WEIGHT=weight is not None,
+            HAS_BIAS=bias is not None,
+        )
+    else:
+        layer_norm_bwd_kernel1[grid](
+            x,
+            weight,
+            bias,
+            y,
+            dy,
+            dx,
+            dw,
+            db,
+            dres,
+            dres_in,
+            mean,
+            rstd,
+            T=T,
+            G=G,
+            D=D,
+            BS=BS,
+            BD=BD,
+            GS=GS,
+            IS_RMS_NORM=is_rms_norm,
+            HAS_DRESIDUAL=dres is not None,
+            STORE_DRESIDUAL=dres_in is not None,
+            HAS_WEIGHT=weight is not None,
+            HAS_BIAS=bias is not None,
+        )
+    dw = dw.view(G, -1, D).sum(1).to(weight).view_as(weight) if weight is not None else None
+    db = db.view(G, -1, D).sum(1).to(bias).view_as(bias) if bias is not None else None
+    # Don't need to compute dres_in separately in this case
+    if has_residual and dx.dtype == x.dtype:
+        dres_in = dx
+    return (dx, dw, db, dres_in) if not recompute_output else (dx, dw, db, dres_in, y)
+class LayerNormFunction(torch.autograd.Function):
+    @staticmethod
+    @input_guard
+    def forward(
+        ctx,
+        x,
+        weight,
+        bias,
+        residual: torch.Tensor = None,
+        eps: float = 1e-5,
+        prenorm: bool = False,
+        residual_in_fp32: bool = False,
+        is_rms_norm: bool = False,
+        num_groups: int = 1,
+    ):
+        x_shape_og = x.shape
+        if x.shape[-1] % num_groups != 0:
+            raise ValueError('num_channels must be divisible by num_groups')
+        # reshape input data into 2D tensor
+        x = x.reshape(-1, (x.shape[-1] // num_groups))
+        if residual is not None:
+            assert residual.shape == x_shape_og
+            residual = residual.reshape_as(x)
+        residual_dtype = (
+            residual.dtype
+            if residual is not None
+            else (torch.float32 if residual_in_fp32 else None)
+        )
+        y, mean, rstd, res_out = layer_norm_fwd(
+            x,
+            weight,
+            bias,
+            eps,
+            residual,
+            residual_dtype=residual_dtype,
+            is_rms_norm=is_rms_norm,
+            num_groups=num_groups,
+        )
+        ctx.save_for_backward(res_out, weight, bias, mean, rstd)
+        ctx.x_shape_og = x_shape_og
+        ctx.eps = eps
+        ctx.is_rms_norm = is_rms_norm
+        ctx.num_groups = num_groups
+        ctx.has_residual = residual is not None
+        ctx.prenorm = prenorm
+        ctx.x_dtype = x.dtype
+        y = y.reshape(x_shape_og)
+        return y if not prenorm else (y, res_out.reshape(x_shape_og))
+    @staticmethod
+    @input_guard
+    def backward(ctx, dy, *args):
+        x, weight, bias, mean, rstd = ctx.saved_tensors
+        dy = dy.reshape(-1, (dy.shape[-1] // ctx.num_groups))
+        assert dy.shape == x.shape
+        if ctx.prenorm:
+            dresidual = args[0]
+            dresidual = dresidual.reshape(-1, x.shape[-1])
+            assert dresidual.shape == x.shape
+        else:
+            dresidual = None
+        dx, dw, db, dresidual_in = layer_norm_bwd(
+            dy,
+            x,
+            weight,
+            bias,
+            mean,
+            rstd,
+            dresidual,
+            ctx.has_residual,
+            ctx.is_rms_norm,
+            x_dtype=ctx.x_dtype,
+            num_groups=ctx.num_groups,
+        )
+        return (
+            dx.reshape(ctx.x_shape_og),
+            dw,
+            db,
+            dresidual_in.reshape(ctx.x_shape_og) if ctx.has_residual else None,
+            None,
+            None,
+            None,
+            None,
+            None,
+        )
+def layer_norm(
+    x: torch.Tensor,
+    weight: torch.Tensor,
+    bias: torch.Tensor,
+    residual: torch.Tensor = None,
+    eps: float = 1e-5,
+    prenorm: bool = False,
+    residual_in_fp32: bool = False,
+    is_rms_norm: bool = False,
+):
+    return LayerNormFunction.apply(
+        x,
+        weight,
+        bias,
+        residual,
+        eps,
+        prenorm,
+        residual_in_fp32,
+        is_rms_norm,
+    )
+def group_norm(
+    x: torch.Tensor,
+    weight: torch.Tensor,
+    bias: torch.Tensor,
+    residual: torch.Tensor = None,
+    eps: float = 1e-5,
+    prenorm: bool = False,
+    residual_in_fp32: bool = False,
+    is_rms_norm: bool = False,
+    num_groups: int = 1,
+):
+    return LayerNormFunction.apply(
+        x,
+        weight,
+        bias,
+        residual,
+        eps,
+        prenorm,
+        residual_in_fp32,
+        is_rms_norm,
+        num_groups,
+    )
+def rms_norm(
+    x: torch.Tensor,
+    weight: torch.Tensor,
+    bias: torch.Tensor,
+    residual: torch.Tensor = None,
+    eps: float = 1e-5,
+    prenorm: bool = False,
+    residual_in_fp32: bool = False,
+):
+    return LayerNormFunction.apply(
+        x,
+        weight,
+        bias,
+        residual,
+        eps,
+        prenorm,
+        residual_in_fp32,
+        True,
+    )
+def layer_norm_linear(
+    x: torch.Tensor,
+    norm_weight: torch.Tensor,
+    norm_bias: torch.Tensor,
+    linear_weight: torch.Tensor,
+    linear_bias: torch.Tensor,
+    residual: torch.Tensor = None,
+    eps: float = 1e-5,
+    prenorm: bool = False,
+    residual_in_fp32: bool = False,
+    is_rms_norm: bool = False,
+    num_groups: int = 1,
+):
+    return LayerNormLinearFunction.apply(
+        x,
+        norm_weight,
+        norm_bias,
+        linear_weight,
+        linear_bias,
+        residual,
+        eps,
+        prenorm,
+        residual_in_fp32,
+        is_rms_norm,
+        num_groups,
+    )
+def rms_norm_linear(
+    x: torch.Tensor,
+    norm_weight: torch.Tensor,
+    norm_bias: torch.Tensor,
+    linear_weight: torch.Tensor,
+    linear_bias: torch.Tensor,
+    residual: torch.Tensor = None,
+    eps: float = 1e-5,
+    prenorm: bool = False,
+    residual_in_fp32: bool = False,
+):
+    return layer_norm_linear(
+        x=x,
+        norm_weight=norm_weight,
+        norm_bias=norm_bias,
+        linear_weight=linear_weight,
+        linear_bias=linear_bias,
+        residual=residual,
+        eps=eps,
+        prenorm=prenorm,
+        residual_in_fp32=residual_in_fp32,
+        is_rms_norm=True,
+    )
+def group_norm_linear(
+    x: torch.Tensor,
+    norm_weight: torch.Tensor,
+    norm_bias: torch.Tensor,
+    linear_weight: torch.Tensor,
+    linear_bias: torch.Tensor,
+    residual: torch.Tensor = None,
+    eps: float = 1e-5,
+    prenorm: bool = False,
+    residual_in_fp32: bool = False,
+    is_rms_norm: bool = False,
+    num_groups: int = 1,
+):
+    return layer_norm_linear(
+        x=x,
+        norm_weight=norm_weight,
+        norm_bias=norm_bias,
+        linear_weight=linear_weight,
+        linear_bias=linear_bias,
+        residual=residual,
+        eps=eps,
+        prenorm=prenorm,
+        residual_in_fp32=residual_in_fp32,
+        is_rms_norm=is_rms_norm,
+        num_groups=num_groups,
+    )
+class LayerNorm(nn.Module):
+    def __init__(
+        self,
+        hidden_size: int,
+        elementwise_affine: bool = True,
+        bias: bool = False,
+        eps: float = 1e-5,
+        device: torch.device | None = None,
+        dtype: torch.dtype | None = None,
+    ) -> LayerNorm:
+        factory_kwargs = {"device": device, "dtype": dtype}
+        super().__init__()
+        self.hidden_size = hidden_size
+        self.elementwise_affine = elementwise_affine
+        self.eps = eps
+        self.register_parameter("weight", None)
+        self.register_parameter("bias", None)
+        if elementwise_affine:
+            self.weight = nn.Parameter(torch.empty(hidden_size, **factory_kwargs))
+            if bias:
+                self.bias = nn.Parameter(torch.empty(hidden_size, **factory_kwargs))
+        self.reset_parameters()
+    def reset_parameters(self):
+        if self.elementwise_affine:
+            nn.init.ones_(self.weight)
+            if self.bias is not None:
+                nn.init.zeros_(self.bias)
+    def __repr__(self) -> str:
+        s = f"{self.__class__.__name__}({self.hidden_size}"
+        if not self.elementwise_affine:
+            s += f", elementwise_affine={self.elementwise_affine}"
+        s += f", eps={self.eps}"
+        s += ")"
+        return s
+    def forward(self, x, residual=None, prenorm=False, residual_in_fp32=False):
+        return layer_norm(
+            x,
+            self.weight,
+            self.bias,
+            residual=residual,
+            eps=self.eps,
+            prenorm=prenorm,
+            residual_in_fp32=residual_in_fp32,
+        )
+class GroupNorm(nn.Module):
+    def __init__(
+        self,
+        num_groups: int,
+        hidden_size: int,
+        elementwise_affine: bool = True,
+        bias: bool = False,
+        eps: float = 1e-5,
+        is_rms_norm: bool = False,
+        device: torch.device | None = None,
+        dtype: torch.dtype | None = None,
+    ) -> GroupNorm:
+        factory_kwargs = {"device": device, "dtype": dtype}
+        super().__init__()
+        if hidden_size % num_groups != 0:
+            raise ValueError('num_channels must be divisible by num_groups')
+        self.num_groups = num_groups
+        self.hidden_size = hidden_size
+        self.elementwise_affine = elementwise_affine
+        self.eps = eps
+        self.is_rms_norm = is_rms_norm
+        self.register_parameter("weight", None)
+        self.register_parameter("bias", None)
+        if elementwise_affine:
+            self.weight = nn.Parameter(torch.empty(hidden_size, **factory_kwargs))
+            if bias:
+                self.bias = nn.Parameter(torch.empty(hidden_size, **factory_kwargs))
+        self.reset_parameters()
+    def reset_parameters(self):
+        if self.elementwise_affine:
+            nn.init.ones_(self.weight)
+            if self.bias is not None:
+                nn.init.zeros_(self.bias)
+    def __repr__(self) -> str:
+        s = f"{self.__class__.__name__}({self.num_groups}, {self.hidden_size}"
+        if not self.elementwise_affine:
+            s += f", elementwise_affine={self.elementwise_affine}"
+        if self.is_rms_norm:
+            s += f", is_rms_norm={self.is_rms_norm}"
+        s += f", eps={self.eps}"
+        s += ")"
+        return s
+    def forward(self, x, residual=None, prenorm=False, residual_in_fp32=False):
+        return group_norm(
+            x,
+            self.weight,
+            self.bias,
+            residual=residual,
+            eps=self.eps,
+            prenorm=prenorm,
+            residual_in_fp32=residual_in_fp32,
+            is_rms_norm=self.is_rms_norm,
+            num_groups=self.num_groups,
+        )
+class RMSNorm(nn.Module):
+    def __init__(
+        self,
+        hidden_size: int,
+        elementwise_affine: bool = True,
+        bias: bool = False,
+        eps: float = 1e-5,
+        device: torch.device | None = None,
+        dtype: torch.dtype | None = None,
+    ) -> RMSNorm:
+        factory_kwargs = {"device": device, "dtype": dtype}
+        super().__init__()
+        self.hidden_size = hidden_size
+        self.elementwise_affine = elementwise_affine
+        self.eps = eps
+        self.register_parameter("weight", None)
+        self.register_parameter("bias", None)
+        if elementwise_affine:
+            self.weight = nn.Parameter(torch.empty(hidden_size, **factory_kwargs))
+            if bias:
+                self.bias = nn.Parameter(torch.empty(hidden_size, **factory_kwargs))
+        self.reset_parameters()
+    def reset_parameters(self):
+        if self.elementwise_affine:
+            nn.init.ones_(self.weight)
+            if self.bias is not None:
+                nn.init.zeros_(self.bias)
+    def __repr__(self) -> str:
+        s = f"{self.__class__.__name__}({self.hidden_size}"
+        if not self.elementwise_affine:
+            s += f", elementwise_affine={self.elementwise_affine}"
+        s += f", eps={self.eps}"
+        s += ")"
+        return s
+    def forward(self, x, residual=None, prenorm=False, residual_in_fp32=False):
+        return rms_norm(
+            x,
+            self.weight,
+            self.bias,
+            residual=residual,
+            eps=self.eps,
+            prenorm=prenorm,
+            residual_in_fp32=residual_in_fp32,
+        )
+class LayerNormLinearFunction(torch.autograd.Function):
+    @staticmethod
+    @input_guard
+    def forward(
+        ctx,
+        x,
+        norm_weight,
+        norm_bias,
+        linear_weight,
+        linear_bias,
+        residual=None,
+        eps=1e-5,
+        prenorm=False,
+        residual_in_fp32=False,
+        is_rms_norm=False,
+        num_groups=1,
+    ):
+        x_shape_og = x.shape
+        if x.shape[-1] % num_groups != 0:
+            raise ValueError('num_channels must be divisible by num_groups')
+        # reshape input data into 2D tensor
+        x = x.reshape(-1, (x.shape[-1] // num_groups))
+        if residual is not None:
+            assert residual.shape == x_shape_og
+            residual = residual.reshape_as(x)
+        residual_dtype = (
+            residual.dtype
+            if residual is not None
+            else (torch.float32 if residual_in_fp32 else None)
+        )
+        y, mean, rstd, res_out = layer_norm_fwd(
+            x,
+            norm_weight,
+            norm_bias,
+            eps,
+            residual,
+            out_dtype=None if not torch.is_autocast_enabled() else torch.get_autocast_gpu_dtype(),
+            residual_dtype=residual_dtype,
+            is_rms_norm=is_rms_norm,
+            num_groups=num_groups,
+        )
+        y = y.reshape(x_shape_og)
+        dtype = torch.get_autocast_gpu_dtype() if torch.is_autocast_enabled() else y.dtype
+        linear_weight = linear_weight.to(dtype)
+        linear_bias = linear_bias.to(dtype) if linear_bias is not None else None
+        out = F.linear(y.to(linear_weight.dtype), linear_weight, linear_bias)
+        # We don't store y, will be recomputed in the backward pass to save memory
+        ctx.save_for_backward(res_out, norm_weight, norm_bias, linear_weight, mean, rstd)
+        ctx.x_shape_og = x_shape_og
+        ctx.eps = eps
+        ctx.is_rms_norm = is_rms_norm
+        ctx.num_groups = num_groups
+        ctx.has_residual = residual is not None
+        ctx.prenorm = prenorm
+        ctx.x_dtype = x.dtype
+        ctx.linear_bias_is_none = linear_bias is None
+        return out if not prenorm else (out, res_out.reshape(x_shape_og))
+    @staticmethod
+    @input_guard
+    def backward(ctx, dout, *args):
+        x, norm_weight, norm_bias, linear_weight, mean, rstd = ctx.saved_tensors
+        dout = dout.reshape(-1, dout.shape[-1])
+        dy = F.linear(dout, linear_weight.t())
+        dy = dy.reshape(-1, (dy.shape[-1] // ctx.num_groups))
+        dlinear_bias = None if ctx.linear_bias_is_none else dout.sum(0)
+        assert dy.shape == x.shape
+        if ctx.prenorm:
+            dresidual = args[0]
+            dresidual = dresidual.reshape(-1, x.shape[-1])
+            assert dresidual.shape == x.shape
+        else:
+            dresidual = None
+        dx, dnorm_weight, dnorm_bias, dresidual_in, y = layer_norm_bwd(
+            dy,
+            x,
+            norm_weight,
+            norm_bias,
+            mean,
+            rstd,
+            dresidual,
+            ctx.has_residual,
+            ctx.is_rms_norm,
+            x_dtype=ctx.x_dtype,
+            recompute_output=True,
+            num_groups=ctx.num_groups,
+        )
+        dlinear_weight = torch.einsum("bo,bi->oi", dout, y.view(-1, linear_weight.shape[-1]))
+        return (
+            dx.reshape(ctx.x_shape_og),
+            dnorm_weight,
+            dnorm_bias,
+            dlinear_weight,
+            dlinear_bias,
+            dresidual_in.reshape(ctx.x_shape_og) if ctx.has_residual else None,
+            None,
+            None,
+            None,
+            None,
+            None,
+        )
+class LayerNormLinear(nn.Module):
+    def __init__(
+        self,
+        hidden_size,
+        elementwise_affine: bool = True,
+        bias: bool = False,
+        eps: float = 1e-5,
+        device: torch.device | None = None,
+        dtype: torch.dtype | None = None,
+    ) -> LayerNormLinear:
+        factory_kwargs = {"device": device, "dtype": dtype}
+        super().__init__()
+        self.hidden_size = hidden_size
+        self.elementwise_affine = elementwise_affine
+        self.eps = eps
+        self.register_parameter("weight", None)
+        self.register_parameter("bias", None)
+        if elementwise_affine:
+            self.weight = nn.Parameter(torch.empty(hidden_size, **factory_kwargs))
+            if bias:
+                self.bias = nn.Parameter(torch.empty(hidden_size, **factory_kwargs))
+        self.reset_parameters()
+    def reset_parameters(self):
+        if self.elementwise_affine:
+            nn.init.ones_(self.weight)
+            if self.bias is not None:
+                nn.init.zeros_(self.bias)
+    def __repr__(self) -> str:
+        s = f"{self.__class__.__name__}({self.hidden_size}"
+        if not self.elementwise_affine:
+            s += f", elementwise_affine={self.elementwise_affine}"
+        s += f", eps={self.eps}"
+        s += ")"
+        return s
+    def forward(self, x, weight, bias, residual=None, prenorm=False, residual_in_fp32=False):
+        return layer_norm_linear(
+            x=x,
+            norm_weight=self.weight,
+            norm_bias=self.bias,
+            linear_weight=weight,
+            linear_bias=bias,
+            residual=residual,
+            eps=self.eps,
+            prenorm=prenorm,
+            residual_in_fp32=residual_in_fp32,
+            is_rms_norm=False,
+        )
+class GroupNormLinear(nn.Module):
+    def __init__(
+        self,
+        num_groups: int,
+        hidden_size: int,
+        elementwise_affine: bool = True,
+        bias: bool = False,
+        eps: float = 1e-5,
+        is_rms_norm: bool = False,
+        device: torch.device | None = None,
+        dtype: torch.dtype | None = None,
+    ) -> GroupNormLinear:
+        factory_kwargs = {"device": device, "dtype": dtype}
+        super().__init__()
+        if hidden_size % num_groups != 0:
+            raise ValueError('num_channels must be divisible by num_groups')
+        self.num_groups = num_groups
+        self.hidden_size = hidden_size
+        self.elementwise_affine = elementwise_affine
+        self.eps = eps
+        self.is_rms_norm = is_rms_norm
+        self.register_parameter("weight", None)
+        self.register_parameter("bias", None)
+        if elementwise_affine:
+            self.weight = nn.Parameter(torch.empty(hidden_size, **factory_kwargs))
+            if bias:
+                self.bias = nn.Parameter(torch.empty(hidden_size, **factory_kwargs))
+        self.reset_parameters()
+    def reset_parameters(self):
+        if self.elementwise_affine:
+            nn.init.ones_(self.weight)
+            if self.bias is not None:
+                nn.init.zeros_(self.bias)
+    def __repr__(self) -> str:
+        s = f"{self.__class__.__name__}({self.num_groups}, {self.hidden_size}"
+        if not self.elementwise_affine:
+            s += f", elementwise_affine={self.elementwise_affine}"
+        if self.is_rms_norm:
+            s += f", is_rms_norm={self.is_rms_norm}"
+        s += f", eps={self.eps}"
+        s += ")"
+        return s
+    def forward(self, x, weight, bias, residual=None, prenorm=False, residual_in_fp32=False):
+        return layer_norm_linear(
+            x=x,
+            norm_weight=self.weight,
+            norm_bias=self.bias,
+            linear_weight=weight,
+            linear_bias=bias,
+            residual=residual,
+            eps=self.eps,
+            prenorm=prenorm,
+            residual_in_fp32=residual_in_fp32,
+            is_rms_norm=self.is_rms_norm,
+            num_groups=self.num_groups,
+        )
+class RMSNormLinear(nn.Module):
+    def __init__(
+        self,
+        hidden_size,
+        elementwise_affine: bool = True,
+        bias: bool = False,
+        eps: float = 1e-5,
+        device: torch.device | None = None,
+        dtype: torch.dtype | None = None,
+    ) -> RMSNormLinear:
+        factory_kwargs = {"device": device, "dtype": dtype}
+        super().__init__()
+        self.hidden_size = hidden_size
+        self.elementwise_affine = elementwise_affine
+        self.eps = eps
+        self.register_parameter("weight", None)
+        self.register_parameter("bias", None)
+        if elementwise_affine:
+            self.weight = nn.Parameter(torch.empty(hidden_size, **factory_kwargs))
+            if bias:
+                self.bias = nn.Parameter(torch.empty(hidden_size, **factory_kwargs))
+        self.reset_parameters()
+    def reset_parameters(self):
+        if self.elementwise_affine:
+            nn.init.ones_(self.weight)
+            if self.bias is not None:
+                nn.init.zeros_(self.bias)
+    def __repr__(self) -> str:
+        s = f"{self.__class__.__name__}({self.hidden_size}"
+        if not self.elementwise_affine:
+            s += f", elementwise_affine={self.elementwise_affine}"
+        s += f", eps={self.eps}"
+        s += ")"
+        return s
+    def forward(self, x, weight, bias, residual=None, prenorm=False, residual_in_fp32=False):
+        return layer_norm_linear(
+            x=x,
+            norm_weight=self.weight,
+            norm_bias=self.bias,
+            linear_weight=weight,
+            linear_bias=bias,
+            residual=residual,
+            eps=self.eps,
+            prenorm=prenorm,
+            residual_in_fp32=residual_in_fp32,
+            is_rms_norm=True,
+        )
+class NormParallel(ParallelStyle):
+    def __init__(self, *, sequence_dim: int = 1, use_local_output: bool = False):
+        super().__init__()
+        self.sequence_sharding = (Shard(sequence_dim),)
+        self.use_local_output = use_local_output
+    def _replicate_module_fn(
+        self, name: str, module: nn.Module, device_mesh: DeviceMesh,
+    ):
+        for p_name, param in module.named_parameters():
+            # simple replication with fixed ones_ init from LayerNorm/RMSNorm, which allow
+            # us to simply just use from_local
+            replicated_param = torch.nn.Parameter(
+                DTensor.from_local(param, device_mesh, [Replicate()], run_check=False),
+            )
+            module.register_parameter(p_name, replicated_param)
+    @staticmethod
+    def _prepare_input_fn(sequence_sharding, mod, inputs, device_mesh):
+        input_tensor = inputs[0]
+        if isinstance(input_tensor, DTensor):
+            # if the passed in input DTensor is not sharded on the sequence dim, we need to redistribute it
+            if input_tensor.placements != sequence_sharding:
+                input_tensor = input_tensor.redistribute(
+                    placements=sequence_sharding, async_op=True,
+                )
+            return input_tensor
+        elif isinstance(input_tensor, torch.Tensor):
+            # assume the input passed in already sharded on the sequence dim and create the DTensor
+            return DTensor.from_local(
+                input_tensor, device_mesh, sequence_sharding, run_check=False,
+            )
+        else:
+            raise ValueError(
+                f"expecting input of {mod} to be a torch.Tensor or DTensor, but got {input_tensor}",
+            )
+    @staticmethod
+    def _prepare_output_fn(use_local_output, mod, outputs, device_mesh):
+        return outputs.to_local() if use_local_output else outputs
+    def _apply(self, module: nn.Module, device_mesh: DeviceMesh) -> nn.Module:
+        return distribute_module(
+            module,
+            device_mesh,
+            self._replicate_module_fn,
+            partial(self._prepare_input_fn, self.sequence_sharding),
+            partial(self._prepare_output_fn, self.use_local_output),
+        )

build/torch-cuda/modules/layernorm_gated.py ADDED Viewed

	@@ -0,0 +1,535 @@

+# Copyright (c) 2023-2026, Songlin Yang, Yu Zhang, Zhiyuan Li
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+# For a list of all contributors, visit:
+#   https://github.com/fla-org/flash-linear-attention/graphs/contributors
+#
+# Copyright (c) 2024, Tri Dao.
+#
+# Based on the Triton LayerNorm tutorial: https://triton-lang.org/main/getting-started/tutorials/05-layer-norm.html
+# For the backward pass, we keep weight_grad and bias_grad in registers and accumulate.
+# This backward pass is faster for dimensions up to 8k, but after that it's much slower due to register spilling.
+# The models we train have hidden dim up to 8k anyway (e.g. Llama 70B), so this is fine.
+import math
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+import triton
+import triton.language as tl
+from einops import rearrange
+from ..utils import get_multiprocessor_count, input_guard
+def rms_norm_ref(x, weight, bias, z=None, eps=1e-6, group_size=None, norm_before_gate=True, upcast=True):
+    dtype = x.dtype
+    weight = weight.float()
+    bias = bias.float() if bias is not None else None
+    if upcast:
+        x = x.float()
+        z = z.float() if z is not None else z
+    if z is not None and not norm_before_gate:
+        x = x * F.silu(z)
+    if group_size is None:
+        rstd = 1 / torch.sqrt((x.square()).mean(dim=-1, keepdim=True) + eps)
+        out = (x * rstd * weight) + bias if bias is not None else (x * rstd * weight)
+    else:
+        x_group = rearrange(x, "... (g d) -> ... g d", d=group_size)
+        rstd = 1 / torch.sqrt((x_group.square()).mean(dim=-1, keepdim=True) + eps)
+        out = rearrange(x_group * rstd, "... g d -> ... (g d)") * weight
+        if bias is not None:
+            out = out + bias
+    if z is not None and norm_before_gate:
+        out *= F.silu(z)
+    return out.to(dtype)
+@triton.heuristics({
+    "HAS_BIAS": lambda args: args["B"] is not None,
+    "HAS_Z": lambda args: args["Z"] is not None,
+})
+@triton.jit
+def layer_norm_fwd_kernel(
+    X,  # pointer to the input
+    Y,  # pointer to the output
+    W,  # pointer to the weights
+    B,  # pointer to the biases
+    Z,  # pointer to the other branch
+    Mean,  # pointer to the mean
+    Rstd,  # pointer to the 1/std
+    stride_x_row,  # how much to increase the pointer when moving by 1 row
+    stride_y_row,
+    stride_z_row,
+    M,  # number of rows in X
+    N,  # number of columns in X
+    eps,  # epsilon to avoid division by zero
+    BLOCK_N: tl.constexpr,
+    HAS_BIAS: tl.constexpr,
+    HAS_Z: tl.constexpr,
+    NORM_BEFORE_GATE: tl.constexpr,
+    IS_RMS_NORM: tl.constexpr,
+):
+    # Map the program id to the row of X and Y it should compute.
+    row = tl.program_id(0)
+    group = tl.program_id(1)
+    X += row * stride_x_row + group * N
+    Y += row * stride_y_row + group * N
+    if HAS_Z:
+        Z += row * stride_z_row + group * N
+    if not IS_RMS_NORM:
+        Mean += group * M
+    Rstd += group * M
+    W += group * N
+    if HAS_BIAS:
+        B += group * N
+    # Compute mean and variance
+    cols = tl.arange(0, BLOCK_N)
+    x = tl.load(X + cols, mask=cols < N, other=0.).to(tl.float32)
+    if HAS_Z and not NORM_BEFORE_GATE:
+        z = tl.load(Z + cols, mask=cols < N).to(tl.float32)
+        x *= z * tl.sigmoid(z)
+    if not IS_RMS_NORM:
+        mean = tl.sum(x, axis=0) / N
+        tl.store(Mean + row, mean)
+        xbar = tl.where(cols < N, x - mean, 0.)
+        var = tl.sum(xbar * xbar, axis=0) / N
+    else:
+        xbar = tl.where(cols < N, x, 0.)
+        var = tl.sum(xbar * xbar, axis=0) / N
+    rstd = 1 / tl.sqrt(var + eps)
+    tl.store(Rstd + row, rstd)
+    # Normalize and apply linear transformation
+    mask = cols < N
+    w = tl.load(W + cols, mask=mask).to(tl.float32)
+    if HAS_BIAS:
+        b = tl.load(B + cols, mask=mask).to(tl.float32)
+    x_hat = (x - mean) * rstd if not IS_RMS_NORM else x * rstd
+    y = x_hat * w + b if HAS_BIAS else x_hat * w
+    if HAS_Z and NORM_BEFORE_GATE:
+        z = tl.load(Z + cols, mask=mask).to(tl.float32)
+        y *= z * tl.sigmoid(z)
+    # Write output
+    tl.store(Y + cols, y, mask=mask)
+def layer_norm_fwd(
+    x: torch.Tensor,
+    weight: torch.Tensor,
+    bias: torch.Tensor,
+    eps: float,
+    z: torch.Tensor = None,
+    out: torch.Tensor = None,
+    group_size: int = None,
+    norm_before_gate: bool = True,
+    is_rms_norm: bool = False,
+):
+    M, N = x.shape
+    if group_size is None:
+        group_size = N
+    assert N % group_size == 0
+    ngroups = N // group_size
+    assert x.stride(-1) == 1
+    if z is not None:
+        assert z.stride(-1) == 1
+        assert z.shape == (M, N)
+    assert weight.shape == (N,)
+    assert weight.stride(-1) == 1
+    if bias is not None:
+        assert bias.stride(-1) == 1
+        assert bias.shape == (N,)
+    # allocate output
+    if out is not None:
+        assert out.shape == x.shape
+    else:
+        out = torch.empty_like(x)
+    assert out.stride(-1) == 1
+    mean = torch.empty((ngroups * M, ), dtype=torch.float32, device=x.device) if not is_rms_norm else None
+    rstd = torch.empty((ngroups * M, ), dtype=torch.float32, device=x.device)
+    # Less than 64KB per feature: enqueue fused kernel
+    MAX_FUSED_SIZE = 65536 // x.element_size()
+    BLOCK_N = min(MAX_FUSED_SIZE, triton.next_power_of_2(group_size))
+    if group_size > BLOCK_N:
+        raise RuntimeError("This layer norm doesn't support feature dim >= 64KB.")
+    # heuristics for number of warps
+    num_warps = min(max(BLOCK_N // 256, 1), 8)
+    grid = (M, ngroups)
+    layer_norm_fwd_kernel[grid](
+        x,
+        out,
+        weight,
+        bias,
+        z,
+        mean,
+        rstd,
+        x.stride(0),
+        out.stride(0),
+        z.stride(0) if z is not None else 0,
+        M,
+        group_size,
+        eps,
+        BLOCK_N=BLOCK_N,
+        NORM_BEFORE_GATE=norm_before_gate,
+        IS_RMS_NORM=is_rms_norm,
+        num_warps=num_warps,
+    )
+    return out, mean, rstd
+@triton.heuristics({
+    "HAS_BIAS": lambda args: args["B"] is not None,
+    "HAS_Z": lambda args: args["Z"] is not None,
+    "RECOMPUTE_OUTPUT": lambda args: args["Y"] is not None,
+})
+@triton.jit
+def layer_norm_bwd_kernel(
+    X,   # pointer to the input
+    W,   # pointer to the weights
+    B,   # pointer to the biases
+    Z,   # pointer to the other branch
+    Y,   # pointer to the output to be recomputed
+    DY,  # pointer to the output gradient
+    DX,  # pointer to the input gradient
+    DW,  # pointer to the partial sum of weights gradient
+    DB,  # pointer to the partial sum of biases gradient
+    DZ,  # pointer to the other branch
+    Mean,   # pointer to the mean
+    Rstd,   # pointer to the 1/std
+    stride_x_row,  # how much to increase the pointer when moving by 1 row
+    stride_z_row,
+    stride_y_row,
+    stride_dy_row,
+    stride_dx_row,
+    stride_dz_row,
+    stride_dw_row,
+    stride_db_row,
+    M,  # number of rows in X
+    N,  # number of columns in X
+    eps,  # epsilon to avoid division by zero
+    rows_per_program,
+    NORM_BEFORE_GATE: tl.constexpr,
+    IS_RMS_NORM: tl.constexpr,
+    HAS_BIAS: tl.constexpr,
+    HAS_Z: tl.constexpr,
+    RECOMPUTE_OUTPUT: tl.constexpr,
+    BLOCK_N: tl.constexpr,
+):
+    # Map the program id to the elements of X, DX, and DY it should compute.
+    row_block_id = tl.program_id(0)
+    group = tl.program_id(1)
+    row_start = row_block_id * rows_per_program
+    cols = tl.arange(0, BLOCK_N)
+    mask = cols < N
+    X += row_start * stride_x_row + group * N
+    if HAS_Z:
+        Z += row_start * stride_z_row + group * N
+        DZ += row_start * stride_dz_row + group * N
+    DY += row_start * stride_dy_row + group * N
+    DX += row_start * stride_dx_row + group * N
+    if RECOMPUTE_OUTPUT:
+        Y += row_start * stride_y_row + group * N
+    if not IS_RMS_NORM:
+        Mean += group * M
+    Rstd += group * M
+    W += group * N
+    w = tl.load(W + cols, mask=mask).to(tl.float32)
+    if (RECOMPUTE_OUTPUT or HAS_Z) and HAS_BIAS:
+        B += group * N
+        b = tl.load(B + cols, mask=mask, other=0.).to(tl.float32)
+    dw = tl.zeros((BLOCK_N,), dtype=tl.float32)
+    if HAS_BIAS:
+        db = tl.zeros((BLOCK_N,), dtype=tl.float32)
+    row_end = min((row_block_id + 1) * rows_per_program, M)
+    for row in range(row_start, row_end):
+        # Load data to SRAM
+        x = tl.load(X + cols, mask=mask, other=0).to(tl.float32)
+        dy = tl.load(DY + cols, mask=mask, other=0).to(tl.float32)
+        if not IS_RMS_NORM:
+            mean = tl.load(Mean + row)
+        if HAS_Z and not NORM_BEFORE_GATE:
+            z = tl.load(Z + cols, mask=mask, other=0.).to(tl.float32)
+            x_og = x
+            x = x_og * z * tl.sigmoid(z)
+        rstd = tl.load(Rstd + row)
+        # Compute dx
+        xhat = (x - mean) * rstd if not IS_RMS_NORM else x * rstd
+        xhat = tl.where(mask, xhat, 0.)
+        if HAS_Z and NORM_BEFORE_GATE:
+            z = tl.load(Z + cols, mask=mask, other=0.).to(tl.float32)
+            z_sigmoid = tl.sigmoid(z)
+            y = xhat * w + b if HAS_BIAS else xhat * w
+            if RECOMPUTE_OUTPUT:
+                tl.store(Y + cols, y * z * z_sigmoid, mask=mask)
+            dz = dy * y * z_sigmoid * (1 + z * (1 - z_sigmoid))
+            tl.store(DZ + cols, dz, mask=mask)
+            dy *= z * z_sigmoid
+        else:
+            if RECOMPUTE_OUTPUT:
+                y = xhat * w + b if HAS_BIAS else xhat * w
+                tl.store(Y + cols, y, mask=mask)
+        wdy = w * dy
+        c1 = tl.sum(xhat * wdy, axis=0) / N
+        if not IS_RMS_NORM:
+            c2 = tl.sum(wdy, axis=0) / N
+            dx = (wdy - (xhat * c1 + c2)) * rstd
+        else:
+            dx = (wdy - xhat * c1) * rstd
+        dw += dy * xhat
+        if HAS_BIAS:
+            db += dy
+        if HAS_Z and not NORM_BEFORE_GATE:
+            z_sigmoid = tl.sigmoid(z)
+            dz = dx * x_og * z_sigmoid * (1 + z * (1 - z_sigmoid))
+            tl.store(DZ + cols, dz, mask=mask)
+            dx *= z * z_sigmoid
+        # Write dx
+        tl.store(DX + cols, dx, mask=mask)
+        X += stride_x_row
+        if HAS_Z:
+            Z += stride_z_row
+            DZ += stride_dz_row
+        if RECOMPUTE_OUTPUT:
+            Y += stride_y_row
+        DY += stride_dy_row
+        DX += stride_dx_row
+    tl.store(DW + row_block_id * stride_dw_row + group * N + cols, dw, mask=mask)
+    if HAS_BIAS:
+        tl.store(DB + row_block_id * stride_db_row + group * N + cols, db, mask=mask)
+def layer_norm_bwd(
+    dy: torch.Tensor,
+    x: torch.Tensor,
+    weight: torch.Tensor,
+    bias: torch.Tensor,
+    eps: float,
+    mean: torch.Tensor,
+    rstd: torch.Tensor,
+    z: torch.Tensor = None,
+    group_size: int = None,
+    norm_before_gate: bool = True,
+    is_rms_norm: bool = False,
+    recompute_output: bool = False,
+    dz: torch.Tensor = None,
+    out: torch.Tensor = None,
+):
+    M, N = x.shape
+    if group_size is None:
+        group_size = N
+    assert N % group_size == 0
+    ngroups = N // group_size
+    assert x.stride(-1) == 1
+    assert dy.stride(-1) == 1
+    assert dy.shape == (M, N)
+    if z is not None:
+        assert z.stride(-1) == 1
+        assert z.shape == (M, N)
+    assert weight.shape == (N,)
+    assert weight.stride(-1) == 1
+    if bias is not None:
+        assert bias.stride(-1) == 1
+        assert bias.shape == (N,)
+    # allocate output
+    dx = torch.empty_like(x)
+    if dz is not None:
+        assert z is not None
+        assert dz.shape == z.shape
+        assert dz.stride(-1) == 1
+    else:
+        dz = torch.empty_like(z) if z is not None else None
+    if recompute_output:
+        if out is None:
+            out = torch.empty_like(x)
+        assert out.shape == x.shape
+    # Less than 64KB per feature: enqueue fused kernel
+    MAX_FUSED_SIZE = 65536 // x.element_size()
+    BLOCK_N = min(MAX_FUSED_SIZE, triton.next_power_of_2(group_size))
+    if group_size > BLOCK_N:
+        raise RuntimeError("This layer norm doesn't support feature dim >= 64KB.")
+    # heuristics for number of warps
+    num_warps = min(max(BLOCK_N // 256, 1), 8)
+    sm_count = get_multiprocessor_count(x.device.index)
+    # If group size is small (e.g., 64), we're only using 1 warp. So having just 108 programs
+    # would limit the occupancy.
+    nrow_groups = math.ceil(sm_count * math.ceil(4 / num_warps) / ngroups)
+    _dw = torch.empty((nrow_groups, N), dtype=torch.float32, device=weight.device)
+    _db = torch.empty((nrow_groups, N), dtype=torch.float32, device=bias.device) if bias is not None else None
+    rows_per_program = math.ceil(M / nrow_groups)
+    grid = (nrow_groups, ngroups)
+    layer_norm_bwd_kernel[grid](
+        x,
+        weight,
+        bias,
+        z,
+        out if recompute_output else None,
+        dy,
+        dx,
+        _dw,
+        _db,
+        dz,
+        mean,
+        rstd,
+        x.stride(0),
+        z.stride(0) if z is not None else 0,
+        0 if not recompute_output else out.stride(0),
+        dy.stride(0),
+        dx.stride(0),
+        dz.stride(0) if dz is not None else 0,
+        _dw.stride(0),
+        _db.stride(0) if _db is not None else 0,
+        M, group_size, eps,
+        rows_per_program,
+        BLOCK_N=BLOCK_N,
+        NORM_BEFORE_GATE=norm_before_gate,
+        IS_RMS_NORM=is_rms_norm,
+        num_warps=num_warps,
+    )
+    dw = _dw.sum(0).to(weight.dtype)
+    db = _db.sum(0).to(bias.dtype) if bias is not None else None
+    return (dx, dw, db, dz) if not recompute_output else (dx, dw, db, dz, out)
+class LayerNormFn(torch.autograd.Function):
+    @input_guard
+    @staticmethod
+    def forward(ctx, x, weight, bias, z=None, eps=1e-6, group_size=None, norm_before_gate=True,
+                is_rms_norm=False):
+        """If z is not None, we do norm(x) * silu(z) if norm_before_gate, else norm(x * silu(z))
+        """
+        x_shape_og = x.shape
+        # reshape input data into 2D tensor
+        x = x.reshape(-1, x.shape[-1])
+        if x.stride(-1) != 1:
+            x = x.contiguous()
+        if z is not None:
+            assert z.shape == x_shape_og
+            z = z.reshape(-1, z.shape[-1])
+            if z.stride(-1) != 1:
+                z = z.contiguous()
+        weight = weight.contiguous()
+        if bias is not None:
+            bias = bias.contiguous()
+        y, mean, rstd = layer_norm_fwd(
+            x,
+            weight,
+            bias,
+            eps,
+            z=z,
+            group_size=group_size,
+            norm_before_gate=norm_before_gate,
+            is_rms_norm=is_rms_norm,
+        )
+        ctx.save_for_backward(x, weight, bias, mean, rstd, z)
+        ctx.x_shape_og = x_shape_og
+        ctx.eps = eps
+        ctx.group_size = group_size
+        ctx.norm_before_gate = norm_before_gate
+        ctx.is_rms_norm = is_rms_norm
+        return y.reshape(x_shape_og)
+    @input_guard
+    @staticmethod
+    def backward(ctx, dy):
+        x, weight, bias, mean, rstd, z = ctx.saved_tensors
+        dy = dy.reshape(-1, dy.shape[-1])
+        if dy.stride(-1) != 1:
+            dy = dy.contiguous()
+        assert dy.shape == x.shape
+        dx, dw, db, dz = layer_norm_bwd(
+            dy,
+            x,
+            weight,
+            bias,
+            ctx.eps,
+            mean,
+            rstd,
+            z,
+            ctx.group_size,
+            ctx.norm_before_gate,
+            ctx.is_rms_norm,
+        )
+        dx = dx.reshape(ctx.x_shape_og)
+        dz = dz.reshape(ctx.x_shape_og) if dz is not None else None
+        return dx, dw, db, dz, None, None, None, None
+def layernorm_fn(x, weight, bias, z=None, eps=1e-6, group_size=None, norm_before_gate=True, is_rms_norm=False):
+    return LayerNormFn.apply(x, weight, bias, z, eps, group_size, norm_before_gate, is_rms_norm)
+def rmsnorm_fn(x, weight, bias, z=None, eps=1e-6, group_size=None, norm_before_gate=True):
+    return LayerNormFn.apply(x, weight, bias, z, eps, group_size, norm_before_gate, True)
+class LayerNormGated(nn.Module):
+    def __init__(
+        self,
+        hidden_size,
+        eps: float = 1e-5,
+        group_size: int | None = None,
+        norm_before_gate: bool = True,
+        device: torch.device | None = None,
+        dtype: torch.dtype | None = None,
+    ):
+        """If group_size is not None, we do GroupNorm with each group having group_size elements.
+        group_size=None is equivalent to group_size=hidden_size (i.e. there's only 1 group).
+        """
+        factory_kwargs = {"device": device, "dtype": dtype}
+        super().__init__()
+        self.eps = eps
+        self.weight = nn.Parameter(torch.empty(hidden_size, **factory_kwargs))
+        self.bias = nn.Parameter(torch.empty(hidden_size, **factory_kwargs))
+        self.group_size = group_size
+        self.norm_before_gate = norm_before_gate
+        self.reset_parameters()
+    def reset_parameters(self):
+        torch.nn.init.ones_(self.weight)
+        torch.nn.init.zeros_(self.bias)
+    def forward(self, x, z=None):
+        """If z is not None, we do norm(x) * silu(z) if norm_before_gate, else norm(x * silu(z))
+        """
+        return layernorm_fn(x, self.weight, self.bias, z=z, group_size=self.group_size, eps=self.eps,
+                            norm_before_gate=self.norm_before_gate)
+class RMSNormGated(nn.Module):
+    def __init__(
+        self,
+        hidden_size,
+        eps: float = 1e-5,
+        group_size: int | None = None,
+        norm_before_gate: bool = False,
+        device: torch.device | None = None,
+        dtype: torch.dtype | None = None,
+    ):
+        """If group_size is not None, we do GroupNorm with each group having group_size elements.
+        group_size=None is equivalent to group_size=hidden_size (i.e. there's only 1 group).
+        """
+        factory_kwargs = {"device": device, "dtype": dtype}
+        super().__init__()
+        self.eps = eps
+        self.weight = nn.Parameter(torch.empty(hidden_size, **factory_kwargs))
+        self.register_parameter("bias", None)
+        self.group_size = group_size
+        self.norm_before_gate = norm_before_gate
+        self.reset_parameters()
+    def reset_parameters(self):
+        torch.nn.init.ones_(self.weight)
+    def forward(self, x, z=None):
+        """If z is not None, we do norm(x) * silu(z) if norm_before_gate, else norm(x * silu(z))
+        """
+        return rmsnorm_fn(x, self.weight, self.bias, z=z, eps=self.eps, group_size=self.group_size,
+                          norm_before_gate=self.norm_before_gate)

build/torch-cuda/modules/mlp.py ADDED Viewed

	@@ -0,0 +1,141 @@

+# Copyright (c) 2023-2026, Songlin Yang, Yu Zhang, Zhiyuan Li
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+# For a list of all contributors, visit:
+#   https://github.com/fla-org/flash-linear-attention/graphs/contributors
+from __future__ import annotations
+from functools import partial
+from typing import TYPE_CHECKING, Any
+import torch
+import torch.nn as nn
+from torch.distributed import DeviceMesh
+from torch.distributed.tensor import Placement, Replicate, Shard, distribute_module
+from torch.distributed.tensor.parallel import ParallelStyle
+from ..modules.activations import powglu, powglu_linear, swiglu, swiglu_linear
+try:
+    from torch.distributed.tensor import DTensor
+except (ImportError, AttributeError):
+    DTensor = None
+if TYPE_CHECKING:
+    from transformers.processing_utils import Unpack
+class GatedMLP(nn.Module):
+    def __init__(
+        self,
+        hidden_size: int,
+        hidden_ratio: int | None = None,
+        intermediate_size: int | None = None,
+        hidden_act: str = 'swish',
+        fuse_swiglu: bool = True,
+        powglu_power: float = 3.0,
+    ) -> GatedMLP:
+        super().__init__()
+        self.hidden_size = hidden_size
+        # the final number of params is `hidden_ratio * hidden_size^2`
+        # `intermediate_size` is chosen to be a multiple of 256 closest to `2/3 * hidden_size * hidden_ratio`
+        if hidden_ratio is None:
+            hidden_ratio = 4
+        if intermediate_size is None:
+            intermediate_size = int(hidden_size * hidden_ratio * 2 / 3)
+            intermediate_size = 256 * ((intermediate_size + 256 - 1) // 256)
+        self.hidden_ratio = hidden_ratio
+        self.intermediate_size = intermediate_size
+        self.hidden_act = hidden_act
+        self.fuse_swiglu = fuse_swiglu
+        self.powglu_power = powglu_power
+        if hidden_act not in ('swish', 'powlu'):
+            raise ValueError(f'Unsupported hidden_act: {hidden_act}')
+        self.gate_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=False)
+        self.up_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=False)
+        self.down_proj = nn.Linear(self.intermediate_size, self.hidden_size, bias=False)
+        if self.fuse_swiglu and hidden_act == 'swish':
+            self.swiglu_linear = SwiGLULinear()
+    def forward(
+        self,
+        x: torch.Tensor,
+        **kwargs: Unpack[Any],
+    ) -> torch.Tensor:
+        gate, y = self.gate_proj(x), self.up_proj(x)
+        if self.hidden_act == 'powlu':
+            if self.fuse_swiglu:
+                return powglu_linear(gate, y, self.down_proj.weight, self.down_proj.bias, self.powglu_power)
+            return self.down_proj(powglu(gate, y, self.powglu_power))
+        if self.fuse_swiglu:
+            return self.swiglu_linear(gate, y, self.down_proj.weight, self.down_proj.bias)
+        return self.down_proj(swiglu(gate, y))
+class SwiGLULinear(nn.Module):
+    def forward(self, x, y, weight, bias):
+        return swiglu_linear(x, y, weight, bias)
+class SwiGLULinearParallel(ParallelStyle):
+    def __init__(
+        self,
+        *,
+        input_layouts: Placement | None = None,
+        output_layouts: Placement | None = None,
+        use_local_output: bool = True,
+    ):
+        super().__init__()
+        self.input_layouts = (input_layouts or Shard(-1),)
+        self.output_layouts = (output_layouts or Replicate(),)
+        self.desired_input_layouts = (Shard(-1),)
+        self.use_local_output = use_local_output
+    @staticmethod
+    def _prepare_input_fn(
+        input_layouts, desired_input_layouts, mod, inputs, device_mesh,
+    ):
+        x, y, weight, bias = inputs
+        if not isinstance(x, DTensor):
+            x = DTensor.from_local(x, device_mesh, input_layouts, run_check=False)
+        if x.placements != desired_input_layouts:
+            x = x.redistribute(placements=desired_input_layouts, async_op=True)
+        if not isinstance(y, DTensor):
+            y = DTensor.from_local(y, device_mesh, input_layouts, run_check=False)
+        if y.placements != desired_input_layouts:
+            y = y.redistribute(placements=desired_input_layouts, async_op=True)
+        if not isinstance(weight, DTensor):
+            weight = DTensor.from_local(weight, device_mesh, (Shard(1),))
+        if bias is not None and not isinstance(bias, DTensor):
+            bias = DTensor.from_local(bias, device_mesh, (Replicate(),))
+        return x, y, weight, bias
+    @staticmethod
+    def _prepare_output_fn(output_layouts, use_local_output, mod, outputs, device_mesh):
+        # Rowwise sharding produces partial output, depending on output layouts:
+        # 1. to replicate -> allreduce
+        # 2. to shard -> reduce_scatter
+        if outputs.placements != output_layouts:
+            outputs = outputs.redistribute(placements=output_layouts, async_op=True)
+        # back to local tensor if use_local_output is True
+        return outputs.to_local() if use_local_output else outputs
+    def _apply(self, module: nn.Module, device_mesh: DeviceMesh) -> nn.Module:
+        return distribute_module(
+            module,
+            device_mesh,
+            partition_fn=None,
+            input_fn=partial(self._prepare_input_fn, self.input_layouts, self.desired_input_layouts),
+            output_fn=partial(self._prepare_output_fn, self.output_layouts, self.use_local_output),
+        )

build/torch-cuda/modules/parallel.py ADDED Viewed

	@@ -0,0 +1,44 @@

+# Copyright (c) 2023-2026, Songlin Yang, Yu Zhang, Zhiyuan Li
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+# For a list of all contributors, visit:
+#   https://github.com/fla-org/flash-linear-attention/graphs/contributors
+import torch.nn as nn
+from torch.distributed import DeviceMesh
+from torch.distributed.tensor import distribute_module
+from torch.distributed.tensor.parallel import ParallelStyle
+from torch.distributed.tensor.placement_types import Placement
+try:
+    from torch.distributed.tensor import DTensor
+except (ImportError, AttributeError):
+    DTensor = None
+class PrepareModuleWeight(ParallelStyle):
+    def __init__(self, *, layouts: Placement | None = None):
+        super().__init__()
+        self.layouts = layouts
+    def _replicate_module_fn(
+        self,
+        name: str,
+        module: nn.Module,
+        device_mesh: DeviceMesh,
+    ):
+        for p_name, param in module.named_parameters():
+            replicated_param = nn.Parameter(
+                DTensor.from_local(param, device_mesh, [self.layouts], run_check=False),
+            )
+            module.register_parameter(p_name, replicated_param)
+    def _apply(self, module: nn.Module, device_mesh: DeviceMesh) -> nn.Module:
+        return distribute_module(
+            module,
+            device_mesh,
+            partition_fn=self._replicate_module_fn,
+            input_fn=None,
+            output_fn=None,
+        )

build/torch-cuda/modules/rotary.py ADDED Viewed

	@@ -0,0 +1,519 @@

+# Copyright (c) 2023-2026, Songlin Yang, Yu Zhang, Zhiyuan Li
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+# For a list of all contributors, visit:
+#   https://github.com/fla-org/flash-linear-attention/graphs/contributors
+import torch
+import torch.nn as nn
+import triton
+import triton.language as tl
+from einops import rearrange, repeat
+from ..modules.backends import dispatch
+from ..ops.utils import prepare_chunk_indices
+from ..utils import IS_AMD, autotune_cache_kwargs, get_multiprocessor_count, input_guard
+NUM_WARPS_AUTOTUNE = [2, 4, 8, 16] if IS_AMD else [2, 4, 8, 16, 32]
+def rotate_half(x, interleaved=False):
+    if not interleaved:
+        x1, x2 = x.chunk(2, dim=-1)
+        return torch.cat((-x2, x1), dim=-1)
+    else:
+        x1, x2 = x[..., ::2], x[..., 1::2]
+        return rearrange(torch.stack((-x2, x1), dim=-1), '... d two -> ... (d two)', two=2)
+def rotary_embedding_ref(x, cos, sin, interleaved=False):
+    ro_dim = cos.shape[-1] * 2
+    assert ro_dim <= x.shape[-1]
+    cos = repeat(cos, '... d -> ... 1 (2 d)' if not interleaved else '... d -> ... 1 (d 2)')
+    sin = repeat(sin, '... d -> ... 1 (2 d)' if not interleaved else '... d -> ... 1 (d 2)')
+    return torch.cat([x[..., :ro_dim] * cos + rotate_half(x[..., :ro_dim], interleaved) * sin, x[..., ro_dim:]], -1)
+@triton.autotune(
+    configs=[
+        triton.Config({}, num_warps=num_warps, num_stages=num_stages)
+        for num_warps in NUM_WARPS_AUTOTUNE
+        for num_stages in [2, 3, 4]
+    ],
+    key=['B', 'H', 'D', 'INTERLEAVED'],
+    **autotune_cache_kwargs,
+)
+@triton.jit(do_not_specialize=['T'])
+def rotary_embedding_kernel(
+    x,
+    cos,
+    sin,
+    y,
+    cu_seqlens,
+    chunk_indices,
+    seq_offsets,
+    T,
+    B: tl.constexpr,
+    H: tl.constexpr,
+    D: tl.constexpr,
+    R: tl.constexpr,
+    TR: tl.constexpr,
+    BT: tl.constexpr,
+    BD: tl.constexpr,
+    IS_SEQLEN_OFFSETS_TENSOR: tl.constexpr,
+    IS_VARLEN: tl.constexpr,
+    INTERLEAVED: tl.constexpr,
+    CONJUGATE: tl.constexpr,
+):
+    i_t, i_b, i_h = tl.program_id(0), tl.program_id(1), tl.program_id(2)
+    if IS_VARLEN:
+        i_n, i_t = tl.load(chunk_indices + i_t * 2).to(tl.int32), tl.load(chunk_indices + i_t * 2 + 1).to(tl.int32)
+        bos, eos = tl.load(cu_seqlens + i_n), tl.load(cu_seqlens + i_n + 1)
+        T = eos - bos
+        x = x + bos * H*D + i_h * D
+        y = y + bos * H*D + i_h * D
+    else:
+        i_n = i_b
+        x = x + i_n * T*H*D + i_h * D
+        y = y + i_n * T*H*D + i_h * D
+    if i_t * BT >= T:
+        return
+    o_t = i_t * BT + tl.arange(0, BT)
+    if not IS_SEQLEN_OFFSETS_TENSOR:
+        o_cs = o_t + seq_offsets
+    else:
+        o_cs = o_t + tl.load(seq_offsets + i_n)
+    m_t = (o_t >= 0) & (o_t < T) & (o_cs >= 0) & (o_cs < TR)
+    if not INTERLEAVED:
+        # Load the 1st and 2nd halves of x, do calculation, then store to 1st and 2nd halves of out
+        o_r = tl.arange(0, BD // 2)
+        p_x = x + o_t[:, None] * H*D + o_r[None, :]
+        p_cos = cos + (o_cs[:, None] * R + o_r[None, :])
+        p_sin = sin + (o_cs[:, None] * R + o_r[None, :])
+        mask = m_t[:, None] & (o_r < R)[None, :]
+        b_cos = tl.load(p_cos, mask=mask, other=1.0).to(tl.float32)
+        b_sin = tl.load(p_sin, mask=mask, other=0.0).to(tl.float32)
+        b_x0 = tl.load(p_x, mask=mask, other=0.0).to(tl.float32)
+        b_x1 = tl.load(p_x + R, mask=mask, other=0.0).to(tl.float32)
+        if CONJUGATE:
+            b_sin = -b_sin
+        b_o0 = b_x0 * b_cos - b_x1 * b_sin
+        b_o1 = b_x0 * b_sin + b_x1 * b_cos
+        # write back result
+        p_y = y + (o_t[:, None] * H*D + o_r[None, :])
+        tl.store(p_y, b_o0, mask=mask)
+        tl.store(p_y + R, b_o1, mask=mask)
+    else:
+        # We don't want to load x[0, 2, 4, ...] and x[1, 3, 5, ...] separately since both are slow.
+        # Instead, we load x0 = x[0, 1, 2, 3, ...] and x1 = x[1, 0, 3, 2, ...].
+        # Loading x0 will be fast but x1 will be slow.
+        # Then we load cos = cos[0, 0, 1, 1, ...] and sin = sin[0, 0, 1, 1, ...].
+        # Then we do the calculation and use tl.where to pick put the right outputs for the even
+        # and for the odd indices.
+        o_d = tl.arange(0, BD)
+        o_d_swap = o_d + ((o_d + 1) % 2) * 2 - 1  # 1, 0, 3, 2, 5, 4, ...
+        o_d_repeat = tl.arange(0, BD) // 2
+        p_x0 = x + o_t[:, None] * H*D + o_d[None, :]
+        p_x1 = x + o_t[:, None] * H*D + o_d_swap[None, :]
+        p_cos = cos + (o_cs[:, None] * R + o_d_repeat[None, :])
+        p_sin = sin + (o_cs[:, None] * R + o_d_repeat[None, :])
+        mask = m_t[:, None] & (o_d_repeat < R)[None, :]
+        b_cos = tl.load(p_cos, mask=mask, other=1.0).to(tl.float32)
+        b_sin = tl.load(p_sin, mask=mask, other=0.0).to(tl.float32)
+        b_x0 = tl.load(p_x0, mask=mask, other=0.0).to(tl.float32)
+        b_x1 = tl.load(p_x1, mask=mask, other=0.0).to(tl.float32)
+        if CONJUGATE:
+            b_sin = -b_sin
+        b_o0 = b_x0 * b_cos
+        b_o1 = b_x1 * b_sin
+        b_y = tl.where(o_d[None, :] % 2 == 0, b_o0 - b_o1, b_o0 + b_o1)
+        p_y = y + (o_t[:, None] * H*D + o_d[None, :])
+        tl.store(p_y, b_y, mask=mask)
+@dispatch('modules')
+def rotary_embedding_fwdbwd(
+    x: torch.Tensor,
+    cos: torch.Tensor,
+    sin: torch.Tensor,
+    seqlen_offsets: int | torch.Tensor = 0,
+    cu_seqlens: torch.Tensor | None = None,
+    interleaved: bool = False,
+    inplace: bool = False,
+    conjugate: bool = False,
+    chunk_indices: torch.LongTensor | None = None,
+) -> torch.Tensor:
+    """
+    Args:
+        x: [B, T, H, D].
+        cos: [TR, R / 2]
+        sin: [TR, R / 2]
+        seqlen_offsets: integer or integer tensor of size [N]
+        cu_seqlens: [N + 1,] or None
+    Returns:
+        y: [B, T, H, D]
+    """
+    is_varlen = cu_seqlens is not None
+    B, T, H, D = x.shape
+    N = B if not is_varlen else cu_seqlens.shape[0] - 1
+    TR, R = cos.shape
+    R2 = R * 2
+    assert D <= 256, "Only support D <= 256"
+    assert TR >= T, f"TR must be >= T, got {TR} and {T}"
+    assert cos.dtype == sin.dtype, f"cos and sin must have the same dtype, got {cos.dtype} and {sin.dtype}"
+    assert x.dtype == cos.dtype, f"Input and cos/sin must have the same dtype, got {x.dtype} and {cos.dtype}"
+    if isinstance(seqlen_offsets, torch.Tensor):
+        assert seqlen_offsets.shape == (N,)
+        assert seqlen_offsets.dtype in [torch.int32, torch.int64]
+    else:
+        assert seqlen_offsets + T <= TR
+    # zeros_like: rows the kernel skips (negative o_cs under left-padding) must be defined, not uninitialized.
+    y = torch.zeros_like(x) if not inplace else x
+    if R2 < D and not inplace:
+        y[..., R2:].copy_(x[..., R2:])
+    BD = triton.next_power_of_2(R2)
+    BT = min(128, triton.next_power_of_2(triton.cdiv(T, get_multiprocessor_count(x.device.index))))
+    if chunk_indices is None and is_varlen:
+        chunk_indices = prepare_chunk_indices(cu_seqlens, BT)
+    NT = len(chunk_indices) if is_varlen else triton.cdiv(T, BT)
+    grid = (NT, B, H)
+    rotary_embedding_kernel[grid](
+        x,
+        cos,
+        sin,
+        y,
+        cu_seqlens,
+        chunk_indices,
+        seqlen_offsets,
+        B=B,
+        T=T,
+        H=H,
+        D=D,
+        R=R,
+        TR=TR,
+        BT=BT,
+        BD=BD,
+        IS_SEQLEN_OFFSETS_TENSOR=isinstance(seqlen_offsets, torch.Tensor),
+        IS_VARLEN=is_varlen,
+        INTERLEAVED=interleaved,
+        CONJUGATE=conjugate,
+    )
+    return y
+class RotaryEmbeddingFunction(torch.autograd.Function):
+    @staticmethod
+    @input_guard
+    def forward(
+        ctx,
+        x,
+        cos,
+        sin,
+        interleaved=False,
+        inplace=False,
+        seqlen_offsets: int | torch.Tensor = 0,
+        cu_seqlens: torch.Tensor | None = None,
+        chunk_indices: torch.LongTensor | None = None,
+    ):
+        y = rotary_embedding_fwdbwd(
+            x,
+            cos,
+            sin,
+            seqlen_offsets=seqlen_offsets,
+            cu_seqlens=cu_seqlens,
+            interleaved=interleaved,
+            inplace=inplace,
+            chunk_indices=chunk_indices,
+        )
+        if isinstance(seqlen_offsets, int):
+            # Can't save int with save_for_backward
+            ctx.save_for_backward(cos, sin, cu_seqlens)
+            ctx.seqlen_offsets = seqlen_offsets
+        else:
+            ctx.save_for_backward(cos, sin, cu_seqlens, seqlen_offsets)
+            ctx.seqlen_offsets = None
+        ctx.interleaved = interleaved
+        ctx.inplace = inplace
+        ctx.chunk_indices = chunk_indices
+        return y if not inplace else x
+    @staticmethod
+    @input_guard
+    def backward(ctx, do):
+        seqlen_offsets = ctx.seqlen_offsets
+        if seqlen_offsets is None:
+            cos, sin, cu_seqlens, seqlen_offsets = ctx.saved_tensors
+        else:
+            cos, sin, cu_seqlens = ctx.saved_tensors
+        # TD [2023-09-02]: For some reason Triton (2.0.0.post1) errors with
+        # "[CUDA]: invalid device context", and cloning makes it work. Idk why. Triton 2.1.0 works.
+        if not ctx.interleaved and not ctx.inplace:
+            do = do.clone()
+        dx = rotary_embedding_fwdbwd(
+            do,
+            cos,
+            sin,
+            seqlen_offsets=seqlen_offsets,
+            cu_seqlens=cu_seqlens,
+            interleaved=ctx.interleaved,
+            inplace=ctx.inplace,
+            conjugate=True,
+            chunk_indices=ctx.chunk_indices,
+        )
+        return dx, None, None, None, None, None, None, None
+def rotary_embedding(
+    x,
+    cos,
+    sin,
+    interleaved=False,
+    inplace=False,
+    seqlen_offsets: int | torch.Tensor = 0,
+    cu_seqlens: torch.Tensor | None = None,
+    chunk_indices: torch.LongTensor | None = None,
+):
+    """
+    Args:
+        x: [B, T, H, D]
+        cos, sin: [TR, R//2]
+        interleaved:
+            If True, rotate pairs of even and odd dimensions (GPT-J style) instead of 1st half and 2nd half (GPT-NeoX style).
+        inplace:
+            If True, apply rotary embedding in-place.
+        seqlen_offsets: [N,] or int.
+            Each sequence in x is shifted by this amount.
+            Most commonly used in inference when we have KV cache.
+        cu_seqlens: [N + 1,] or None
+    Returns:
+        out: [B, T, H, D]
+    """
+    return RotaryEmbeddingFunction.apply(
+        x,
+        cos,
+        sin,
+        interleaved,
+        inplace,
+        seqlen_offsets,
+        cu_seqlens,
+        chunk_indices,
+    )
+class RotaryEmbedding(nn.Module):
+    """
+    The rotary position embeddings from RoFormer_ (Su et. al).
+    A crucial insight from the method is that the query and keys are
+    transformed by rotation matrices which depend on the relative positions.
+    Other implementations are available in the Rotary Transformer repo_ and in
+    GPT-NeoX_, GPT-NeoX was an inspiration
+    .. _RoFormer: https://arxiv.org/abs/2104.09864
+    .. _repo: https://github.com/ZhuiyiTechnology/roformer
+    .. _GPT-NeoX: https://github.com/EleutherAI/gpt-neox
+    If scale_base is not None, this implements XPos (Sun et al., https://arxiv.org/abs/2212.10554).
+    A recommended value for scale_base is 512: https://github.com/HazyResearch/flash-attention/issues/96
+    Reference: https://github.com/sunyt32/torchscale/blob/main/torchscale/component/xpos_relative_position.py
+    """
+    def __init__(
+        self,
+        dim: int,
+        base: float = 10000.0,
+        scale_base: float | None = None,
+        interleaved: bool = False,
+        pos_idx_in_fp32: bool = True,
+        device: torch.device | None = None,
+    ):
+        """
+        interleaved:
+            If True, rotate pairs of even and odd dimensions (GPT-J style) instead of 1st half and 2nd half (GPT-NeoX style).
+        pos_idx_in_fp32:
+            If True, the position indices [0.0, ..., seqlen - 1] are in fp32, otherwise they might be in lower precision.
+            This option was added because previously (before 2023-07-02), when we construct
+            the position indices, we use the dtype of self.inv_freq.
+            In most cases this would be fp32, but if the model is trained in pure bf16 (not mixed precision), then
+            self.inv_freq would be bf16, and the position indices are also in bf16.
+            Because of the limited precision of bf16 (e.g. 1995.0 is rounded to 2000.0), the
+            embeddings for some positions will coincide.
+            To maintain compatibility with models previously trained in pure bf16, we add this option.
+        """
+        super().__init__()
+        self.dim = dim
+        self.base = float(base)
+        self.scale_base = scale_base
+        self.interleaved = interleaved
+        self.pos_idx_in_fp32 = pos_idx_in_fp32
+        self.device = device
+        # Generate and save the inverse frequency buffer (non trainable)
+        self.register_buffer("inv_freq", torch.empty(-(dim // -2), dtype=torch.float32, device=device), persistent=False)
+        scale = None
+        if scale_base is not None:
+            scale = torch.empty(-(dim // -2), dtype=torch.float32, device=device)
+        self.register_buffer("scale", scale, persistent=False)
+        self._seq_len_cached = 0
+        self._cos_cached = None
+        self._sin_cached = None
+        self._cos_k_cached = None
+        self._sin_k_cached = None
+        self.reset_parameters()
+    def reset_parameters(self):
+        with torch.no_grad():
+            self.inv_freq.copy_(self._compute_inv_freq(device=self.inv_freq.device))
+            if self.scale_base is not None:
+                self.scale.copy_(self._compute_scale(device=self.scale.device))
+    def __repr__(self):
+        s = f"{self.__class__.__name__}("
+        s += f"dim={self.dim}, "
+        s += f"base={self.base}, "
+        s += f"interleaved={self.interleaved}, "
+        if self.scale_base is not None:
+            s += f"scale_base={self.scale_base}, "
+        s += f"pos_idx_in_fp32={self.pos_idx_in_fp32})"
+        return s
+    def _compute_inv_freq(self, device=None):
+        return 1.0 / (
+            self.base
+            ** (torch.arange(0, self.dim, 2, device=device, dtype=torch.float32) / self.dim)
+        )
+    def _compute_scale(self, device=None):
+        return (torch.arange(0, self.dim, 2, device=device, dtype=torch.float32) + 0.4 * self.dim) / (1.4 * self.dim)
+    def _update_cos_sin_cache(self, seqlen, device=None, dtype=None):
+        # Reset the tables if the sequence length has changed,
+        # if we're on a new device (possibly due to tracing for instance),
+        # or if we're switching from inference mode to training
+        if (
+            seqlen > self._seq_len_cached
+            or self._cos_cached is None
+            or self._cos_cached.device != device
+            or self._cos_cached.dtype != dtype
+            or (self.training and self._cos_cached.is_inference())
+        ):
+            self._seq_len_cached = seqlen
+            # We want fp32 here, not self.inv_freq.dtype, since the model could be loaded in bf16
+            # And the output of arange can be quite large, so bf16 would lose a lot of precision.
+            # However, for compatibility reason, we add an option to use the dtype of self.inv_freq.
+            if self.pos_idx_in_fp32:
+                t = torch.arange(seqlen, device=device, dtype=torch.float32)
+                # We want fp32 here as well since inv_freq will be multiplied with t, and the output
+                # will be large. Having it in bf16 will lose a lot of precision and cause the
+                # cos & sin output to change significantly.
+                # We want to recompute self.inv_freq if it was not loaded in fp32
+                if self.inv_freq.dtype != torch.float32:
+                    inv_freq = self._compute_inv_freq(device=device)
+                else:
+                    inv_freq = self.inv_freq
+            else:
+                t = torch.arange(seqlen, device=device, dtype=self.inv_freq.dtype)
+                inv_freq = self.inv_freq
+            # Don't do einsum, it converts fp32 to fp16 under AMP
+            # freqs = torch.einsum("i,j->ij", t, self.inv_freq)
+            freqs = torch.outer(t, inv_freq)
+            if self.scale is None:
+                self._cos_cached = torch.cos(freqs).to(dtype)
+                self._sin_cached = torch.sin(freqs).to(dtype)
+            else:
+                power = (
+                    torch.arange(seqlen, dtype=self.scale.dtype, device=self.scale.device)
+                    - seqlen // 2
+                ) / self.scale_base
+                scale = self.scale.to(device=power.device) ** rearrange(power, "s -> s 1")
+                # We want the multiplication by scale to happen in fp32
+                self._cos_cached = (torch.cos(freqs) * scale).to(dtype)
+                self._sin_cached = (torch.sin(freqs) * scale).to(dtype)
+                self._cos_k_cached = (torch.cos(freqs) / scale).to(dtype)
+                self._sin_k_cached = (torch.sin(freqs) / scale).to(dtype)
+    def forward(
+        self,
+        q: torch.Tensor,
+        k: torch.Tensor,
+        seqlen_offset: int | torch.Tensor = 0,
+        cu_seqlens: torch.Tensor | None = None,
+        max_seqlen: int | None = None,
+        chunk_indices: torch.LongTensor | None = None,
+    ) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
+        """
+        q: [B, T, H, D]
+        k: [B, T, H, D]
+        seqlen_offset:
+            [N] or int.
+            Each sequence in x is shifted by this amount.
+            Most commonly used in inference when we have KV cache.
+        cu_seqlens: [N + 1] or None
+        max_seqlen: int
+        """
+        if max_seqlen is not None:
+            self._update_cos_sin_cache(max_seqlen, device=q.device, dtype=q.dtype)
+        elif isinstance(seqlen_offset, int):
+            self._update_cos_sin_cache(q.shape[1] + seqlen_offset, device=q.device, dtype=q.dtype)
+        if self.scale is None:
+            q = rotary_embedding(
+                q,
+                self._cos_cached,
+                self._sin_cached,
+                interleaved=self.interleaved,
+                seqlen_offsets=seqlen_offset,
+                cu_seqlens=cu_seqlens,
+                chunk_indices=chunk_indices,
+            )
+            k = rotary_embedding(
+                k,
+                self._cos_cached,
+                self._sin_cached,
+                interleaved=self.interleaved,
+                seqlen_offsets=seqlen_offset,
+                cu_seqlens=cu_seqlens,
+                chunk_indices=chunk_indices,
+            )
+        else:
+            q = rotary_embedding(
+                q,
+                self._cos_cached,
+                self._sin_cached,
+                interleaved=self.interleaved,
+                seqlen_offsets=seqlen_offset,
+                cu_seqlens=cu_seqlens,
+                chunk_indices=chunk_indices,
+            )
+            k = rotary_embedding(
+                k,
+                self._cos_k_cached,
+                self._sin_k_cached,
+                interleaved=self.interleaved,
+                seqlen_offsets=seqlen_offset,
+                cu_seqlens=cu_seqlens,
+                chunk_indices=chunk_indices,
+            )
+        return q, k

build/torch-cuda/modules/token_shift.py ADDED Viewed

	@@ -0,0 +1,573 @@

+# Copyright (c) 2023-2026, Songlin Yang, Yu Zhang, Zhiyuan Li
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+# For a list of all contributors, visit:
+#   https://github.com/fla-org/flash-linear-attention/graphs/contributors
+import torch
+import triton
+import triton.language as tl
+from ..ops.utils import prepare_chunk_indices
+from ..utils import IS_AMD, IS_NPU, autotune_cache_kwargs, get_multiprocessor_count, input_guard, tensor_cache
+NUM_WARPS_AUTOTUNE = [2, 4, 8, 16] if IS_AMD else [2, 4, 8, 16, 32]
+# Ascend Triton rejects 2-D grids whose product exceeds 65535 unless
+# TRITON_ALL_BLOCKS_PARALLEL=1. Fall back to the long kernel instead.
+_NPU_MAX_TRITON_GRID = 65535
+def token_shift_ref(
+    x: torch.Tensor,
+    cu_seqlens: torch.Tensor | None = None,
+) -> torch.Tensor:
+    if cu_seqlens is not None:
+        # Variable length mode with cu_seqlens
+        assert x.dim() == 3, "Input must be [B, T, D]"
+        B, T, D = x.shape
+        assert B == 1, "Batch size must be 1 when using cu_seqlens"
+        result = torch.zeros_like(x)
+        N = cu_seqlens.shape[0] - 1
+        for i in range(N):
+            start = cu_seqlens[i].item()
+            end = cu_seqlens[i+1].item()
+            seq_len = end - start
+            if seq_len <= 1:
+                # For sequences of length 1 or 0, delta is simply -x
+                result[0, start:end] = -x[0, start:end]
+            else:
+                # For longer sequences, handle padding manually
+                shifted = torch.zeros_like(x[0, start:end])
+                shifted[1:] = x[0, start:end-1]
+                delta = shifted - x[0, start:end]
+                result[0, start:end] = delta
+        return result
+    else:
+        time_shift = torch.nn.ZeroPad2d((0, 0, 1, -1))
+        shifted = time_shift(x)
+        delta = shifted - x
+        return delta
+@triton.heuristics({
+    'IS_VARLEN': lambda args: args['cu_seqlens'] is not None,
+    'USE_INITIAL_STATE': lambda args: args['cache'] is not None,
+})
+@triton.autotune(
+    configs=[
+        triton.Config({}, num_warps=num_warps, num_stages=num_stages)
+        for num_warps in NUM_WARPS_AUTOTUNE
+        for num_stages in [1, 2, 3]
+    ],
+    key=['BD'],
+    **autotune_cache_kwargs,
+)
+@triton.jit
+def token_shift_fwd_kernel_short(
+    x,
+    y,
+    cu_seqlens,
+    cache,
+    cache_out,
+    T,
+    D: tl.constexpr,
+    BD: tl.constexpr,
+    IS_VARLEN: tl.constexpr,
+    USE_INITIAL_STATE: tl.constexpr,
+    STORE_FINAL_STATE: tl.constexpr,
+    IS_DECODE: tl.constexpr,
+):
+    i_b, i_t = tl.program_id(0), tl.program_id(1)
+    if IS_VARLEN:
+        i_n = i_b
+        bos, eos = tl.load(cu_seqlens + i_n).to(tl.int32), tl.load(cu_seqlens + i_n + 1).to(tl.int32)
+        g_t = i_t + bos
+        if g_t >= eos:
+            return
+        is_first_pos = (i_t == 0)
+        is_last_pos = (g_t == eos - 1)
+    else:
+        g_t = i_t
+        is_first_pos = (g_t == 0)
+        is_last_pos = (g_t == T - 1)
+    o_d = tl.arange(0, BD)
+    m_d = o_d < D
+    if IS_VARLEN:
+        base_offset = g_t * D + o_d
+    else:
+        base_offset = i_b * T*D + g_t * D + o_d
+    b_x = tl.load(x + base_offset, mask=m_d)
+    if IS_VARLEN:
+        cache_offset = i_n * D + o_d  # i_n is seq index
+    else:
+        cache_offset = i_b * D + o_d  # i_b is batch index
+    if IS_DECODE and USE_INITIAL_STATE:
+        b_cache = tl.load(cache + cache_offset, mask=m_d)
+        delta = b_cache - b_x
+        tl.store(y + base_offset, delta, mask=m_d)
+        if STORE_FINAL_STATE:
+            tl.store(cache_out + cache_offset, b_x, mask=m_d)
+        return
+    if is_first_pos:
+        # First position in sequence: delta = -hidden_states
+        if USE_INITIAL_STATE:
+            # cache shape: [N, D]
+            b_cache = tl.load(cache + cache_offset, mask=m_d)
+            delta = b_cache - b_x
+            tl.store(y + base_offset, delta, mask=m_d)
+        else:
+            tl.store(y + base_offset, -b_x, mask=m_d)
+        return
+    # Other positions: delta = prev - curr
+    if IS_VARLEN:
+        prev_offset = (g_t-1) * D + o_d
+    else:
+        prev_offset = i_b * T*D + (g_t-1) * D + o_d
+    prev_values = tl.load(x + prev_offset, mask=m_d)
+    delta = prev_values - b_x
+    tl.store(y + base_offset, delta, mask=m_d)
+    if STORE_FINAL_STATE:
+        if is_last_pos:
+            tl.store(cache_out + cache_offset, b_x, mask=m_d)
+@triton.heuristics({
+    'IS_VARLEN': lambda args: args['cu_seqlens'] is not None,
+    'USE_INITIAL_STATE': lambda args: args['cache'] is not None,
+})
+@triton.autotune(
+    configs=[
+        triton.Config({}, num_warps=num_warps, num_stages=num_stages)
+        for num_warps in NUM_WARPS_AUTOTUNE
+        for num_stages in [1, 2, 3]
+    ],
+    key=['BD', 'NB'],
+    **autotune_cache_kwargs,
+)
+@triton.jit
+def token_shift_fwd_kernel_long(
+    x,
+    y,
+    cu_seqlens,
+    chunk_indices,
+    cache,
+    cache_out,
+    T,
+    D: tl.constexpr,
+    BD: tl.constexpr,
+    BT: tl.constexpr,
+    NB: tl.constexpr,
+    ND: tl.constexpr,
+    IS_VARLEN: tl.constexpr,
+    USE_INITIAL_STATE: tl.constexpr,
+    STORE_FINAL_STATE: tl.constexpr,
+):
+    i_dt, i_b = tl.program_id(0), tl.program_id(1)
+    i_d, i_t = i_dt % ND, i_dt // ND
+    if IS_VARLEN:
+        i_n, i_t = tl.load(chunk_indices + i_t * 2).to(tl.int32), \
+            tl.load(chunk_indices + i_t * 2 + 1).to(tl.int32)
+        bos, eos = tl.load(cu_seqlens + i_n), tl.load(cu_seqlens + i_n + 1)
+        t_start = i_t * BT
+        t_end = tl.minimum(t_start + BT, eos - bos)
+    else:
+        i_n = i_b
+        bos, eos = i_b * T, (i_b + 1) * T
+        t_start = i_t * BT
+        t_end = tl.minimum(t_start + BT, T)
+    o_d = i_d * BD + tl.arange(0, BD)
+    m_d = o_d < D
+    for t in range(t_start, t_end):
+        global_t = bos + t
+        offset = global_t * D + o_d
+        b_x = tl.load(x + offset, mask=m_d)
+        is_first = (global_t == bos)
+        if is_first:
+            if USE_INITIAL_STATE:
+                # cache shape: [N, D]
+                cache_off = i_n * D + o_d if IS_VARLEN else i_b * D + o_d
+                b_cache = tl.load(cache + cache_off, mask=m_d)
+                delta = b_cache - b_x
+            else:
+                delta = -b_x
+        else:
+            prev_off = offset - D
+            b_prev = tl.load(x + prev_off, mask=m_d)
+            delta = b_prev - b_x
+        tl.store(y + offset, delta, mask=m_d)
+        if STORE_FINAL_STATE:
+            if global_t == eos - 1:
+                cache_out_off = i_n * D + o_d if IS_VARLEN else i_b * D + o_d
+                tl.store(cache_out + cache_out_off, b_x, mask=m_d)
+@triton.heuristics({
+    'IS_VARLEN': lambda args: args['cu_seqlens'] is not None,
+    'USE_INITIAL_STATE': lambda args: args['grad_cache_out'] is not None,
+    'HAS_DCACHE': lambda args: args['grad_cache_in'] is not None,
+})
+@triton.autotune(
+    configs=[
+        triton.Config({}, num_warps=num_warps, num_stages=num_stages)
+        for num_warps in NUM_WARPS_AUTOTUNE
+        for num_stages in [1, 2, 3]
+    ],
+    key=['BD'],
+    **autotune_cache_kwargs,
+)
+@triton.jit
+def token_shift_bwd_kernel_short(
+    dx,
+    dy,
+    cu_seqlens,
+    grad_cache_in,
+    grad_cache_out,
+    T,
+    D: tl.constexpr,
+    BD: tl.constexpr,
+    IS_VARLEN: tl.constexpr,
+    USE_INITIAL_STATE: tl.constexpr,
+    HAS_DCACHE: tl.constexpr,
+):
+    i_b, i_t = tl.program_id(0), tl.program_id(1)
+    if IS_VARLEN:
+        i_n = i_b
+        bos, eos = tl.load(cu_seqlens + i_n).to(tl.int32), tl.load(cu_seqlens + i_n + 1).to(tl.int32)
+        g_t = i_t + bos
+        if g_t >= eos:
+            return
+        is_first_pos = (g_t == bos)
+        is_last_pos = (g_t == eos - 1)
+    else:
+        g_t = i_t
+        is_first_pos = (g_t == 0)
+        is_last_pos = (g_t == T - 1)
+    o_d = tl.arange(0, BD)
+    m_d = o_d < D
+    if IS_VARLEN:
+        base_offset = g_t * D + o_d
+        # This should not be used for varlen
+        cache_off = i_n * D + o_d
+    else:
+        base_offset = i_b * T * D + g_t * D + o_d
+        cache_off = i_b * D + o_d
+    b_dy = tl.load(dy + base_offset, mask=m_d)
+    if is_last_pos:
+        # grad = -grad_delta[t] + grad_cache_in（from next rank）
+        if HAS_DCACHE:
+            b_dy_cache = tl.load(grad_cache_in + cache_off, mask=m_d)
+            b_dx = -b_dy + b_dy_cache
+        else:
+            b_dx = -b_dy
+    else:
+        # grad = -grad_delta[t] + grad_delta[t+1]
+        if IS_VARLEN:
+            next_offset = (g_t + 1) * D + o_d
+        else:
+            next_offset = i_b * T * D + (g_t + 1) * D + o_d
+        b_dx = -b_dy + tl.load(dy + next_offset, mask=m_d)
+    tl.store(dx + base_offset, b_dx, mask=m_d)
+    if USE_INITIAL_STATE:
+        if is_first_pos:
+            tl.store(grad_cache_out + cache_off, b_dy, mask=m_d)
+@triton.heuristics({
+    'IS_VARLEN': lambda args: args['cu_seqlens'] is not None,
+    'USE_INITIAL_STATE': lambda args: args['grad_cache_out'] is not None,
+    'HAS_DCACHE': lambda args: args['grad_cache_in'] is not None,
+})
+@triton.autotune(
+    configs=[
+        triton.Config({}, num_warps=num_warps, num_stages=num_stages)
+        for num_warps in NUM_WARPS_AUTOTUNE
+        for num_stages in [1, 2, 3]
+    ],
+    key=['BD', 'NB'],
+    **autotune_cache_kwargs,
+)
+@triton.jit
+def token_shift_bwd_kernel_long(
+    dx,
+    dy,
+    cu_seqlens,
+    chunk_indices,
+    grad_cache_in,
+    grad_cache_out,
+    T,
+    D: tl.constexpr,
+    BD: tl.constexpr,
+    BT: tl.constexpr,
+    NB: tl.constexpr,
+    ND: tl.constexpr,
+    IS_VARLEN: tl.constexpr,
+    USE_INITIAL_STATE: tl.constexpr,
+    HAS_DCACHE: tl.constexpr,
+):
+    i_dt, i_b = tl.program_id(0), tl.program_id(1)
+    i_d, i_t_blk = i_dt % ND, i_dt // ND
+    if IS_VARLEN:
+        i_n, i_t_blk = tl.load(chunk_indices + i_t_blk * 2).to(tl.int32), \
+            tl.load(chunk_indices + i_t_blk * 2 + 1).to(tl.int32)
+        bos, eos = tl.load(cu_seqlens + i_n), tl.load(cu_seqlens + i_n + 1)
+        t_start = i_t_blk * BT
+        t_end = tl.minimum(t_start + BT, eos - bos)
+    else:
+        i_n = i_b
+        bos, eos = i_b * T, (i_b + 1) * T
+        t_start = i_t_blk * BT
+        t_end = tl.minimum(t_start + BT, T)
+    o_d = i_d * BD + tl.arange(0, BD)
+    m_d = o_d < D
+    cache_off = i_n * D + o_d if IS_VARLEN else i_b * D + o_d
+    for t in range(t_start, t_end):
+        global_t = bos + t
+        offset = global_t * D + o_d
+        b_dy = tl.load(dy + offset, mask=m_d)
+        if global_t == eos - 1:
+            if HAS_DCACHE:
+                b_dy_cache = tl.load(grad_cache_in + cache_off, mask=m_d)
+                b_dx = -b_dy + b_dy_cache
+            else:
+                b_dx = -b_dy
+        else:
+            next_off = offset + D
+            b_dx = -b_dy + tl.load(dy + next_off, mask=m_d)
+        tl.store(dx + offset, b_dx, mask=m_d)
+        if USE_INITIAL_STATE:
+            if global_t == bos:
+                tl.store(grad_cache_out + cache_off, b_dy, mask=m_d)
+@tensor_cache
+def prepare_maxlens(cu_seqlens: torch.LongTensor) -> int:
+    return torch.max(cu_seqlens.diff()).item()
+def token_shift_fwd(
+    x: torch.Tensor,
+    cu_seqlens: torch.Tensor | None = None,
+    cache: torch.Tensor | None = None,
+    output_cache: bool = False,
+    chunk_indices: torch.LongTensor | None = None,
+) -> torch.Tensor:
+    B, T, D = x.shape
+    y = torch.empty_like(x)
+    if cu_seqlens is not None:
+        T = prepare_maxlens(cu_seqlens)
+        N = len(cu_seqlens) - 1
+    else:
+        N = B
+    use_short_kernel = T <= 4096
+    if IS_NPU and use_short_kernel and N * T > _NPU_MAX_TRITON_GRID:
+        use_short_kernel = False
+    if output_cache:
+        cache_out = torch.empty((N, D), device=x.device, dtype=x.dtype)
+    else:
+        cache_out = None
+    if use_short_kernel:
+        if cu_seqlens is not None:
+            N = len(cu_seqlens) - 1
+        else:
+            N = B
+        BD = triton.next_power_of_2(D)
+        grid = (N, T)
+        IS_DECODE = T == 1 or (B == 1 and T == N)
+        token_shift_fwd_kernel_short[grid](
+            x=x,
+            y=y,
+            cu_seqlens=cu_seqlens,
+            cache=cache,
+            cache_out=cache_out,
+            T=T,
+            D=D,
+            BD=BD,
+            STORE_FINAL_STATE=output_cache,
+            IS_DECODE=IS_DECODE,
+        )
+    else:
+        BT = min(64, triton.next_power_of_2(triton.cdiv(max(16, B*T), get_multiprocessor_count(x.device.index))))
+        if chunk_indices is None and cu_seqlens is not None:
+            chunk_indices = prepare_chunk_indices(cu_seqlens, BT)
+        NT = len(chunk_indices) if cu_seqlens is not None else triton.cdiv(T, BT)
+        BD = triton.next_power_of_2(D)
+        ND = triton.cdiv(D, BD)
+        NB = triton.cdiv(B*T, 1024)
+        def grid(meta): return (ND * NT, 1 if cu_seqlens is not None else N)
+        token_shift_fwd_kernel_long[grid](
+            x,
+            y,
+            cu_seqlens,
+            chunk_indices,
+            cache,
+            cache_out,
+            T,
+            D=D,
+            BD=BD,
+            BT=BT,
+            NB=NB,
+            ND=ND,
+            STORE_FINAL_STATE=output_cache,
+        )
+    return y, N, T, use_short_kernel, cache_out
+def token_shift_bwd(
+    dy: torch.Tensor,
+    N: int,
+    T: int,
+    dcache: torch.Tensor | None = None,
+    cu_seqlens: torch.Tensor | None = None,
+    use_short_kernel: bool = True,
+    has_init_cache: bool = False,
+    chunk_indices: torch.LongTensor | None = None,
+) -> torch.Tensor:
+    D = dy.shape[2]
+    BD = triton.next_power_of_2(D)
+    dx = torch.empty_like(dy)
+    if has_init_cache:
+        grad_cache_out = torch.empty((N, D), device=dy.device, dtype=dy.dtype)
+    else:
+        grad_cache_out = None
+    if use_short_kernel:
+        grid = (N, T)
+        token_shift_bwd_kernel_short[grid](
+            dy=dy,
+            dx=dx,
+            cu_seqlens=cu_seqlens,
+            grad_cache_in=dcache,
+            grad_cache_out=grad_cache_out,
+            T=T,
+            D=D,
+            BD=BD,
+        )
+    else:
+        BT = min(64, triton.next_power_of_2(triton.cdiv(max(16, dy.numel() // D),
+                                                        get_multiprocessor_count(dy.device.index))))
+        if chunk_indices is None and cu_seqlens is not None:
+            chunk_indices = prepare_chunk_indices(cu_seqlens, BT)
+        NT = len(chunk_indices) if cu_seqlens is not None else triton.cdiv(T, BT)
+        NB = triton.cdiv(N * dy.shape[1], 1024)
+        BD = triton.next_power_of_2(D)
+        ND = triton.cdiv(D, BD)
+        def grid(meta): return (ND * NT, 1 if cu_seqlens is not None else N)
+        token_shift_bwd_kernel_long[grid](
+            dx,
+            dy,
+            cu_seqlens,
+            chunk_indices,
+            dcache,
+            grad_cache_out,
+            T,
+            D=D,
+            BD=BD,
+            BT=BT,
+            NB=NB,
+            ND=ND,
+        )
+    return dx, grad_cache_out
+class TokenShift(torch.autograd.Function):
+    @staticmethod
+    @input_guard
+    def forward(ctx, x: torch.Tensor, cu_seqlens: torch.Tensor | None = None,
+                cache: torch.Tensor | None = None, output_cache: bool = False,
+                chunk_indices: torch.LongTensor | None = None):
+        output, N, T, use_short_kernel, cache_out = token_shift_fwd(x, cu_seqlens, cache, output_cache, chunk_indices)
+        ctx.cu_seqlens = cu_seqlens
+        ctx.chunk_indices = chunk_indices
+        ctx.N = N
+        ctx.T = T
+        ctx.use_short_kernel = use_short_kernel
+        ctx.has_cache = cache is not None
+        return output, cache_out
+    @staticmethod
+    @input_guard
+    def backward(ctx, dy: torch.Tensor, dcache: torch.Tensor | None = None):
+        dx, grad_cache = token_shift_bwd(dy, ctx.N, ctx.T, dcache, ctx.cu_seqlens,
+                                         ctx.use_short_kernel, ctx.has_cache, ctx.chunk_indices)
+        return dx, None, grad_cache, None, None
+@torch.compiler.disable
+def token_shift(
+    x: torch.Tensor,
+    cu_seqlens: torch.LongTensor | None = None,
+    cache: torch.Tensor | None = None,
+    output_cache: bool = False,
+    chunk_indices: torch.LongTensor | None = None,
+):
+    """
+    Token-shift operation implemented with Triton kernels.
+    Args:
+        x: Input tensor of shape [B, T, D] (or [1, T, D] when `cu_seqlens` is supplied).
+        cu_seqlens: Optional cumulative sequence lengths of shape [B + 1].
+                    When supplied, `x.shape[0]` must be 1 and `x.dim()` must be 3.
+        cache: Optional cache tensor of shape [N, D] that holds the last token
+               from the previous call.
+        output_cache: Whether to return the updated cache alongside the output.
+                      In previous versions this parameter did not exist and the
+                      cache was always dropped; to preserve backward compatibility
+                      the default is False.
+    Returns:
+        output: Tensor of shape [B, T, D] after applying the token-shift.
+        cache_out: Tensor of shape [B, 1, D] containing the last token that
+                   should be fed as `cache` in the next call.  Only returned
+                   when `output_cache=True`.
+    """
+    if cu_seqlens is not None:
+        assert x.dim() == 3, "Input must be [B, T, D]"
+        assert x.shape[0] == 1, "Batch size must be 1 when using cu_seqlens"
+    output, cache_out = TokenShift.apply(x, cu_seqlens, cache, output_cache, chunk_indices)
+    if output_cache:
+        return output, cache_out
+    else:
+        return output

build/torch-cuda/modules/token_shift_cp.py ADDED Viewed

	@@ -0,0 +1,229 @@

+# Copyright (c) 2023-2026, Songlin Yang, Yu Zhang, Zhiyuan Li
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+# For a list of all contributors, visit:
+#   https://github.com/fla-org/flash-linear-attention/graphs/contributors
+"""
+Context Parallel support for Token Shift.
+Token shift has a 1-token dependency on previous tokens:
+    y[t] = x[t-1] - x[t]  (for t > 0)
+    y[0] = cache - x[0]   (cache is the last token from previous rank)
+In CP mode, non-first ranks need the last token from the previous rank as cache.
+Backward: non-last ranks need to send the last token's gradient to previous rank.
+"""
+import torch
+import torch.distributed as dist
+from ..modules.token_shift import token_shift_bwd, token_shift_fwd
+from ..ops.cp import FLACPContext, conv_cp_send_recv_bwd, conv_cp_send_recv_fwd
+class TokenShiftCPFunction(torch.autograd.Function):
+    """
+    Context Parallel version of TokenShift.
+    Forward:
+        1. Get last token from previous rank to construct cache
+        2. Call token_shift_fwd with cache
+    Backward:
+        1. Call token_shift_bwd to get dx
+        2. Sync communication: add next rank's first token gradient to current rank's last token
+    """
+    @staticmethod
+    def _prepare_cache_for_cp(
+        x: torch.Tensor,
+        cu_seqlens: torch.Tensor | None,
+        context: FLACPContext,
+        group: dist.ProcessGroup | None,
+    ) -> tuple[torch.Tensor | None, int]:
+        """Prepare cache for CP forward pass by communicating with previous rank.
+        Args:
+            x: Input tensor of shape [1, T, D]
+            cu_seqlens: Cumulative sequence lengths
+            context: CP context
+            group: Process group for communication
+        Returns:
+            cache: Cache tensor of shape [N, D] or None
+            pre_num_tokens: Number of tokens from previous rank for the first sequence
+        """
+        if group is None:
+            return None, 0
+        D = x.shape[-1]
+        cache = None
+        pre_num_tokens = 0
+        if not context.is_first_rank:
+            # Non-first rank: need cache from previous rank
+            assert x.dim() == 3 and x.shape[0] == 1, f"CP requires [1, T, D], got {x.shape}"
+            x_2d = x.squeeze(0)  # [T, D]
+            last_token = x_2d[-1:].contiguous()  # [1, D]
+            prev_last_token = conv_cp_send_recv_fwd(last_token, group)  # [1, D]
+            # For varlen: only the first sequence needs cache from prev rank
+            N = len(cu_seqlens) - 1 if cu_seqlens is not None else 1
+            cache = torch.zeros(N, D, device=x.device, dtype=x.dtype)
+            # pre_num_conv_tokens tells us how many tokens from prev rank
+            # belong to the first sequence on this rank
+            pre_num_tokens = getattr(context, 'pre_num_conv_tokens', 0)
+            if pre_num_tokens > 0:
+                # The prev rank's last token is used as cache for first sequence
+                cache[0] = prev_last_token[0]
+        else:
+            # First rank: participate in send but don't use received data
+            x_2d = x.squeeze(0)
+            last_token = x_2d[-1:].contiguous()
+            _ = conv_cp_send_recv_fwd(last_token, group)
+        return cache, pre_num_tokens
+    @staticmethod
+    def _correct_dx_for_cp(
+        dx: torch.Tensor,
+        grad_cache: torch.Tensor | None,
+        group: dist.ProcessGroup | None,
+        is_first_rank: bool,
+        pre_num_tokens: int = 0,
+    ) -> None:
+        """Correct dx gradients for CP backward pass.
+        Args:
+            dx: Gradient tensor to be corrected, shape [1, T, D]
+            grad_cache: Gradient w.r.t. cache, shape [N, D] or None
+            group: Process group
+            is_first_rank: Whether this is the first rank
+            pre_num_tokens: Number of tokens from previous rank for first sequence
+        """
+        if group is None:
+            return
+        D = dx.shape[-1]
+        # Prepare gradient to send to previous rank
+        if grad_cache is not None and pre_num_tokens > 0:
+            # Only first sequence's cache gradient is relevant
+            d_cache = grad_cache[0:1]  # [1, D]
+        else:
+            d_cache = torch.zeros(1, D, device=dx.device, dtype=dx.dtype)
+        # Send to previous rank, receive from next rank
+        recv_grad = conv_cp_send_recv_bwd(d_cache, group)  # [1, D]
+        # Add received gradient to current rank's last token
+        dx[0, -1, :].add_(recv_grad[0])
+    @staticmethod
+    def forward(
+        ctx,
+        x: torch.Tensor,
+        cu_seqlens: torch.Tensor | None,
+        chunk_indices: torch.Tensor | None,
+        cp_context: FLACPContext | None,
+    ):
+        if cp_context is None:
+            raise ValueError("cp_context must be provided for TokenShiftCPFunction")
+        cu_seqlens = cp_context.cu_seqlens
+        group = cp_context.group
+        # Prepare cache for CP
+        cache, pre_num_tokens = TokenShiftCPFunction._prepare_cache_for_cp(
+            x=x,
+            cu_seqlens=cu_seqlens,
+            context=cp_context,
+            group=group,
+        )
+        # Save for backward
+        ctx.cu_seqlens = cu_seqlens
+        ctx.chunk_indices = chunk_indices
+        ctx.group = group
+        ctx.has_cache = cache is not None
+        ctx.is_first_rank = cp_context.is_first_rank
+        ctx.pre_num_tokens = pre_num_tokens
+        # Call original forward
+        y, N, T, use_short_kernel, cache_out = token_shift_fwd(
+            x=x,
+            cu_seqlens=cu_seqlens,
+            cache=cache,
+            output_cache=True,
+            chunk_indices=chunk_indices,
+        )
+        ctx.N = N
+        ctx.T = T
+        ctx.use_short_kernel = use_short_kernel
+        return y
+    @staticmethod
+    def backward(ctx, dy: torch.Tensor):
+        group = ctx.group
+        # Prepare dcache for backward
+        # For CP: non-last rank needs to receive gradient from next rank
+        # This is handled in _correct_dx_for_cp after computing dx
+        dcache = None  # Will be computed by token_shift_bwd
+        # Call original backward
+        dx, grad_cache = token_shift_bwd(
+            dy=dy,
+            N=ctx.N,
+            T=ctx.T,
+            dcache=dcache,
+            cu_seqlens=ctx.cu_seqlens,
+            use_short_kernel=ctx.use_short_kernel,
+            has_init_cache=ctx.has_cache,
+            chunk_indices=ctx.chunk_indices,
+        )
+        # Correct dx gradients for CP
+        TokenShiftCPFunction._correct_dx_for_cp(
+            dx=dx,
+            grad_cache=grad_cache,
+            group=group,
+            is_first_rank=ctx.is_first_rank,
+            pre_num_tokens=ctx.pre_num_tokens,
+        )
+        return dx, None, None, None
+@torch.compiler.disable
+def token_shift_cp(
+    x: torch.Tensor,
+    cp_context: FLACPContext,
+    cu_seqlens: torch.Tensor | None = None,
+    chunk_indices: torch.Tensor | None = None,
+):
+    """
+    Context Parallel version of token_shift.
+    Args:
+        x: Input tensor of shape [1, T, D]
+        cp_context: CP context (required for CP mode)
+        cu_seqlens: Cumulative sequence lengths
+        chunk_indices: Chunk indices for variable-length sequences
+    Returns:
+        output: Tensor of shape [1, T, D] after applying token-shift
+    """
+    if cp_context is None:
+        raise ValueError("cp_context must be provided for token_shift_cp")
+    assert cp_context.cu_seqlens is not None, "cu_seqlens must be provided for token_shift_cp"
+    return TokenShiftCPFunction.apply(
+        x, cu_seqlens, chunk_indices, cp_context
+    )

build/torch-cuda/ops/__init__.py ADDED Viewed

	@@ -0,0 +1,91 @@

+# Copyright (c) 2023-2026, Songlin Yang, Yu Zhang, Zhiyuan Li
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+# For a list of all contributors, visit:
+#   https://github.com/fla-org/flash-linear-attention/graphs/contributors
+from .abc import chunk_abc
+from .attn import parallel_attn
+from .attnres import fused_attnres
+from .based import fused_chunk_based, parallel_based
+from .comba import chunk_comba, fused_recurrent_comba
+from .delta_rule import chunk_delta_rule, fused_chunk_delta_rule, fused_recurrent_delta_rule
+from .forgetting_attn import parallel_forgetting_attn
+from .gated_delta_rule import chunk_gated_delta_rule, chunk_gdn, fused_recurrent_gated_delta_rule, fused_recurrent_gdn
+from .generalized_delta_rule import (
+    chunk_dplr_delta_rule,
+    chunk_iplr_delta_rule,
+    fused_recurrent_dplr_delta_rule,
+    fused_recurrent_iplr_delta_rule,
+)
+from .gla import chunk_gla, fused_chunk_gla, fused_recurrent_gla
+from .gsa import chunk_gsa, fused_recurrent_gsa
+from .hgrn import fused_recurrent_hgrn
+from .kda import chunk_kda, fused_recurrent_kda
+from .lightning_attn import chunk_lightning_attn, fused_recurrent_lightning_attn
+from .linear_attn import chunk_linear_attn, fused_chunk_linear_attn, fused_recurrent_linear_attn
+from .log_linear_attn import chunk_log_linear_attn
+from .mesa_net import chunk_mesa_net
+from .nsa import parallel_nsa
+from .parallax import parallel_parallax
+from .path_attn import parallel_path_attn
+from .retention import chunk_retention, fused_chunk_retention, fused_recurrent_retention, parallel_retention
+from .rwkv6 import chunk_rwkv6, fused_recurrent_rwkv6
+from .rwkv7 import chunk_rwkv7, fused_recurrent_rwkv7
+from .simple_gla import chunk_simple_gla, fused_chunk_simple_gla, fused_recurrent_simple_gla, parallel_simple_gla
+from .wall_attn import parallel_wall_attn, parallel_wall_attn_decode
+__all__ = [
+    'chunk_abc',
+    'chunk_comba',
+    'chunk_delta_rule',
+    'chunk_dplr_delta_rule',
+    'chunk_gated_delta_rule',
+    'chunk_gdn',
+    'chunk_gla',
+    'chunk_gsa',
+    'chunk_iplr_delta_rule',
+    'chunk_kda',
+    'chunk_lightning_attn',
+    'chunk_linear_attn',
+    'chunk_log_linear_attn',
+    'chunk_mesa_net',
+    'chunk_retention',
+    'chunk_rwkv6',
+    'chunk_rwkv7',
+    'chunk_simple_gla',
+    'fused_attnres',
+    'fused_chunk_based',
+    'fused_chunk_delta_rule',
+    'fused_chunk_gla',
+    'fused_chunk_linear_attn',
+    'fused_chunk_retention',
+    'fused_chunk_simple_gla',
+    'fused_recurrent_comba',
+    'fused_recurrent_delta_rule',
+    'fused_recurrent_dplr_delta_rule',
+    'fused_recurrent_gated_delta_rule',
+    'fused_recurrent_gdn',
+    'fused_recurrent_gla',
+    'fused_recurrent_gsa',
+    'fused_recurrent_hgrn',
+    'fused_recurrent_iplr_delta_rule',
+    'fused_recurrent_kda',
+    'fused_recurrent_lightning_attn',
+    'fused_recurrent_linear_attn',
+    'fused_recurrent_retention',
+    'fused_recurrent_rwkv6',
+    'fused_recurrent_rwkv7',
+    'fused_recurrent_simple_gla',
+    'parallel_attn',
+    'parallel_based',
+    'parallel_forgetting_attn',
+    'parallel_nsa',
+    'parallel_parallax',
+    'parallel_path_attn',
+    'parallel_retention',
+    'parallel_simple_gla',
+    'parallel_wall_attn',
+    'parallel_wall_attn_decode',
+]

build/torch-cuda/ops/abc/__init__.py ADDED Viewed

	@@ -0,0 +1,12 @@

+# Copyright (c) 2023-2026, Songlin Yang, Yu Zhang, Zhiyuan Li
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+# For a list of all contributors, visit:
+#   https://github.com/fla-org/flash-linear-attention/graphs/contributors
+from .chunk import chunk_abc
+__all__ = [
+    'chunk_abc',
+]

build/torch-cuda/ops/abc/chunk.py ADDED Viewed

	@@ -0,0 +1,1119 @@

+# Copyright (c) 2023-2026, Songlin Yang, Yu Zhang, Zhiyuan Li
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+# For a list of all contributors, visit:
+#   https://github.com/fla-org/flash-linear-attention/graphs/contributors
+import torch
+import triton
+import triton.language as tl
+from ...ops.utils import softmax_bwd, softmax_fwd
+from ...ops.utils.logcumsumexp import logcumsumexp_fwd_kernel
+from ...ops.utils.op import exp
+from ...utils import input_guard
+@triton.jit(do_not_specialize=['T'])
+def chunk_abc_fwd_kernel_h(
+    k,
+    v,
+    z,
+    h,
+    h0,
+    ht,
+    T,
+    K: tl.constexpr,
+    V: tl.constexpr,
+    BT: tl.constexpr,
+    BK: tl.constexpr,
+    BV: tl.constexpr,
+    NT: tl.constexpr,
+    NORMK: tl.constexpr,
+    USE_INITIAL_STATE: tl.constexpr,
+    STORE_FINAL_STATE: tl.constexpr,
+):
+    i_v, i_k, i_bh = tl.program_id(0), tl.program_id(1), tl.program_id(2)
+    b_h = tl.zeros([BK, BV], dtype=tl.float32)
+    if USE_INITIAL_STATE:
+        p_h = tl.make_block_ptr(h0 + i_bh * K * V, (K, V), (V, 1), (i_k * BK, i_v * BV), (BK, BV), (1, 0))
+        b_h += tl.load(p_h, boundary_check=(0, 1)).to(tl.float32)
+    if NORMK:
+        p_z0 = tl.make_block_ptr(z + i_bh * T*K, (T * K,), (1,), (i_k * BK,), (BK,), (0,))
+    else:
+        p_z0 = tl.make_block_ptr(z + i_bh * T*V, (T * V,), (1,), (i_v * BV,), (BV,), (0,))
+    b_zp = tl.load(p_z0).to(tl.float32)
+    for i_t in range(NT):
+        p_k = tl.make_block_ptr(k + i_bh * T*K, (K, T), (1, K), (i_k * BK, i_t * BT), (BK, BT), (0, 1))
+        p_v = tl.make_block_ptr(v + i_bh * T*V, (T, V), (V, 1), (i_t * BT, i_v * BV), (BT, BV), (1, 0))
+        p_h = tl.make_block_ptr(h + i_bh * NT*K*V + i_t * K * V, (K, V), (V, 1), (i_k * BK, i_v * BV), (BK, BV), (1, 0))
+        tl.store(p_h, b_h.to(p_h.dtype.element_ty), boundary_check=(0, 1))
+        # [BK, BT]
+        b_k = tl.load(p_k, boundary_check=(0, 1))
+        # [BT, BV]
+        b_v = tl.load(p_v, boundary_check=(0, 1))
+        if NORMK:
+            p_zc = tl.make_block_ptr(z + i_bh * T*K, (T * K,), (1,), ((i_t * BT + BT - 1) * K + i_k * BK,), (BK,), (0,))
+            # [BK,]
+            b_zc = tl.load(p_zc, boundary_check=(0,))
+            b_r, b_zp = exp(b_zp - b_zc), b_zc
+            # [BK, BV]
+            b_h = b_h * b_r[:, None]
+            b_k = exp(b_k - b_zc[:, None]).to(b_k.dtype)
+        else:
+            p_zc = tl.make_block_ptr(z + i_bh * T*V, (T * V,), (1,), ((i_t * BT + BT - 1) * V + i_v * BV,), (BV,), (0,))
+            # [BV,]
+            b_zc = tl.load(p_zc, boundary_check=(0,))
+            b_r, b_zp = exp(b_zp - b_zc), b_zc
+            # [BK, BV]
+            b_h = b_h * b_r[None, :]
+            b_v = exp(b_v - b_zc[None, :]).to(b_v.dtype)
+        # [BK, BV]
+        b_h += tl.dot(b_k, b_v, allow_tf32=False)
+    if STORE_FINAL_STATE:
+        p_h = tl.make_block_ptr(ht + i_bh * K * V, (K, V), (V, 1), (i_k * BK, i_v * BV), (BK, BV), (1, 0))
+        tl.store(p_h, b_h.to(p_h.dtype.element_ty), boundary_check=(0, 1))
+@triton.jit(do_not_specialize=['T'])
+def chunk_abc_fwd_kernel_intra_K(
+    v,
+    z,
+    o,
+    A,
+    T,
+    V: tl.constexpr,
+    BT: tl.constexpr,
+    BC: tl.constexpr,
+    BV: tl.constexpr,
+    NC: tl.constexpr,
+):
+    i_v, i_c, i_bh = tl.program_id(0), tl.program_id(1), tl.program_id(2)
+    i_t, i_i = i_c // NC, i_c % NC
+    p_z = tl.make_block_ptr(z + i_bh * T*V, (T, V), (V, 1), (i_t * BT + i_i * BC, i_v * BV), (BC, BV), (1, 0))
+    p_zn = tl.make_block_ptr(z + i_bh * T*V, (T * V,), (1,), ((i_t * BT + i_i * BC) * V + i_v * BV,), (BV,), (0,))
+    # [BV,]
+    b_zn = tl.load(p_zn, boundary_check=(0,))
+    # [BC, BV]
+    b_o = tl.zeros([BC, BV], dtype=tl.float32)
+    for i_j in range(0, i_i):
+        p_A = tl.make_block_ptr(A + i_bh * T * BT, (T, BT), (BT, 1), (i_t * BT + i_i * BC, i_j * BC), (BC, BC), (1, 0))
+        p_v = tl.make_block_ptr(v + i_bh * T*V, (T, V), (V, 1), (i_t * BT + i_j * BC, i_v * BV), (BC, BV), (1, 0))
+        # [BC, BV]
+        b_v = tl.load(p_v, boundary_check=(0, 1))
+        # [BC, BC]
+        b_A = tl.load(p_A, boundary_check=(0, 1))
+        b_o += tl.dot(b_A, exp(b_v - b_zn[None, :]).to(b_v.dtype), allow_tf32=False)
+    b_z = tl.load(p_z, boundary_check=(0, 1))
+    b_o *= exp(b_zn[None, :] - b_z)
+    o_i = tl.arange(0, BC)
+    o_A = i_bh * T * BT + (i_t * BT + i_i * BC + tl.arange(0, BC)) * BT + i_i * BC
+    m_A = (i_t * BT + i_i * BC + tl.arange(0, BC)) < T
+    for j in range(0, BC):
+        p_v = tl.make_block_ptr(v + i_bh * T*V, (T * V,), (1,), ((i_t * BT + i_i * BC + j) * V + i_v * BV,), (BV,), (0,))
+        # [BC,]
+        b_A = tl.load(A + o_A + j, mask=m_A, other=0)
+        # [BV,]
+        b_v = tl.load(p_v, boundary_check=(0,)).to(tl.float32)
+        # [BC, BV]
+        # avoid 0 * inf = inf
+        m_i = o_i[:, None] >= j
+        b_o += tl.where(m_i, b_A[:, None] * exp(b_v[None, :] - b_z), 0)
+    p_o = tl.make_block_ptr(o + i_bh * T*V, (T, V), (V, 1), (i_t * BT + i_i * BC, i_v * BV), (BC, BV), (1, 0))
+    tl.store(p_o, b_o.to(p_o.dtype.element_ty), boundary_check=(0, 1))
+@triton.jit(do_not_specialize=['T'])
+def chunk_abc_fwd_kernel_K(
+    q,
+    k,
+    z,
+    h,
+    o,
+    A,
+    scale,
+    T,
+    K: tl.constexpr,
+    V: tl.constexpr,
+    BT: tl.constexpr,
+    BK: tl.constexpr,
+    BV: tl.constexpr,
+    NT: tl.constexpr,
+):
+    i_v, i_t, i_bh = tl.program_id(0), tl.program_id(1), tl.program_id(2)
+    i_p = tl.maximum(i_t * BT - 1, 0)
+    o_i = tl.arange(0, BT)
+    m_s = o_i[:, None] >= o_i[None, :]
+    b_o = tl.zeros([BT, BV], dtype=tl.float32)
+    b_A = tl.zeros([BT, BT], dtype=tl.float32)
+    for i_k in range(tl.cdiv(K, BK)):
+        p_q = tl.make_block_ptr(q + i_bh * T*K, (T, K), (K, 1), (i_t * BT, i_k * BK), (BT, BK), (1, 0))
+        p_k = tl.make_block_ptr(k + i_bh * T*K, (K, T), (1, K), (i_k * BK, i_t * BT), (BK, BT), (0, 1))
+        p_h = tl.make_block_ptr(h + i_bh * NT*K*V + i_t * K * V, (K, V), (V, 1), (i_k * BK, i_v * BV), (BK, BV), (1, 0))
+        # [BT, BK]
+        b_q = tl.load(p_q, boundary_check=(0, 1))
+        b_q = (b_q * scale).to(b_q.dtype)
+        # [BK, BT]
+        b_k = tl.load(p_k, boundary_check=(0, 1))
+        # [BK, BV]
+        b_h = tl.load(p_h, boundary_check=(0, 1))
+        # [BT, BV]
+        b_o += tl.dot(b_q, b_h, allow_tf32=False)
+        # [BT, BT]
+        b_A += tl.dot(b_q, b_k, allow_tf32=False)
+    p_z = tl.make_block_ptr(z + i_bh * T*V, (T, V), (V, 1), (i_t * BT, i_v * BV), (BT, BV), (1, 0))
+    p_o = tl.make_block_ptr(o + i_bh * T*V, (T, V), (V, 1), (i_t * BT, i_v * BV), (BT, BV), (1, 0))
+    # [BT, BV]
+    b_z = tl.load(p_z, boundary_check=(0, 1))
+    # [BT, BV]
+    p_zp = tl.make_block_ptr(z + i_bh * T*V, (T * V,), (1,), (i_p * V + i_v * BV,), (BV,), (0,))
+    b_zp = tl.load(p_zp, boundary_check=(0,))
+    b_o = b_o * exp(b_zp[None, :] - b_z)
+    tl.store(p_o, b_o.to(p_o.dtype.element_ty), boundary_check=(0, 1))
+    p_A = tl.make_block_ptr(A + i_bh * T * BT, (T, BT), (BT, 1), (i_t * BT, 0), (BT, BT), (1, 0))
+    # [BT, BT]
+    b_A = tl.where(m_s, b_A, 0.)
+    if i_v == 0:
+        tl.store(p_A, b_A.to(p_A.dtype.element_ty), boundary_check=(0, 1))
+@triton.jit(do_not_specialize=['T'])
+def chunk_abc_fwd_kernel_intra_V(
+    q,
+    k,
+    z,
+    A,
+    scale,
+    T,
+    K: tl.constexpr,
+    BT: tl.constexpr,
+    BC: tl.constexpr,
+    BK: tl.constexpr,
+    NC: tl.constexpr,
+):
+    i_k, i_c, i_bh = tl.program_id(0), tl.program_id(1), tl.program_id(2)
+    i_t, i_i, i_j = i_c // (NC * NC), (i_c % (NC * NC)) // NC, (i_c % (NC * NC)) % NC
+    n_bh = tl.num_programs(2)
+    if i_i > i_j:
+        p_q = tl.make_block_ptr(q + i_bh * T*K, (T, K), (K, 1), (i_t * BT + i_i * BC, i_k * BK), (BC, BK), (1, 0))
+        p_k = tl.make_block_ptr(k + i_bh * T*K, (K, T), (1, K), (i_k * BK, i_t * BT + i_j * BC), (BK, BC), (0, 1))
+        p_z = tl.make_block_ptr(z + i_bh * T*K, (T, K), (K, 1), (i_t * BT + i_i * BC, i_k * BK), (BC, BK), (1, 0))
+        p_A = tl.make_block_ptr(A + (i_k*n_bh+i_bh)*T*BT, (T, BT), (BT, 1), (i_t * BT + i_i * BC, i_j * BC), (BC, BC), (1, 0))
+        p_zn = tl.make_block_ptr(z + i_bh * T*K, (T * K,), (1,), ((i_t * BT + i_i * BC) * K + i_k * BK,), (BK,), (0,))
+        # [BK,]
+        b_zn = tl.load(p_zn, boundary_check=(0,))
+        # [BC, BK]
+        b_q = tl.load(p_q, boundary_check=(0, 1))
+        b_z = tl.load(p_z, boundary_check=(0, 1))
+        b_q = (b_q * exp(b_zn[None, :] - b_z) * scale).to(b_q.dtype)
+        # [BK, BC]
+        b_k = tl.load(p_k, boundary_check=(0, 1))
+        b_k = exp(b_k - b_zn[:, None]).to(b_k.dtype)
+        # [BC, BC]
+        b_A = tl.dot(b_q, b_k, allow_tf32=False)
+        tl.store(p_A, b_A.to(A.dtype.element_ty), boundary_check=(0, 1))
+    elif i_i == i_j:
+        p_q = tl.make_block_ptr(q + i_bh * T*K, (T, K), (K, 1), (i_t * BT + i_i * BC, i_k * BK), (BC, BK), (1, 0))
+        p_k = tl.make_block_ptr(k + i_bh * T*K, (T * K,), (1,), ((i_t * BT + i_j * BC) * K + i_k * BK,), (BK,), (0,))
+        p_z = tl.make_block_ptr(z + i_bh * T*K, (T, K), (K, 1), (i_t * BT + i_i * BC, i_k * BK), (BC, BK), (1, 0))
+        # [BC, BK]
+        b_q = tl.load(p_q, boundary_check=(0, 1))
+        b_z = tl.load(p_z, boundary_check=(0, 1))
+        o_i = tl.arange(0, BC)
+        o_A = (i_bh + i_k * n_bh) * T * BT + (i_t * BT + i_i * BC + tl.arange(0, BC)) * BT + i_j * BC
+        m_A = (i_t * BT + i_i * BC + tl.arange(0, BC)) < T
+        for j in range(0, BC):
+            # [BK,]
+            b_k = tl.load(p_k, boundary_check=(0,)).to(tl.float32)
+            # [BC,]
+            b_A = tl.sum(b_q * exp(b_k[None, :] - b_z) * scale, 1)
+            b_A = tl.where(o_i >= j, b_A, 0.)
+            tl.store(A + o_A + j, b_A.to(b_q.dtype), mask=m_A)
+            p_k = tl.advance(p_k, (K,))
+@triton.jit(do_not_specialize=['T'])
+def chunk_abc_fwd_kernel_V(
+    q,
+    v,
+    z,
+    h,
+    o,
+    A,
+    scale,
+    T,
+    K: tl.constexpr,
+    V: tl.constexpr,
+    BT: tl.constexpr,
+    BK: tl.constexpr,
+    BV: tl.constexpr,
+    NT: tl.constexpr,
+):
+    i_v, i_t, i_bh = tl.program_id(0), tl.program_id(1), tl.program_id(2)
+    i_p = tl.maximum(i_t * BT - 1, 0)
+    b_o = tl.zeros([BT, BV], dtype=tl.float32)
+    for i_k in range(tl.cdiv(K, BK)):
+        p_q = tl.make_block_ptr(q + i_bh * T*K, (T, K), (K, 1), (i_t * BT, i_k * BK), (BT, BK), (1, 0))
+        p_z = tl.make_block_ptr(z + i_bh * T*K, (T, K), (K, 1), (i_t * BT, i_k * BK), (BT, BK), (1, 0))
+        p_h = tl.make_block_ptr(h + i_bh * NT*K*V + i_t * K * V, (K, V), (V, 1), (i_k * BK, i_v * BV), (BK, BV), (1, 0))
+        p_zp = tl.make_block_ptr(z + i_bh * T*K, (T * K,), (1,), (i_p * K + i_k * BK,), (BK,), (0,))
+        # [BT, BK]
+        b_q = tl.load(p_q, boundary_check=(0, 1))
+        b_q = (b_q * scale).to(b_q.dtype)
+        # [BT, BK]
+        b_z = tl.load(p_z, boundary_check=(0, 1))
+        # [BT, BK]
+        b_zp = tl.load(p_zp, boundary_check=(0,))
+        b_q = (b_q * exp(b_zp[None, :] - b_z)).to(b_q.dtype)
+        # [BK, BV]
+        b_h = tl.load(p_h, boundary_check=(0, 1))
+        # works but dkw, owing to divine benevolence
+        # [BT, BV]
+        if i_k >= 0:
+            b_o += tl.dot(b_q, b_h, allow_tf32=False)
+    p_v = tl.make_block_ptr(v + i_bh * T*V, (T, V), (V, 1), (i_t * BT, i_v * BV), (BT, BV), (1, 0))
+    p_o = tl.make_block_ptr(o + i_bh * T*V, (T, V), (V, 1), (i_t * BT, i_v * BV), (BT, BV), (1, 0))
+    p_A = tl.make_block_ptr(A + i_bh * T * BT, (T, BT), (BT, 1), (i_t * BT, 0), (BT, BT), (1, 0))
+    # [BT, BV]
+    b_v = tl.load(p_v, boundary_check=(0, 1))
+    # [BT, BT]
+    b_A = tl.load(p_A, boundary_check=(0, 1))
+    b_o += tl.dot(b_A.to(b_v.dtype), b_v, allow_tf32=False)
+    tl.store(p_o, b_o.to(p_o.dtype.element_ty), boundary_check=(0, 1))
+@triton.jit(do_not_specialize=['T'])
+def chunk_abc_bwd_kernel_dh(
+    q,
+    z,
+    do,
+    dh,
+    scale,
+    T,
+    K: tl.constexpr,
+    V: tl.constexpr,
+    BT: tl.constexpr,
+    BK: tl.constexpr,
+    BV: tl.constexpr,
+    NT: tl.constexpr,
+    NORMK: tl.constexpr,
+):
+    i_k, i_v, i_bh = tl.program_id(0), tl.program_id(1), tl.program_id(2)
+    b_dh = tl.zeros([BK, BV], dtype=tl.float32)
+    b_zp = tl.full([BK if NORMK else BV], float('inf'), dtype=tl.float32)
+    for i_t in range(NT - 1, -1, -1):
+        i_p = tl.maximum(i_t * BT - 1, 0)
+        p_q = tl.make_block_ptr(q + i_bh * T*K, (K, T), (1, K), (i_k * BK, i_t * BT), (BK, BT), (0, 1))
+        p_do = tl.make_block_ptr(do + i_bh * T*V, (T, V), (V, 1), (i_t * BT, i_v * BV), (BT, BV), (1, 0))
+        p_dh = tl.make_block_ptr(dh + i_bh * NT*K*V + i_t * K*V, (K, V), (V, 1), (i_k * BK, i_v * BV), (BK, BV), (1, 0))
+        # [BK, BT]
+        b_q = tl.load(p_q, boundary_check=(0, 1))
+        b_q = (b_q * scale).to(b_q.dtype)
+        # [BT, BV]
+        b_do = tl.load(p_do, boundary_check=(0, 1))
+        tl.store(p_dh, b_dh.to(p_dh.dtype.element_ty), boundary_check=(0, 1))
+        if NORMK:
+            p_z = tl.make_block_ptr(z + i_bh * T*K, (K, T), (1, K), (i_k * BK, i_t * BT), (BK, BT), (0, 1))
+            p_zc = tl.make_block_ptr(z + i_bh * T*K, (T * K,), (1,), (i_p * K + i_k * BK,), (BK,), (0,))
+            # [BK,]
+            b_zc = tl.load(p_zc, boundary_check=(0,))
+            b_r, b_zp = exp(b_zc - b_zp), b_zc
+            # [BK, BT]
+            b_z = tl.load(p_z, boundary_check=(0, 1))
+            b_q = (b_q * exp(b_zc[:, None] - b_z)).to(b_q.dtype)
+            # [BK, BV]
+            b_dh = b_dh * b_r[:, None]
+        else:
+            p_z = tl.make_block_ptr(z + i_bh * T*V, (T, V), (V, 1), (i_t * BT, i_v * BV), (BT, BV), (1, 0))
+            p_zc = tl.make_block_ptr(z + i_bh * T*V, (T * V,), (1,), (i_p * V + i_v * BV,), (BV,), (0,))
+            # [BV,]
+            b_zc = tl.load(p_zc, boundary_check=(0,))
+            b_r, b_zp = exp(b_zc - b_zp), b_zc
+            # [BT, BV]
+            b_z = tl.load(p_z, boundary_check=(0,))
+            b_do = (b_do * exp(b_zc[None, :] - b_z)).to(b_do.dtype)
+            # [BK, BV]
+            b_dh = b_dh * b_r[None, :]
+        # [BK, BV]
+        b_dh += tl.dot(b_q, b_do, allow_tf32=False)
+@triton.jit(do_not_specialize=['T'])
+def chunk_abc_bwd_kernel_V(
+    k,
+    v,
+    z,
+    h,
+    A,
+    do,
+    dh,
+    dq,
+    dk,
+    dv,
+    dA,
+    scale,
+    T,
+    K: tl.constexpr,
+    V: tl.constexpr,
+    BT: tl.constexpr,
+    BK: tl.constexpr,
+    BV: tl.constexpr,
+    NT: tl.constexpr,
+):
+    i_k, i_t, i_bh = tl.program_id(0), tl.program_id(1), tl.program_id(2)
+    i_p = tl.maximum(i_t * BT - 1, 0)
+    n_bh = tl.num_programs(2)
+    p_k = tl.make_block_ptr(k + i_bh * T*K, (T, K), (K, 1), (i_t * BT, i_k * BK), (BT, BK), (1, 0))
+    p_zc = tl.make_block_ptr(z + i_bh * T*K, (T * K,), (1,), ((i_t * BT + BT - 1) * K + i_k * BK,), (BK,), (0,))
+    p_A = tl.make_block_ptr(A + i_bh * T * BT, (BT, T), (1, BT), (0, i_t * BT), (BT, BT), (0, 1))
+    # [BK,]
+    b_zc = tl.load(p_zc, boundary_check=(0,))
+    # [BT, BK]
+    b_k = tl.load(p_k, boundary_check=(0, 1))
+    b_k = exp(b_k - b_zc[None, :]).to(b_k.dtype)
+    # [BT, BT]
+    b_A = tl.load(p_A, boundary_check=(0, 1))
+    b_dq = tl.zeros([BT, BK], dtype=tl.float32)
+    b_dk = tl.zeros([BT, BK], dtype=tl.float32)
+    b_dA = tl.zeros([BT, BT], dtype=tl.float32)
+    for i_v in range(tl.cdiv(V, BV)):
+        p_v = tl.make_block_ptr(v + i_bh * T*V, (T, V), (V, 1), (i_t * BT, i_v * BV), (BT, BV), (1, 0))
+        p_h = tl.make_block_ptr(h + i_bh * NT*K*V + i_t * V * K, (V, K), (1, V), (i_v * BV, i_k * BK), (BV, BK), (0, 1))
+        p_do = tl.make_block_ptr(do + i_bh * T*V, (T, V), (V, 1), (i_t * BT, i_v * BV), (BT, BV), (1, 0))
+        p_dh = tl.make_block_ptr(dh + i_bh * NT*K*V + i_t * K*V, (K, V), (V, 1), (i_k * BK, i_v * BV), (BK, BV), (1, 0))
+        p_dv = tl.make_block_ptr(dv + (i_k*n_bh+i_bh) * T*V, (T, V), (V, 1), (i_t * BT, i_v * BV), (BT, BV), (1, 0))
+        # [BT, BV]
+        b_v = tl.load(p_v, boundary_check=(0, 1))
+        # [BV, BK]
+        b_h = tl.load(p_h, boundary_check=(0, 1))
+        # [BT, BV]
+        b_do = tl.load(p_do, boundary_check=(0, 1))
+        # [BK, BV]
+        b_dh = tl.load(p_dh, boundary_check=(0, 1))
+        # [BT, BV]
+        b_dv = tl.dot(b_k, b_dh, allow_tf32=False)
+        if i_k == 0:
+            b_dv += tl.dot(b_A.to(b_do.dtype), b_do, allow_tf32=False)
+        b_do = (b_do * scale).to(b_do.dtype)
+        tl.store(p_dv, b_dv.to(p_dv.dtype.element_ty), boundary_check=(0, 1))
+        # [BT, BT]
+        b_dA += tl.dot(b_do, tl.trans(b_v), allow_tf32=False)
+        # [BT, BK]
+        b_dq += tl.dot(b_do, b_h, allow_tf32=False)
+        # [BT, BK]
+        b_dk += tl.dot(b_v, tl.trans(b_dh), allow_tf32=False)
+    p_z = tl.make_block_ptr(z + i_bh * T*K, (T, K), (K, 1), (i_t * BT, i_k * BK), (BT, BK), (1, 0))
+    p_zp = tl.make_block_ptr(z + i_bh * T*K, (T * K,), (1,), (i_p * K + i_k * BK,), (BK,), (0,))
+    # [BK,]
+    b_zp = tl.load(p_zp, boundary_check=(0,))
+    # [BT, BK]
+    b_z = tl.load(p_z, boundary_check=(0, 1))
+    b_z = exp(b_zp[None, :] - b_z)
+    # [BT, BK]
+    b_dq = b_dq * b_z
+    b_dk = b_dk * b_k
+    p_dq = tl.make_block_ptr(dq + i_bh * T*K, (T, K), (K, 1), (i_t * BT, i_k * BK), (BT, BK), (1, 0))
+    p_dk = tl.make_block_ptr(dk + i_bh * T*K, (T, K), (K, 1), (i_t * BT, i_k * BK), (BT, BK), (1, 0))
+    p_dA = tl.make_block_ptr(dA + i_bh * T * BT, (T, BT), (BT, 1), (i_t * BT, 0), (BT, BT), (1, 0))
+    tl.store(p_dq, b_dq.to(p_dq.dtype.element_ty), boundary_check=(0, 1))
+    tl.store(p_dk, b_dk.to(p_dk.dtype.element_ty), boundary_check=(0, 1))
+    o_i = tl.arange(0, BT)
+    m_s = o_i[:, None] >= o_i[None, :]
+    # [BT, BT]
+    b_dA = tl.where(m_s, b_dA, 0.).to(b_k.dtype)
+    if i_k == 0:
+        tl.store(p_dA, b_dA.to(p_dA.dtype.element_ty), boundary_check=(0, 1))
+@triton.jit(do_not_specialize=['T'])
+def chunk_abc_bwd_kernel_intra_V(
+    q,
+    k,
+    z,
+    dA,
+    dq,
+    dk,
+    T,
+    K: tl.constexpr,
+    BT: tl.constexpr,
+    BC: tl.constexpr,
+    BK: tl.constexpr,
+    NC: tl.constexpr,
+):
+    i_k, i_c, i_bh = tl.program_id(0), tl.program_id(1), tl.program_id(2)
+    i_t, i_i = i_c // NC, i_c % NC
+    p_z = tl.make_block_ptr(z + i_bh * T*K, (T, K), (K, 1), (i_t * BT + i_i * BC, i_k * BK), (BC, BK), (1, 0))
+    p_zn = tl.make_block_ptr(z + i_bh * T*K, (T * K,), (1,), ((i_t * BT + i_i * BC) * K + i_k * BK,), (BK,), (0,))
+    # [BK,]
+    b_zn = tl.load(p_zn, boundary_check=(0,))
+    # [BC, BK]
+    b_z = tl.load(p_z, boundary_check=(0, 1))
+    b_zq = exp(b_zn[None, :] - b_z)
+    b_dq = tl.zeros([BC, BK], dtype=tl.float32)
+    for i_j in range(0, i_i):
+        p_k = tl.make_block_ptr(k + i_bh * T*K, (T, K), (K, 1), (i_t * BT + i_j * BC, i_k * BK), (BC, BK), (1, 0))
+        p_dA = tl.make_block_ptr(dA + i_bh * T * BT, (T, BT), (BT, 1), (i_t * BT + i_i * BC, i_j * BC), (BC, BC), (1, 0))
+        # [BC, BK]
+        b_k = tl.load(p_k, boundary_check=(0, 1))
+        b_kz = exp(b_k - b_zn[None, :]).to(b_k.dtype)
+        # [BC, BC]
+        b_dA = tl.load(p_dA, boundary_check=(0, 1))
+        # [BC, BK]
+        b_dq += tl.dot(b_dA, b_kz, allow_tf32=False)
+    b_dq *= b_zq
+    o_i = tl.arange(0, BC)
+    o_dA = i_bh * T * BT + (i_t * BT + i_i * BC + tl.arange(0, BC)) * BT + i_i * BC
+    m_dA = (i_t * BT + i_i * BC + tl.arange(0, BC)) < T
+    for j in range(0, BC):
+        p_kj = tl.make_block_ptr(k + i_bh * T*K, (T * K,), (1,), ((i_t * BT + i_i*BC+j) * K + i_k * BK,), (BK,), (0,))
+        # [BC,]
+        b_dA = tl.load(dA + o_dA + j, mask=m_dA, other=0)
+        # [BK,]
+        b_kj = tl.load(p_kj, boundary_check=(0,)).to(tl.float32)
+        # [BC, BK]
+        m_i = o_i[:, None] >= j
+        # [BC, BK]
+        b_dq += tl.where(m_i, b_dA[:, None] * exp(b_kj[None, :] - b_z), 0.)
+    p_dq = tl.make_block_ptr(dq + i_bh * T*K, (T, K), (K, 1), (i_t * BT + i_i * BC, i_k * BK), (BC, BK), (1, 0))
+    tl.store(p_dq, b_dq.to(p_dq.dtype.element_ty), boundary_check=(0, 1))
+    tl.debug_barrier()
+    p_k = tl.make_block_ptr(k + i_bh * T*K, (T, K), (K, 1), (i_t * BT + i_i * BC, i_k * BK), (BC, BK), (1, 0))
+    p_zn = tl.make_block_ptr(z + i_bh * T*K, (T*K,), (1,), ((i_t * BT + i_i * BC + BC - 1) * K + i_k * BK,), (BK,), (0,))
+    # [BK,]
+    b_zn = tl.load(p_zn, boundary_check=(0,))
+    # [BC, BK]
+    b_k = tl.load(p_k, boundary_check=(0, 1))
+    b_kz = exp(b_k - b_zn[None, :])
+    b_dk = tl.zeros([BC, BK], dtype=tl.float32)
+    for i_j in range(i_i + 1, NC):
+        p_q = tl.make_block_ptr(q + i_bh * T*K, (T, K), (K, 1), (i_t * BT + i_j * BC, i_k * BK), (BC, BK), (1, 0))
+        p_z = tl.make_block_ptr(z + i_bh * T*K, (T, K), (K, 1), (i_t * BT + i_j * BC, i_k * BK), (BC, BK), (1, 0))
+        p_dA = tl.make_block_ptr(dA + i_bh * T * BT, (T, BT), (BT, 1), (i_t * BT + i_j * BC, i_i * BC), (BC, BC), (1, 0))
+        # [BC, BK]
+        b_q = tl.load(p_q, boundary_check=(0, 1))
+        b_z = tl.load(p_z, boundary_check=(0, 1))
+        b_qz = (b_q * exp(b_zn[None, :] - b_z)).to(b_q.dtype)
+        # [BC, BC]
+        b_dA = tl.load(p_dA, boundary_check=(0, 1))
+        # [BC, BK]
+        b_dk += tl.dot(tl.trans(b_dA), b_qz, allow_tf32=False)
+    b_dk *= b_kz
+    o_dA = i_bh * T * BT + (i_t * BT + i_i * BC) * BT + i_i * BC + tl.arange(0, BC)
+    for j in range(0, BC):
+        p_qj = tl.make_block_ptr(q + i_bh * T*K, (T * K,), (1,), ((i_t * BT + i_i * BC + j) * K + i_k * BK,), (BK,), (0,))
+        p_zj = tl.make_block_ptr(z + i_bh * T*K, (T * K,), (1,), ((i_t * BT + i_i * BC + j) * K + i_k * BK,), (BK,), (0,))
+        # [BC,]
+        b_dA = tl.load(dA + o_dA + j * BT, mask=(i_t * BT + i_i * BC + j < T), other=0)
+        # [BK,]
+        b_qj = tl.load(p_qj, boundary_check=(0,)).to(tl.float32)
+        b_zj = tl.load(p_zj, boundary_check=(0,)).to(tl.float32)
+        # [BC, BK]
+        m_i = o_i[:, None] <= j
+        b_dk += tl.where(m_i, b_dA[:, None] * b_qj[None, :] * exp(b_k - b_zj[None, :]), 0.)
+    p_dk = tl.make_block_ptr(dk + i_bh * T*K, (T, K), (K, 1), (i_t * BT + i_i * BC, i_k * BK), (BC, BK), (1, 0))
+    tl.store(p_dk, b_dk.to(p_dk.dtype.element_ty), boundary_check=(0, 1))
+@triton.jit(do_not_specialize=['T'])
+def chunk_abc_bwd_kernel_intra_K(
+    v,
+    z,
+    do,
+    dA,
+    scale,
+    T,
+    V: tl.constexpr,
+    BT: tl.constexpr,
+    BC: tl.constexpr,
+    BV: tl.constexpr,
+    NC: tl.constexpr,
+):
+    i_v, i_c, i_bh = tl.program_id(0), tl.program_id(1), tl.program_id(2)
+    i_t, i_i, i_j = i_c // (NC * NC), (i_c % (NC * NC)) // NC, (i_c % (NC * NC)) % NC
+    n_bh = tl.num_programs(2)
+    if i_i > i_j:
+        p_v = tl.make_block_ptr(v + i_bh * T*V, (V, T), (1, V), (i_v * BV, i_t * BT + i_j * BC), (BV, BC), (0, 1))
+        p_z = tl.make_block_ptr(z + i_bh * T*V, (T, V), (V, 1), (i_t * BT + i_i * BC, i_v * BV), (BC, BV), (1, 0))
+        p_zn = tl.make_block_ptr(z + i_bh * T*V, (T * V,), (1,), ((i_t * BT + i_i * BC) * V + i_v * BV,), (BV,), (0,))
+        p_do = tl.make_block_ptr(do + i_bh * T*V, (T, V), (V, 1), (i_t * BT + i_i * BC, i_v * BV), (BC, BV), (1, 0))
+        p_dA = tl.make_block_ptr(dA+(i_bh+i_v*n_bh)*T*BT, (T, BT), (BT, 1), (i_t * BT + i_i * BC, i_j * BC), (BC, BC), (1, 0))
+        # [BV,]
+        b_zn = tl.load(p_zn, boundary_check=(0,))
+        # [BC, BV]
+        b_z = tl.load(p_z, boundary_check=(0, 1))
+        b_do = tl.load(p_do, boundary_check=(0, 1))
+        b_do = (b_do * exp(b_zn[None, :] - b_z) * scale).to(b_do.dtype)
+        # [BV, BC]
+        b_v = tl.load(p_v, boundary_check=(0, 1))
+        b_v = exp(b_v - b_zn[:, None]).to(b_v.dtype)
+        # [BC, BC]
+        b_dA = tl.dot(b_do, b_v, allow_tf32=False)
+        tl.store(p_dA, b_dA.to(dA.dtype.element_ty), boundary_check=(0, 1))
+    elif i_i == i_j:
+        p_v = tl.make_block_ptr(v + i_bh * T*V, (T * V,), (1,), ((i_t * BT + i_j * BC) * V + i_v * BV,), (BV,), (0,))
+        p_z = tl.make_block_ptr(z + i_bh * T*V, (T, V), (V, 1), (i_t * BT + i_i * BC, i_v * BV), (BC, BV), (1, 0))
+        p_do = tl.make_block_ptr(do + i_bh * T*V, (T, V), (V, 1), (i_t * BT + i_i * BC, i_v * BV), (BC, BV), (1, 0))
+        # [BC, BV]
+        b_z = tl.load(p_z, boundary_check=(0, 1))
+        b_do = tl.load(p_do, boundary_check=(0, 1)) * scale
+        o_i = tl.arange(0, BC)
+        o_A = (i_bh + i_v * n_bh) * T * BT + (i_t * BT + i_i * BC + tl.arange(0, BC)) * BT + i_j * BC
+        m_A = (i_t * BT + i_i * BC + tl.arange(0, BC)) < T
+        for j in range(0, BC):
+            # [BV,]
+            b_v = tl.load(p_v, boundary_check=(0,)).to(tl.float32)
+            # [BC,]
+            b_dA = tl.sum(b_do * exp(b_v[None, :] - b_z), 1)
+            b_dA = tl.where(o_i >= j, b_dA, 0)
+            tl.store(dA + o_A + j, b_dA.to(b_do.dtype), mask=m_A)
+            p_v = tl.advance(p_v, (V,))
+@triton.jit(do_not_specialize=['T'])
+def chunk_abc_bwd_kernel_K(
+    q,
+    k,
+    v,
+    z,
+    h,
+    A,
+    do,
+    dh,
+    dq,
+    dk,
+    dv,
+    dA,
+    scale,
+    T,
+    K: tl.constexpr,
+    V: tl.constexpr,
+    BT: tl.constexpr,
+    BK: tl.constexpr,
+    BV: tl.constexpr,
+    NT: tl.constexpr,
+):
+    i_k, i_t, i_bh = tl.program_id(0), tl.program_id(1), tl.program_id(2)
+    i_p = tl.maximum(i_t * BT - 1, 0)
+    n_bh = tl.num_programs(2)
+    o_i = tl.arange(0, BT)
+    m_s = o_i[:, None] >= o_i[None, :]
+    p_q = tl.make_block_ptr(q + i_bh * T*K, (T, K), (K, 1), (i_t * BT, i_k * BK), (BT, BK), (1, 0))
+    p_k = tl.make_block_ptr(k + i_bh * T*K, (T, K), (K, 1), (i_t * BT, i_k * BK), (BT, BK), (1, 0))
+    p_A = tl.make_block_ptr(A + (i_k*n_bh+i_bh) * T * BT, (T, BT), (BT, 1), (i_t * BT, 0), (BT, BT), (1, 0))
+    # [BT, BK]
+    b_q = tl.load(p_q, boundary_check=(0, 1))
+    b_k = tl.load(p_k, boundary_check=(0, 1))
+    # [BT, BT]
+    b_A = tl.dot((b_q * scale).to(b_q.dtype), tl.trans(b_k), allow_tf32=False)
+    b_A = tl.where(m_s, b_A, 0.)
+    tl.store(p_A, b_A.to(p_A.dtype.element_ty), boundary_check=(0, 1))
+    b_dq = tl.zeros([BT, BK], dtype=tl.float32)
+    b_dk = tl.zeros([BT, BK], dtype=tl.float32)
+    for i_v in range(tl.cdiv(V, BV)):
+        p_v = tl.make_block_ptr(v + i_bh * T*V, (T, V), (V, 1), (i_t * BT, i_v * BV), (BT, BV), (1, 0))
+        p_z = tl.make_block_ptr(z + i_bh * T*V, (T, V), (V, 1), (i_t * BT, i_v * BV), (BT, BV), (1, 0))
+        p_zp = tl.make_block_ptr(z + i_bh * T*V, (T * V,), (1,), (i_p * V + i_v * BV,), (BV,), (0,))
+        p_zc = tl.make_block_ptr(z + i_bh * T*V, (T * V,), (1,), ((i_t * BT + BT - 1) * V + i_v * BV,), (BV,), (0,))
+        p_h = tl.make_block_ptr(h + i_bh * NT*K*V + i_t * K*V, (V, K), (1, V), (i_v * BV, i_k * BK), (BV, BK), (0, 1))
+        p_do = tl.make_block_ptr(do + i_bh * T*V, (T, V), (V, 1), (i_t * BT, i_v * BV), (BT, BV), (1, 0))
+        p_dh = tl.make_block_ptr(dh + i_bh * NT*K*V + i_t * K*V, (K, V), (V, 1), (i_k * BK, i_v * BV), (BK, BV), (1, 0))
+        p_dv = tl.make_block_ptr(dv + (i_k*n_bh+i_bh) * T*V, (T, V), (V, 1), (i_t * BT, i_v * BV), (BT, BV), (1, 0))
+        # [BV,]
+        b_zp = tl.load(p_zp, boundary_check=(0,))
+        b_zc = tl.load(p_zc, boundary_check=(0,))
+        # [BT, BV]
+        b_v = tl.load(p_v, boundary_check=(0, 1))
+        b_v = exp(b_v - b_zc[None, :]).to(b_v.dtype)
+        b_z = tl.load(p_z, boundary_check=(0, 1))
+        b_z = exp(b_zp[None, :] - b_z)
+        # [BV, BK]
+        b_h = tl.load(p_h, boundary_check=(0, 1))
+        # [BT, BV]
+        b_do = tl.load(p_do, boundary_check=(0, 1))
+        b_do = (b_do * b_z * scale).to(b_do.dtype)
+        # [BK, BV]
+        b_dh = tl.load(p_dh, boundary_check=(0, 1))
+        # [BT, BK]
+        b_dq += tl.dot(b_do, b_h, allow_tf32=False)
+        b_dk += tl.dot(b_v, tl.trans(b_dh), allow_tf32=False)
+        # [BT, BV]
+        b_dv = b_v * tl.dot(b_k, b_dh, allow_tf32=False)
+        tl.store(p_dv, b_dv.to(p_dv.dtype.element_ty), boundary_check=(0, 1))
+    p_dA = tl.make_block_ptr(dA + i_bh * T * BT, (T, BT), (BT, 1), (i_t * BT, 0), (BT, BT), (1, 0))
+    # [BT, BT]
+    b_dA = tl.load(p_dA, boundary_check=(0, 1))
+    # [BT, BK]
+    b_dq += tl.dot(b_dA, b_k, allow_tf32=False)
+    b_dk += tl.dot(tl.trans(b_dA).to(b_k.dtype), b_q, allow_tf32=False)
+    p_dq = tl.make_block_ptr(dq + i_bh * T*K, (T, K), (K, 1), (i_t * BT, i_k * BK), (BT, BK), (1, 0))
+    p_dk = tl.make_block_ptr(dk + i_bh * T*K, (T, K), (K, 1), (i_t * BT, i_k * BK), (BT, BK), (1, 0))
+    tl.store(p_dq, b_dq.to(p_dq.dtype.element_ty), boundary_check=(0, 1))
+    tl.store(p_dk, b_dk.to(p_dk.dtype.element_ty), boundary_check=(0, 1))
+@triton.jit(do_not_specialize=['T'])
+def chunk_abc_bwd_kernel_intra_KV(
+    v,
+    z,
+    A,
+    do,
+    dv,
+    T,
+    V: tl.constexpr,
+    BT: tl.constexpr,
+    BC: tl.constexpr,
+    BV: tl.constexpr,
+    NC: tl.constexpr,
+):
+    i_v, i_c, i_bh = tl.program_id(0), tl.program_id(1), tl.program_id(2)
+    i_t, i_i = i_c // NC, i_c % NC
+    p_v = tl.make_block_ptr(v + i_bh * T*V, (T, V), (V, 1), (i_t * BT + i_i * BC, i_v * BV), (BC, BV), (1, 0))
+    p_zn = tl.make_block_ptr(z + i_bh * T*V, (T*V,), (1,), ((i_t * BT + i_i * BC + BC - 1) * V + i_v * BV,), (BV,), (0,))
+    # [BV,]
+    b_zn = tl.load(p_zn, boundary_check=(0,))
+    # [BC, BV]
+    b_v = tl.load(p_v, boundary_check=(0, 1))
+    b_dv = tl.zeros([BC, BV], dtype=tl.float32)
+    for i_j in range(i_i + 1, NC):
+        p_z = tl.make_block_ptr(z + i_bh * T*V, (T, V), (V, 1), (i_t * BT + i_j * BC, i_v * BV),  (BC, BV), (1, 0))
+        p_A = tl.make_block_ptr(A + i_bh * T * BT, (BT, T), (1, BT), (i_i * BC, i_t * BT + i_j * BC), (BC, BC), (0, 1))
+        p_do = tl.make_block_ptr(do + i_bh * T*V, (T, V), (V, 1), (i_t * BT + i_j * BC, i_v * BV), (BC, BV), (1, 0))
+        # [BC, BV]
+        b_z = tl.load(p_z, boundary_check=(0, 1))
+        b_do = tl.load(p_do, boundary_check=(0, 1))
+        b_do = (b_do * exp(b_zn[None, :] - b_z)).to(b_do.dtype)
+        # [BC, BC]
+        b_A = tl.load(p_A, boundary_check=(0, 1))
+        b_dv += tl.dot(b_A, b_do, allow_tf32=False)
+    b_dv *= exp(b_v - b_zn[None, :])
+    o_i = tl.arange(0, BC)
+    for j in range(0, BC):
+        p_z = tl.make_block_ptr(z + i_bh * T*V, (T * V,), (1,), ((i_t * BT + i_i * BC + j) * V + i_v * BV,), (BV,), (0,))
+        p_A = tl.make_block_ptr(A + i_bh * T * BT, (T * BT,), (1,), ((i_t * BT + i_i * BC + j) * BT + i_i * BC,), (BC,), (0,))
+        p_do = tl.make_block_ptr(do + i_bh * T*V, (T * V,), (1,), ((i_t * BT + i_i * BC + j) * V + i_v * BV,), (BV,), (0,))
+        # [BC,]
+        b_A = tl.load(p_A, boundary_check=(0,))
+        # [BV,]
+        b_z = tl.load(p_z, boundary_check=(0,))
+        b_do = tl.load(p_do, boundary_check=(0,))
+        # [BC, BV]
+        m_i = o_i[:, None] <= j
+        b_dv += tl.where(m_i, exp(b_v - b_z[None, :]) * b_A[:, None] * b_do[None, :], 0.)
+    p_dv = tl.make_block_ptr(dv + i_bh * T*V, (T, V), (V, 1), (i_t * BT + i_i * BC, i_v * BV), (BC, BV), (1, 0))
+    tl.store(p_dv, b_dv.to(p_dv.dtype.element_ty), boundary_check=(0, 1))
+@triton.jit(do_not_specialize=['T'])
+def chunk_abc_bwd_kernel_rcum_inter(
+    s,
+    z,
+    ss,
+    doo,
+    T,
+    S: tl.constexpr,
+    BT: tl.constexpr,
+    BS: tl.constexpr,
+    NT: tl.constexpr,
+):
+    i_m, i_bh = tl.program_id(0), tl.program_id(1)
+    b_sp = tl.zeros([BS], dtype=tl.float32)
+    b_zp = tl.full([BS], float('inf'), dtype=tl.float32)
+    for i_t in range(NT - 1, -1, -1):
+        p_s = tl.make_block_ptr(s + i_bh * T*S, (T, S), (S, 1), (i_t * BT, i_m * BS), (BT, BS), (1, 0))
+        p_z = tl.make_block_ptr(z + i_bh * T*S, (T, S), (S, 1), (i_t * BT, i_m * BS), (BT, BS), (1, 0))
+        p_zc = tl.make_block_ptr(z + i_bh * T*S, (T*S,), (1,), ((i_t * BT) * S + i_m * BS,), (BS,), (0,))
+        p_ss = tl.make_block_ptr(ss + i_bh * T*S, (T, S), (S, 1), (i_t * BT, i_m * BS), (BT, BS), (1, 0))
+        p_doo = tl.make_block_ptr(doo + i_bh * T*S, (T, S), (S, 1), (i_t * BT, i_m * BS), (BT, BS), (1, 0))
+        # [BS,]
+        b_zc = tl.load(p_zc, boundary_check=(0,))
+        # [BT, BS]
+        b_s = tl.load(p_s, boundary_check=(0, 1))
+        b_z = tl.load(p_z, boundary_check=(0, 1))
+        b_ss = tl.load(p_ss, boundary_check=(0, 1))
+        b_doo = exp(b_s - b_zp[None, :]) * b_sp[None, :]
+        tl.store(p_doo, b_doo.to(p_doo.dtype.element_ty), boundary_check=(0, 1))
+        # [BS,]
+        b_sp = b_sp * exp(b_zc - b_zp) + tl.sum(b_ss * exp(b_zc[None, :] - b_z), 0)
+        b_zp = b_zc
+@triton.jit(do_not_specialize=['T'])
+def chunk_abc_bwd_kernel_rcum_intra(
+    s,
+    z,
+    ss,
+    doo,
+    T,
+    S: tl.constexpr,
+    BT: tl.constexpr,
+    BC: tl.constexpr,
+    BS: tl.constexpr,
+    NC: tl.constexpr,
+):
+    i_s, i_c, i_bh = tl.program_id(0), tl.program_id(1), tl.program_id(2)
+    i_t, i_i = i_c // NC, i_c % NC
+    o_i = tl.arange(0, BC)
+    m_o = tl.full([BC, BC], 1., dtype=tl.float32)
+    p_s = tl.make_block_ptr(s + i_bh * T*S, (T, S), (S, 1), (i_t * BT + i_i * BC, i_s * BS), (BC, BS), (1, 0))
+    p_zn = tl.make_block_ptr(z + i_bh * T*S, (T*S,), (1,), ((i_t * BT + i_i * BC + BC - 1) * S + i_s * BS,), (BS,), (0,))
+    p_doo = tl.make_block_ptr(doo + i_bh * T*S, (T, S), (S, 1), (i_t * BT + i_i * BC, i_s * BS), (BC, BS), (1, 0))
+    # [BC, BS]
+    b_s = tl.load(p_s, boundary_check=(0, 1))
+    # [BS,]
+    b_zn = tl.load(p_zn, boundary_check=(0,))
+    b_doo = tl.zeros([BC, BS], dtype=tl.float32)
+    for i_j in range(i_i + 1, NC):
+        p_z = tl.make_block_ptr(z + i_bh * T*S, (T, S), (S, 1), (i_t * BT + i_j * BC, i_s * BS), (BC, BS), (1, 0))
+        p_ss = tl.make_block_ptr(ss + i_bh * T*S, (T, S), (S, 1), (i_t * BT + i_j * BC, i_s * BS), (BC, BS), (1, 0))
+        # [BC, BS]
+        b_z = tl.load(p_z, boundary_check=(0, 1))
+        b_ss = tl.load(p_ss, boundary_check=(0, 1))
+        # [BC, BS]
+        b_doo += b_ss * exp(b_zn[None, :] - b_z)
+    b_doo = exp(b_s - b_zn[None, :]) * tl.dot(m_o.to(b_s.dtype), b_doo.to(b_s.dtype), allow_tf32=False)
+    for j in range(0, BC):
+        p_z = tl.make_block_ptr(z + i_bh * T*S, (T*S,), (1,), ((i_t * BT + i_i * BC + j) * S + i_s * BS,), (BS,), (0,))
+        p_ss = tl.make_block_ptr(ss + i_bh * T*S, (T*S,), (1,), ((i_t * BT + i_i * BC + j) * S + i_s * BS,), (BS,), (0,))
+        # [BS,]
+        b_z = tl.load(p_z, boundary_check=(0,))
+        b_ss = tl.load(p_ss, boundary_check=(0,))
+        # [BC, BS]
+        m_i = o_i[:, None] <= j
+        b_doo += tl.where(m_i, exp(b_s - b_z[None, :]) * b_ss[None, :], 0.)
+    b_doo += tl.load(p_doo, boundary_check=(0, 1))
+    tl.store(p_doo, b_doo.to(p_doo.dtype.element_ty), boundary_check=(0, 1))
+class ChunkABCFunction(torch.autograd.Function):
+    @staticmethod
+    @input_guard
+    def forward(ctx, q, k, v, s, initial_state, output_final_state):
+        B, H, T, K, V, M = *q.shape, v.shape[-1], s.shape[-1]
+        BT, BC = 64, 16
+        BK = min(64, triton.next_power_of_2(K))
+        BV = min(64, triton.next_power_of_2(V))
+        BM = min(64, triton.next_power_of_2(M))
+        NT, NC = triton.cdiv(T, BT), triton.cdiv(BT, BC)
+        NV, NM = triton.cdiv(V, BV), triton.cdiv(M, BM)
+        num_warps = 4 if BK == 64 else 2
+        num_stages = 1
+        def fwd_pre(s, B, H, T, S):
+            # keep cummulative normalizer in fp32
+            z = torch.empty_like(s, dtype=torch.float)
+            grid = (B * H,)
+            logcumsumexp_fwd_kernel[grid](
+                s, z,
+                T=T, S=S,
+            )
+            return z
+        def fwd_inner(q, k, v, z, B, H, T, K, V, BT, BK, BV, NT, normk=False, h0=None, ht=None):
+            NK, NV = triton.cdiv(K, BK), triton.cdiv(V, BV)
+            h = q.new_empty(B, H, NT * K, V)
+            grid = (NV, NK, B * H)
+            chunk_abc_fwd_kernel_h[grid](
+                k, v, z, h, h0, ht,
+                T=T, K=K, V=V, BT=BT, BK=BK, BV=BV, NT=NT,
+                NORMK=normk,
+                USE_INITIAL_STATE=h0 is not None,
+                STORE_FINAL_STATE=ht is not None,
+                num_warps=num_warps,
+                num_stages=num_stages,
+            )
+            return h
+        final_state = None
+        if output_final_state:
+            final_state = (q.new_empty(B, H, K, M, dtype=torch.float),
+                           q.new_empty(B, H, M, V, dtype=torch.float))
+        z = fwd_pre(s, B, H, T, M)
+        scale = K ** -0.5
+        hk = fwd_inner(
+            q=q, k=k, v=s, z=z,
+            B=B, H=H, T=T, K=K, V=M, BT=BT, BK=BK, BV=BM, NT=NT,
+            normk=False,
+            h0=initial_state[0] if initial_state is not None else None,
+            ht=final_state[0] if final_state is not None else None,
+        )
+        ok1 = torch.empty_like(s)
+        Ak = q.new_empty(B, H, T, BT)
+        grid = (NM, NT, B * H)
+        chunk_abc_fwd_kernel_K[grid](
+            q, k, z, hk, ok1, Ak,
+            scale=scale,
+            T=T, K=K, V=M, BT=BT, BK=BK, BV=BM, NT=NT,
+            num_warps=num_warps,
+            num_stages=num_stages,
+        )
+        ok0 = torch.empty_like(s)
+        grid = (NM, NT * NC, B * H)
+        chunk_abc_fwd_kernel_intra_K[grid](
+            s, z, ok0, Ak,
+            T=T, V=M, BT=BT, BC=BC, BV=BM, NC=NC,
+            num_warps=2,
+            num_stages=num_stages,
+        )
+        ok = ok0.add_(ok1)
+        scale = 1.
+        # p is kept in fp32 for safe softmax backward
+        p = softmax_fwd(ok, dtype=torch.float)
+        qv = p.to(q.dtype)
+        scale = 1.
+        hv = fwd_inner(
+            q=qv, k=s, v=v, z=z,
+            B=B, H=H, T=T, K=M, V=V, BT=BT, BK=BM, BV=BV, NT=NT,
+            normk=True,
+            h0=initial_state[1] if initial_state is not None else None,
+            ht=final_state[1] if final_state is not None else None,
+        )
+        Av = q.new_zeros(NM, B, H, T, BT)
+        grid = (NM, NT * NC * NC, B * H)
+        chunk_abc_fwd_kernel_intra_V[grid](
+            qv, s, z, Av,
+            scale=scale,
+            T=T, K=M, BT=BT, BC=BC, BK=BM, NC=NC,
+            num_warps=2,
+            num_stages=num_stages,
+        )
+        Av = Av.sum(0)
+        ov = torch.empty_like(v)
+        grid = (NV, NT, B * H)
+        chunk_abc_fwd_kernel_V[grid](
+            qv, v, z, hv, ov, Av,
+            scale=scale,
+            T=T,
+            K=M,
+            V=V,
+            BT=BT,
+            BK=BM,
+            BV=BV,
+            NT=NT,
+            num_warps=num_warps,
+            num_stages=num_stages,
+        )
+        ctx.save_for_backward(q, k, v, s, z, ok, p, hk, hv, Av)
+        ctx.BT = BT
+        return ov, final_state
+    @staticmethod
+    @input_guard
+    def backward(ctx, dov, dht=None):
+        q, k, v, s, z, ok, p, hk, hv, Av = ctx.saved_tensors
+        B, H, T, K, V, M = *q.shape, v.shape[-1], s.shape[-1]
+        BT, BC = ctx.BT, 16
+        BK = min(64, triton.next_power_of_2(K))
+        BV = min(64, triton.next_power_of_2(V))
+        BM = min(64, triton.next_power_of_2(M))
+        NT, NC = triton.cdiv(T, BT), triton.cdiv(BT, BC)
+        NK, NM = triton.cdiv(K, BK), triton.cdiv(M, BM)
+        num_warps = 4 if BK == 64 else 2
+        num_stages = 1
+        def bwd_inner(q, z, do, B, H, T, K, V, BT, BK, BV, NT, scale, normk=False):
+            NK, NV = triton.cdiv(K, BK), triton.cdiv(V, BV)
+            dh = q.new_empty(B, H, NT * K, V)
+            grid = (NK, NV, B * H)
+            chunk_abc_bwd_kernel_dh[grid](
+                q, z, do, dh,
+                scale=scale,
+                T=T, K=K, V=V, BT=BT, BK=BK, BV=BV, NT=NT,
+                NORMK=normk,
+                num_warps=num_warps,
+                num_stages=num_stages,
+            )
+            return dh
+        def bwd_post(s, z, ss, B, H, T, S, BT, BC, BS, NT, NC, NS):
+            doo = torch.empty_like(s)
+            grid = (NS, B * H)
+            chunk_abc_bwd_kernel_rcum_inter[grid](
+                s, z, ss, doo,
+                T=T, S=S, BT=BT, BS=BS, NT=NT,
+                num_warps=num_warps,
+                num_stages=num_stages,
+            )
+            grid = (NS, NT * NC, B * H)
+            chunk_abc_bwd_kernel_rcum_intra[grid](
+                s, z, ss, doo,
+                T=T, S=S, BT=BT, BC=BC, BS=BS, NC=NC,
+                num_warps=num_warps,
+                num_stages=num_stages,
+            )
+            return doo
+        scale = 1.
+        qv = p.to(q.dtype)
+        dhv = bwd_inner(
+            qv, z, dov,
+            B=B, H=H, T=T, K=M, V=V, BT=BT, BK=BM, BV=BV, NT=NT,
+            scale=scale,
+            normk=True,
+        )
+        dp1 = torch.empty_like(p)
+        dsv1 = torch.empty_like(s, dtype=torch.float)
+        dv = v.new_empty(NM, *v.shape)
+        dAv = q.new_zeros(B, H, T, BT)
+        grid = (NM, NT, B * H)
+        chunk_abc_bwd_kernel_V[grid](
+            s, v, z, hv, Av, dov, dhv, dp1, dsv1, dv, dAv,
+            scale=scale,
+            T=T, K=M, V=V, BT=BT, BK=BM, BV=BV, NT=NT,
+            num_warps=num_warps,
+            num_stages=num_stages,
+        )
+        dv = dv.sum(0)
+        dp0 = torch.empty_like(p)
+        dsv0 = s.new_zeros(s.shape, dtype=torch.float)
+        grid = (NM, NT * NC, B * H)
+        chunk_abc_bwd_kernel_intra_V[grid](
+            qv, s, z, dAv, dp0, dsv0,
+            T=T, K=M, BT=BT, BC=BC, BK=BM, NC=NC,
+            num_warps=2,
+            num_stages=num_stages,
+        )
+        dp = dp1.add_(dp0)
+        dsv = dsv1.add_(dsv0)
+        # softmax gradient, equivalent to:
+        # dok = p * (dp - (p * dp).sum(-1, True))
+        dok = softmax_bwd(p, dp, dtype=ok.dtype)
+        scale = K ** -0.5
+        dhk = bwd_inner(
+            q, z, dok,
+            B=B, H=H, T=T, K=K, V=M, BT=BT, BK=BK, BV=BM, NT=NT,
+            scale=scale,
+            normk=False,
+        )
+        dAk = q.new_zeros(NM, B, H, T, BT)
+        grid = (NM, NT * NC * NC, B * H)
+        chunk_abc_bwd_kernel_intra_K[grid](
+            s, z, dok, dAk,
+            scale=scale,
+            T=T, V=M, BT=BT, BC=BC, BV=BM, NC=NC,
+            num_warps=2,
+            num_stages=num_stages,
+        )
+        dAk = dAk.sum(0)
+        Ak = q.new_zeros(NK, B, H, T, BT)
+        dq = torch.empty_like(q)
+        dk = torch.empty_like(k)
+        dsk1 = s.new_empty(NK, *s.shape, dtype=torch.float)
+        grid = (NK, NT, B * H)
+        chunk_abc_bwd_kernel_K[grid](
+            q, k, s, z, hk, Ak, dok, dhk, dq, dk, dsk1, dAk,
+            scale=scale,
+            T=T, K=K, V=M, BT=BT, BK=BK, BV=BM, NT=NT,
+            num_warps=num_warps,
+            num_stages=num_stages,
+        )
+        Ak = Ak.sum(0)
+        dsk1 = dsk1.sum(0)
+        dsk0 = torch.empty_like(s, dtype=torch.float)
+        grid = (NM, NT * NC, B * H)
+        chunk_abc_bwd_kernel_intra_KV[grid](
+            s, z, Ak, dok, dsk0,
+            T=T, V=M, BT=BT, BC=BC, BV=BM, NC=NC,
+            num_warps=2,
+            num_stages=num_stages,
+        )
+        ds = dsv.add_(dsk1.add_(dsk0))
+        ds -= bwd_post(s, z, ok * dok + p * dp, B, H, T, M, BT, BC, BM, NT, NC, NM)
+        ds = ds.to(s.dtype)
+        return dq, dk, dv, ds, None, None
+@torch.compiler.disable
+def chunk_abc(
+    q: torch.Tensor,
+    k: torch.Tensor,
+    v: torch.Tensor,
+    s: torch.Tensor,
+    initial_state: tuple[torch.Tensor] | None = None,
+    output_final_state: bool = False,
+    head_first: bool = False,
+) -> tuple[torch.Tensor, torch.Tensor]:
+    r"""
+    Args:
+        q (torch.Tensor):
+            queries of shape `[B, T, H, K]`.
+        k (torch.Tensor):
+            keys of shape `[B, T, H, K]`.
+        v (torch.Tensor):
+            values of shape `[B, T, H, V]`.
+        s (torch.Tensor):
+            slot representations of shape `[B, T, H, M]`.
+        initial_state (Optional[Tuple[torch.Tensor, torch.Tensor]]):
+            Initial states of shape `[B, H, K, M]` and `[B, H, M, V]`. Default: `None`.
+        output_final_state (Optional[bool]):
+            Whether to output the final state of shape `[B, H, K, M]` and `[B, H, M, V]`. Default: `False`.
+        head_first (Optional[bool]):
+            Whether the inputs are in the head-first format. Default: `False`.
+            This argument has been deprecated.
+    Returns:
+        o (torch.Tensor):
+            Outputs of shape `[B, T, H, V]`.
+        final_state (torch.Tensor):
+            Final state of shape `[B, H, K, M]` and `[B, H, M, V]` if `output_final_state=True` else `None`.
+    """
+    if not head_first:
+        q, k, v, s = map(lambda x: x.transpose(1, 2), (q, k, v, s))
+    o, final_state = ChunkABCFunction.apply(q, k, v, s, initial_state, output_final_state)
+    if not head_first:
+        o = o.transpose(1, 2)
+    return o, final_state

build/torch-cuda/ops/abc/naive.py ADDED Viewed

	@@ -0,0 +1,99 @@

+# Copyright (c) 2023-2026, Songlin Yang, Yu Zhang, Zhiyuan Li
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+# For a list of all contributors, visit:
+#   https://github.com/fla-org/flash-linear-attention/graphs/contributors
+import torch
+from einops import repeat
+def naive_recurrent_abc(
+    q: torch.Tensor,
+    k: torch.Tensor,
+    v: torch.Tensor,
+    s: torch.Tensor,
+    g: torch.Tensor | None = None,
+    scale: int | None = None,
+    initial_state: torch.Tensor | None = None,
+    output_final_state: bool | None = False,
+) -> torch.Tensor:
+    dtype = q.dtype
+    NG = q.shape[1]//k.shape[1]
+    # [batch_size, n_heads, seq_len, n_slots]
+    if g is None:
+        z = s.float().logcumsumexp(2)
+        g = torch.cat((z[:, :, :1], z[:, :, :-1]), 2) - z
+        s = torch.exp(s - z)
+    q, k, v, s, g = map(lambda x: x.float(), (q, k, v, s, g))
+    k, v, s, g = map(lambda x: repeat(x, 'b h t d -> b (h g) t d', g=NG), (k, v, s, g))
+    if initial_state is not None:
+        initial_state = tuple(map(lambda x: repeat(x, 'b h k v -> b (h g) k v', g=NG), initial_state))
+    B, H, T, K, V, M = *q.shape, v.shape[-1], s.shape[-1]
+    hk = torch.zeros(B, H, K, M, dtype=torch.float, device=q.device)
+    ok = torch.zeros_like(s)
+    if scale is None:
+        scale = q.shape[-1] ** -0.5
+    final_state = None
+    if initial_state is not None:
+        hk += initial_state[0]
+    for i in range(T):
+        q_i = q[:, :, i] * scale
+        k_i = k[:, :, i]
+        v_i = s[:, :, i]
+        g_i = g[:, :, i].exp()
+        hk = hk * g_i[..., None, :] + k_i[..., None] * v_i[..., None, :]
+        ok[:, :, i] = (q_i[..., None] * hk).sum(-2)
+    qv = ok.softmax(-1)
+    hv = torch.zeros(B, H, M, V, dtype=torch.float, device=q.device)
+    ov = torch.zeros_like(v)
+    if initial_state is not None:
+        hv += initial_state[1]
+    for i in range(T):
+        q_i = qv[:, :, i]
+        k_i = s[:, :, i]
+        v_i = v[:, :, i]
+        g_i = g[:, :, i].exp()
+        hv = hv * g_i[..., :, None] + k_i[..., None] * v_i[..., None, :]
+        ov[:, :, i] = (q_i[..., None] * hv).sum(-2)
+    if output_final_state:
+        final_state = (hk.view(B, -1, NG, K, M)[:, :, 0], hv.view(B, -1, NG, M, V)[:, :, 0])
+    return ov.to(dtype), final_state
+def naive_cumsum_abc(
+    q: torch.Tensor,
+    k: torch.Tensor,
+    v: torch.Tensor,
+    s: torch.Tensor,
+) -> torch.Tensor:
+    """
+    A simple implementation of vanilla ABC that is more aligned with the descriptions in the paper.
+    This is just for demonstration purposes, with no numerical stabilities guaranteed.
+    """
+    dtype = q.dtype
+    q, k, v, s = map(lambda x: x.float(), (q, k, v, s))
+    scale = q.shape[-1] ** -0.5
+    # [batch_size, n_heads, seq_len, n_slots]
+    s = (s - s.max(2, True)[0]).exp()
+    z = s.cumsum(2)
+    # [batch_size, n_heads, seq_len, n_slots, d_head]
+    K = (s.unsqueeze(-1) * k.unsqueeze(-2)).cumsum(2) / z.unsqueeze(-1)
+    V = (s.unsqueeze(-1) * v.unsqueeze(-2)).cumsum(2) / z.unsqueeze(-1)
+    # [batch_size, n_heads, seq_len, n_slots]
+    p = torch.einsum('...d,...md->...m', q * scale, K).softmax(-1)
+    # [batch_size, n_heads, seq_len, d_head]
+    o = torch.einsum('...m,...md->...d', p, V)
+    return o.to(dtype), None

build/torch-cuda/ops/attn/__init__.py ADDED Viewed

	@@ -0,0 +1,14 @@

+# Copyright (c) 2023-2026, Songlin Yang, Yu Zhang, Zhiyuan Li
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+# For a list of all contributors, visit:
+#   https://github.com/fla-org/flash-linear-attention/graphs/contributors
+from .naive import naive_parallel_attn
+from .parallel import parallel_attn
+__all__ = [
+    'naive_parallel_attn',
+    'parallel_attn',
+]