liangsu9988 commited on 27 days ago

Commit

b9f8d14

verified ·

1 Parent(s): 85d7854

Uploaded using `kernel-builder`.

Browse files

Files changed (21) hide show

benchmarks/benchmark_nvfp4_sf_reshape.py +127 -0
build/torch211-cxx11-cu128-x86_64-linux/__init__.py +55 -0
build/torch211-cxx11-cu128-x86_64-linux/_flashrt_nvfp4_cuda_e9a1fe0.abi3.so +3 -0
build/torch211-cxx11-cu128-x86_64-linux/_ops.py +9 -0
build/torch211-cxx11-cu128-x86_64-linux/flashrt_nvfp4/__init__.py +26 -0
build/torch211-cxx11-cu128-x86_64-linux/metadata.json +13 -0
build/torch211-cxx11-cu130-x86_64-linux/__init__.py +55 -0
build/torch211-cxx11-cu130-x86_64-linux/_flashrt_nvfp4_cuda_e9a1fe0.abi3.so +3 -0
build/torch211-cxx11-cu130-x86_64-linux/_ops.py +9 -0
build/torch211-cxx11-cu130-x86_64-linux/flashrt_nvfp4/__init__.py +26 -0
build/torch211-cxx11-cu130-x86_64-linux/metadata.json +13 -0
build/torch212-cxx11-cu130-x86_64-linux/__init__.py +55 -0
build/torch212-cxx11-cu130-x86_64-linux/_flashrt_nvfp4_cuda_e9a1fe0.abi3.so +3 -0
build/torch212-cxx11-cu130-x86_64-linux/_ops.py +9 -0
build/torch212-cxx11-cu130-x86_64-linux/flashrt_nvfp4/__init__.py +26 -0
build/torch212-cxx11-cu130-x86_64-linux/metadata.json +13 -0
build/torch212-cxx11-cu132-x86_64-linux/__init__.py +55 -0
build/torch212-cxx11-cu132-x86_64-linux/_flashrt_nvfp4_cuda_e9a1fe0.abi3.so +3 -0
build/torch212-cxx11-cu132-x86_64-linux/_ops.py +9 -0
build/torch212-cxx11-cu132-x86_64-linux/flashrt_nvfp4/__init__.py +26 -0
build/torch212-cxx11-cu132-x86_64-linux/metadata.json +13 -0

benchmarks/benchmark_nvfp4_sf_reshape.py ADDED Viewed

	@@ -0,0 +1,127 @@

+import torch
+from kernels.benchmark import Benchmark
+LAYOUT_SHAPES = [
+    ("rows1_d4096", 1, 4096),
+    ("rows2_d4096", 2, 4096),
+    ("rows31_d4096", 31, 4096),
+    ("rows32_d4096", 32, 4096),
+    ("rows33_d4096", 33, 4096),
+    ("rows127_d4096", 127, 4096),
+    ("rows128_d4096", 128, 4096),
+    ("rows129_d4096", 129, 4096),
+    ("rows16_d1024", 16, 1024),
+    ("rows16_d2048", 16, 2048),
+    ("rows16_d8192", 16, 8192),
+    ("rows16_d12288", 16, 12288),
+    ("rows64_d16384", 64, 16384),
+]
+def _reference_swizzle(scales: torch.Tensor) -> torch.Tensor:
+    rows, n_blocks = scales.shape
+    n_col_super = (n_blocks + 3) // 4
+    src = scales.cpu()
+    out = torch.zeros(
+        ((rows + 127) // 128) * n_col_super * 512,
+        dtype=torch.uint8,
+    )
+    for row in range(rows):
+        rb = row // 128
+        ri = row % 128
+        for blk in range(n_blocks):
+            cb = blk // 4
+            ci = blk % 4
+            super_idx = rb * n_col_super + cb
+            inner_off = (ri % 32) * 16 + (ri // 32) * 4 + ci
+            out[super_idx * 512 + inner_off] = src[row, blk]
+    return out.to(scales.device)
+class Nvfp4ScaleFactorReshapeBenchmark(Benchmark):
+    seed = 7
+    def _setup_shape(self, rows: int, D: int) -> None:
+        self.scales = torch.randint(
+            0,
+            256,
+            (rows, D // 16),
+            device=self.device,
+            dtype=torch.uint8,
+        )
+        n_col_super = ((D // 16) + 3) // 4
+        self.out = torch.zeros(
+            ((rows + 127) // 128) * n_col_super * 512,
+            device=self.device,
+            dtype=torch.uint8,
+        )
+    def _reference(self):
+        return _reference_swizzle(self.scales)
+    def setup_rows1_d4096(self):
+        self._setup_shape(1, 4096)
+    def benchmark_rows1_d4096(self):
+        self.kernel.nvfp4_sf_linear_to_swizzled(self.scales, out=self.out)
+    def verify_rows1_d4096(self):
+        return self._reference()
+    def setup_rows16_d12288(self):
+        self._setup_shape(16, 12288)
+    def benchmark_rows16_d12288(self):
+        self.kernel.nvfp4_sf_linear_to_swizzled(self.scales, out=self.out)
+    def verify_rows16_d12288(self):
+        return self._reference()
+    def setup_rows64_d16384(self):
+        self._setup_shape(64, 16384)
+    def benchmark_rows64_d16384(self):
+        self.kernel.nvfp4_sf_linear_to_swizzled(self.scales, out=self.out)
+    def verify_rows64_d16384(self):
+        return self._reference()
+    def setup_rows128_d4096(self):
+        self._setup_shape(128, 4096)
+    def benchmark_rows128_d4096(self):
+        self.kernel.nvfp4_sf_linear_to_swizzled(self.scales, out=self.out)
+    def verify_rows128_d4096(self):
+        return self._reference()
+    def setup_rows129_d4096(self):
+        self._setup_shape(129, 4096)
+    def benchmark_rows129_d4096(self):
+        self.kernel.nvfp4_sf_linear_to_swizzled(self.scales, out=self.out)
+    def verify_rows129_d4096(self):
+        return self._reference()
+def _register_layout_shapes() -> None:
+    for label, rows, D in LAYOUT_SHAPES:
+        def setup(self, rows=rows, D=D) -> None:
+            self._setup_shape(rows, D)
+        def benchmark(self) -> None:
+            self.kernel.nvfp4_sf_linear_to_swizzled(self.scales, out=self.out)
+        def verify(self):
+            return self._reference()
+        setattr(Nvfp4ScaleFactorReshapeBenchmark, f"setup_{label}", setup)
+        setattr(Nvfp4ScaleFactorReshapeBenchmark, f"benchmark_{label}", benchmark)
+        setattr(Nvfp4ScaleFactorReshapeBenchmark, f"verify_{label}", verify)
+_register_layout_shapes()

build/torch211-cxx11-cu128-x86_64-linux/__init__.py ADDED Viewed

	@@ -0,0 +1,55 @@

+"""FlashRT NVFP4 layout kernels."""
+from __future__ import annotations
+from typing import Optional
+import torch
+from ._ops import ops
+def nvfp4_sf_swizzled_bytes(rows: int, D: int) -> int:
+    """Return byte count for a CUTLASS Sm1xx NVFP4 swizzled SF buffer."""
+    if rows <= 0:
+        raise ValueError("rows must be positive")
+    if D <= 0 or D % 16 != 0:
+        raise ValueError("D must be positive and divisible by 16")
+    n_blocks = D // 16
+    n_row_super = (rows + 127) // 128
+    n_col_super = (n_blocks + 3) // 4
+    return n_row_super * n_col_super * 512
+def nvfp4_sf_linear_to_swizzled(
+    scales: torch.Tensor,
+    *,
+    out: Optional[torch.Tensor] = None,
+    is_sfb: bool = False,
+) -> torch.Tensor:
+    """Convert linear NVFP4 scale bytes to CUTLASS Sm1xx swizzled layout.
+    ``scales`` must be contiguous CUDA ``torch.uint8`` with shape
+    ``(rows, D / 16)``. If ``out`` is omitted, a flat ``torch.uint8`` output
+    tensor with ``nvfp4_sf_swizzled_bytes(rows, D)`` bytes is allocated.
+    """
+    if scales.dim() != 2:
+        raise ValueError("scales must have shape (rows, D / 16)")
+    rows = scales.shape[0]
+    D = scales.shape[1] * 16
+    if out is None:
+        out = torch.zeros(
+            (nvfp4_sf_swizzled_bytes(rows, D),),
+            device=scales.device,
+            dtype=torch.uint8,
+        )
+    ops.nvfp4_sf_linear_to_swizzled(scales, out, D, is_sfb)
+    return out
+__all__ = [
+    "nvfp4_sf_linear_to_swizzled",
+    "nvfp4_sf_swizzled_bytes",
+]

build/torch211-cxx11-cu128-x86_64-linux/_flashrt_nvfp4_cuda_e9a1fe0.abi3.so ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:695400c280dd6f770e837da25ff054f0caee876890f88aff1ab9fa300aa1daf8
+size 95200

build/torch211-cxx11-cu128-x86_64-linux/_ops.py ADDED Viewed

	@@ -0,0 +1,9 @@

+import torch
+from . import _flashrt_nvfp4_cuda_e9a1fe0
+ops = torch.ops._flashrt_nvfp4_cuda_e9a1fe0
+def add_op_namespace_prefix(op_name: str):
+    """
+    Prefix op by namespace.
+    """
+    return f"_flashrt_nvfp4_cuda_e9a1fe0::{op_name}"

build/torch211-cxx11-cu128-x86_64-linux/flashrt_nvfp4/__init__.py ADDED Viewed

	@@ -0,0 +1,26 @@

+import ctypes
+import importlib.util
+import sys
+from pathlib import Path
+from types import ModuleType
+def _import_from_path(file_path: Path) -> ModuleType:
+    # We cannot use the module name as-is, after adding it to `sys.modules`,
+    # it would also be used for other imports. So, we make a module name that
+    # depends on the path for it to be unique using the hex-encoded hash of
+    # the path.
+    path_hash = "{:x}".format(ctypes.c_size_t(hash(file_path.absolute())).value)
+    module_name = path_hash
+    spec = importlib.util.spec_from_file_location(module_name, file_path)
+    if spec is None:
+        raise ImportError(f"Cannot load spec for {module_name} from {file_path}")
+    module = importlib.util.module_from_spec(spec)
+    if module is None:
+        raise ImportError(f"Cannot load module {module_name} from spec")
+    sys.modules[module_name] = module
+    spec.loader.exec_module(module)  # type: ignore
+    return module
+globals().update(vars(_import_from_path(Path(__file__).parent.parent / "__init__.py")))

build/torch211-cxx11-cu128-x86_64-linux/metadata.json ADDED Viewed

	@@ -0,0 +1,13 @@

+{
+  "name": "flashrt-nvfp4",
+  "id": "_flashrt_nvfp4_cuda_e9a1fe0",
+  "version": 1,
+  "license": "Apache-2.0",
+  "python-depends": [],
+  "backend": {
+    "type": "cuda",
+    "archs": [
+      "12.0"
+    ]
+  }
+}

build/torch211-cxx11-cu130-x86_64-linux/__init__.py ADDED Viewed

	@@ -0,0 +1,55 @@

+"""FlashRT NVFP4 layout kernels."""
+from __future__ import annotations
+from typing import Optional
+import torch
+from ._ops import ops
+def nvfp4_sf_swizzled_bytes(rows: int, D: int) -> int:
+    """Return byte count for a CUTLASS Sm1xx NVFP4 swizzled SF buffer."""
+    if rows <= 0:
+        raise ValueError("rows must be positive")
+    if D <= 0 or D % 16 != 0:
+        raise ValueError("D must be positive and divisible by 16")
+    n_blocks = D // 16
+    n_row_super = (rows + 127) // 128
+    n_col_super = (n_blocks + 3) // 4
+    return n_row_super * n_col_super * 512
+def nvfp4_sf_linear_to_swizzled(
+    scales: torch.Tensor,
+    *,
+    out: Optional[torch.Tensor] = None,
+    is_sfb: bool = False,
+) -> torch.Tensor:
+    """Convert linear NVFP4 scale bytes to CUTLASS Sm1xx swizzled layout.
+    ``scales`` must be contiguous CUDA ``torch.uint8`` with shape
+    ``(rows, D / 16)``. If ``out`` is omitted, a flat ``torch.uint8`` output
+    tensor with ``nvfp4_sf_swizzled_bytes(rows, D)`` bytes is allocated.
+    """
+    if scales.dim() != 2:
+        raise ValueError("scales must have shape (rows, D / 16)")
+    rows = scales.shape[0]
+    D = scales.shape[1] * 16
+    if out is None:
+        out = torch.zeros(
+            (nvfp4_sf_swizzled_bytes(rows, D),),
+            device=scales.device,
+            dtype=torch.uint8,
+        )
+    ops.nvfp4_sf_linear_to_swizzled(scales, out, D, is_sfb)
+    return out
+__all__ = [
+    "nvfp4_sf_linear_to_swizzled",
+    "nvfp4_sf_swizzled_bytes",
+]

build/torch211-cxx11-cu130-x86_64-linux/_flashrt_nvfp4_cuda_e9a1fe0.abi3.so ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:a29dd302ad9728a9bea7899650db2a9671cd63298138658f830c6c7776243229
+size 100344

build/torch211-cxx11-cu130-x86_64-linux/_ops.py ADDED Viewed

	@@ -0,0 +1,9 @@

+import torch
+from . import _flashrt_nvfp4_cuda_e9a1fe0
+ops = torch.ops._flashrt_nvfp4_cuda_e9a1fe0
+def add_op_namespace_prefix(op_name: str):
+    """
+    Prefix op by namespace.
+    """
+    return f"_flashrt_nvfp4_cuda_e9a1fe0::{op_name}"

build/torch211-cxx11-cu130-x86_64-linux/flashrt_nvfp4/__init__.py ADDED Viewed

	@@ -0,0 +1,26 @@

+import ctypes
+import importlib.util
+import sys
+from pathlib import Path
+from types import ModuleType
+def _import_from_path(file_path: Path) -> ModuleType:
+    # We cannot use the module name as-is, after adding it to `sys.modules`,
+    # it would also be used for other imports. So, we make a module name that
+    # depends on the path for it to be unique using the hex-encoded hash of
+    # the path.
+    path_hash = "{:x}".format(ctypes.c_size_t(hash(file_path.absolute())).value)
+    module_name = path_hash
+    spec = importlib.util.spec_from_file_location(module_name, file_path)
+    if spec is None:
+        raise ImportError(f"Cannot load spec for {module_name} from {file_path}")
+    module = importlib.util.module_from_spec(spec)
+    if module is None:
+        raise ImportError(f"Cannot load module {module_name} from spec")
+    sys.modules[module_name] = module
+    spec.loader.exec_module(module)  # type: ignore
+    return module
+globals().update(vars(_import_from_path(Path(__file__).parent.parent / "__init__.py")))

build/torch211-cxx11-cu130-x86_64-linux/metadata.json ADDED Viewed

	@@ -0,0 +1,13 @@

+{
+  "name": "flashrt-nvfp4",
+  "id": "_flashrt_nvfp4_cuda_e9a1fe0",
+  "version": 1,
+  "license": "Apache-2.0",
+  "python-depends": [],
+  "backend": {
+    "type": "cuda",
+    "archs": [
+      "12.0"
+    ]
+  }
+}

build/torch212-cxx11-cu130-x86_64-linux/__init__.py ADDED Viewed

	@@ -0,0 +1,55 @@

+"""FlashRT NVFP4 layout kernels."""
+from __future__ import annotations
+from typing import Optional
+import torch
+from ._ops import ops
+def nvfp4_sf_swizzled_bytes(rows: int, D: int) -> int:
+    """Return byte count for a CUTLASS Sm1xx NVFP4 swizzled SF buffer."""
+    if rows <= 0:
+        raise ValueError("rows must be positive")
+    if D <= 0 or D % 16 != 0:
+        raise ValueError("D must be positive and divisible by 16")
+    n_blocks = D // 16
+    n_row_super = (rows + 127) // 128
+    n_col_super = (n_blocks + 3) // 4
+    return n_row_super * n_col_super * 512
+def nvfp4_sf_linear_to_swizzled(
+    scales: torch.Tensor,
+    *,
+    out: Optional[torch.Tensor] = None,
+    is_sfb: bool = False,
+) -> torch.Tensor:
+    """Convert linear NVFP4 scale bytes to CUTLASS Sm1xx swizzled layout.
+    ``scales`` must be contiguous CUDA ``torch.uint8`` with shape
+    ``(rows, D / 16)``. If ``out`` is omitted, a flat ``torch.uint8`` output
+    tensor with ``nvfp4_sf_swizzled_bytes(rows, D)`` bytes is allocated.
+    """
+    if scales.dim() != 2:
+        raise ValueError("scales must have shape (rows, D / 16)")
+    rows = scales.shape[0]
+    D = scales.shape[1] * 16
+    if out is None:
+        out = torch.zeros(
+            (nvfp4_sf_swizzled_bytes(rows, D),),
+            device=scales.device,
+            dtype=torch.uint8,
+        )
+    ops.nvfp4_sf_linear_to_swizzled(scales, out, D, is_sfb)
+    return out
+__all__ = [
+    "nvfp4_sf_linear_to_swizzled",
+    "nvfp4_sf_swizzled_bytes",
+]

build/torch212-cxx11-cu130-x86_64-linux/_flashrt_nvfp4_cuda_e9a1fe0.abi3.so ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:26f4c11dc535b3cce288d30dd0df4a6262031286fe39db48149fd25fa8165457
+size 111104

build/torch212-cxx11-cu130-x86_64-linux/_ops.py ADDED Viewed

	@@ -0,0 +1,9 @@

+import torch
+from . import _flashrt_nvfp4_cuda_e9a1fe0
+ops = torch.ops._flashrt_nvfp4_cuda_e9a1fe0
+def add_op_namespace_prefix(op_name: str):
+    """
+    Prefix op by namespace.
+    """
+    return f"_flashrt_nvfp4_cuda_e9a1fe0::{op_name}"

build/torch212-cxx11-cu130-x86_64-linux/flashrt_nvfp4/__init__.py ADDED Viewed

	@@ -0,0 +1,26 @@

+import ctypes
+import importlib.util
+import sys
+from pathlib import Path
+from types import ModuleType
+def _import_from_path(file_path: Path) -> ModuleType:
+    # We cannot use the module name as-is, after adding it to `sys.modules`,
+    # it would also be used for other imports. So, we make a module name that
+    # depends on the path for it to be unique using the hex-encoded hash of
+    # the path.
+    path_hash = "{:x}".format(ctypes.c_size_t(hash(file_path.absolute())).value)
+    module_name = path_hash
+    spec = importlib.util.spec_from_file_location(module_name, file_path)
+    if spec is None:
+        raise ImportError(f"Cannot load spec for {module_name} from {file_path}")
+    module = importlib.util.module_from_spec(spec)
+    if module is None:
+        raise ImportError(f"Cannot load module {module_name} from spec")
+    sys.modules[module_name] = module
+    spec.loader.exec_module(module)  # type: ignore
+    return module
+globals().update(vars(_import_from_path(Path(__file__).parent.parent / "__init__.py")))

build/torch212-cxx11-cu130-x86_64-linux/metadata.json ADDED Viewed

	@@ -0,0 +1,13 @@

+{
+  "name": "flashrt-nvfp4",
+  "id": "_flashrt_nvfp4_cuda_e9a1fe0",
+  "version": 1,
+  "license": "Apache-2.0",
+  "python-depends": [],
+  "backend": {
+    "type": "cuda",
+    "archs": [
+      "12.0"
+    ]
+  }
+}

build/torch212-cxx11-cu132-x86_64-linux/__init__.py ADDED Viewed

	@@ -0,0 +1,55 @@

+"""FlashRT NVFP4 layout kernels."""
+from __future__ import annotations
+from typing import Optional
+import torch
+from ._ops import ops
+def nvfp4_sf_swizzled_bytes(rows: int, D: int) -> int:
+    """Return byte count for a CUTLASS Sm1xx NVFP4 swizzled SF buffer."""
+    if rows <= 0:
+        raise ValueError("rows must be positive")
+    if D <= 0 or D % 16 != 0:
+        raise ValueError("D must be positive and divisible by 16")
+    n_blocks = D // 16
+    n_row_super = (rows + 127) // 128
+    n_col_super = (n_blocks + 3) // 4
+    return n_row_super * n_col_super * 512
+def nvfp4_sf_linear_to_swizzled(
+    scales: torch.Tensor,
+    *,
+    out: Optional[torch.Tensor] = None,
+    is_sfb: bool = False,
+) -> torch.Tensor:
+    """Convert linear NVFP4 scale bytes to CUTLASS Sm1xx swizzled layout.
+    ``scales`` must be contiguous CUDA ``torch.uint8`` with shape
+    ``(rows, D / 16)``. If ``out`` is omitted, a flat ``torch.uint8`` output
+    tensor with ``nvfp4_sf_swizzled_bytes(rows, D)`` bytes is allocated.
+    """
+    if scales.dim() != 2:
+        raise ValueError("scales must have shape (rows, D / 16)")
+    rows = scales.shape[0]
+    D = scales.shape[1] * 16
+    if out is None:
+        out = torch.zeros(
+            (nvfp4_sf_swizzled_bytes(rows, D),),
+            device=scales.device,
+            dtype=torch.uint8,
+        )
+    ops.nvfp4_sf_linear_to_swizzled(scales, out, D, is_sfb)
+    return out
+__all__ = [
+    "nvfp4_sf_linear_to_swizzled",
+    "nvfp4_sf_swizzled_bytes",
+]

build/torch212-cxx11-cu132-x86_64-linux/_flashrt_nvfp4_cuda_e9a1fe0.abi3.so ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:23ef46609656ff3387039e83a410b131ab72c2da5c3099df19acea47c6e38d45
+size 111104

build/torch212-cxx11-cu132-x86_64-linux/_ops.py ADDED Viewed

	@@ -0,0 +1,9 @@

+import torch
+from . import _flashrt_nvfp4_cuda_e9a1fe0
+ops = torch.ops._flashrt_nvfp4_cuda_e9a1fe0
+def add_op_namespace_prefix(op_name: str):
+    """
+    Prefix op by namespace.
+    """
+    return f"_flashrt_nvfp4_cuda_e9a1fe0::{op_name}"

build/torch212-cxx11-cu132-x86_64-linux/flashrt_nvfp4/__init__.py ADDED Viewed

	@@ -0,0 +1,26 @@

+import ctypes
+import importlib.util
+import sys
+from pathlib import Path
+from types import ModuleType
+def _import_from_path(file_path: Path) -> ModuleType:
+    # We cannot use the module name as-is, after adding it to `sys.modules`,
+    # it would also be used for other imports. So, we make a module name that
+    # depends on the path for it to be unique using the hex-encoded hash of
+    # the path.
+    path_hash = "{:x}".format(ctypes.c_size_t(hash(file_path.absolute())).value)
+    module_name = path_hash
+    spec = importlib.util.spec_from_file_location(module_name, file_path)
+    if spec is None:
+        raise ImportError(f"Cannot load spec for {module_name} from {file_path}")
+    module = importlib.util.module_from_spec(spec)
+    if module is None:
+        raise ImportError(f"Cannot load module {module_name} from spec")
+    sys.modules[module_name] = module
+    spec.loader.exec_module(module)  # type: ignore
+    return module
+globals().update(vars(_import_from_path(Path(__file__).parent.parent / "__init__.py")))

build/torch212-cxx11-cu132-x86_64-linux/metadata.json ADDED Viewed

	@@ -0,0 +1,13 @@

+{
+  "name": "flashrt-nvfp4",
+  "id": "_flashrt_nvfp4_cuda_e9a1fe0",
+  "version": 1,
+  "license": "Apache-2.0",
+  "python-depends": [],
+  "backend": {
+    "type": "cuda",
+    "archs": [
+      "12.0"
+    ]
+  }
+}