diff --git a/build/torch210-cxx11-cu126-x86_64-linux/__init__.py b/build/torch210-cxx11-cu126-x86_64-linux/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..9de56043369487facc1f163df6bd319c9806e5ca --- /dev/null +++ b/build/torch210-cxx11-cu126-x86_64-linux/__init__.py @@ -0,0 +1,21 @@ +from ._custom_ops import ( + convert_fp8, + copy_blocks, + paged_attention_v1, + paged_attention_v2, + reshape_and_cache, + reshape_and_cache_flash, + swap_blocks, +) +from ._ops import ops + +__all__ = [ + "convert_fp8", + "copy_blocks", + "ops", + "paged_attention_v1", + "paged_attention_v2", + "reshape_and_cache", + "reshape_and_cache_flash", + "swap_blocks", +] diff --git a/build/torch28-cxx11-cu126-x86_64-linux/paged_attention/_custom_ops.py b/build/torch210-cxx11-cu126-x86_64-linux/_custom_ops.py similarity index 100% rename from build/torch28-cxx11-cu126-x86_64-linux/paged_attention/_custom_ops.py rename to build/torch210-cxx11-cu126-x86_64-linux/_custom_ops.py diff --git a/build/torch210-cxx11-cu126-x86_64-linux/_ops.py b/build/torch210-cxx11-cu126-x86_64-linux/_ops.py new file mode 100644 index 0000000000000000000000000000000000000000..2bd4e7e759659af7e4cc1eac77bd7d389685c212 --- /dev/null +++ b/build/torch210-cxx11-cu126-x86_64-linux/_ops.py @@ -0,0 +1,9 @@ +import torch +from . import _paged_attention_61fdee0 +ops = torch.ops._paged_attention_61fdee0 + +def add_op_namespace_prefix(op_name: str): + """ + Prefix op by namespace. + """ + return f"_paged_attention_61fdee0::{op_name}" \ No newline at end of file diff --git a/build/torch210-cxx11-cu126-x86_64-linux/_paged_attention_61fdee0.abi3.so b/build/torch210-cxx11-cu126-x86_64-linux/_paged_attention_61fdee0.abi3.so new file mode 100644 index 0000000000000000000000000000000000000000..5bf23e9d256fb8551f70deb1e086383c80127fb0 --- /dev/null +++ b/build/torch210-cxx11-cu126-x86_64-linux/_paged_attention_61fdee0.abi3.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:03f696c4c474048cff123b8717faee0360b50e6bd4f409a2bedf6f897f58731b +size 140162672 diff --git a/build/torch210-cxx11-cu126-x86_64-linux/metadata.json b/build/torch210-cxx11-cu126-x86_64-linux/metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..76bafa5f33b6818aa6bb4cab04be811b87519b44 --- /dev/null +++ b/build/torch210-cxx11-cu126-x86_64-linux/metadata.json @@ -0,0 +1 @@ +{"python-depends":[]} \ No newline at end of file diff --git a/build/torch210-cxx11-cu126-x86_64-linux/paged_attention/__init__.py b/build/torch210-cxx11-cu126-x86_64-linux/paged_attention/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..03dbc1afe1cf156661a2b1b22003cd5f599a0309 --- /dev/null +++ b/build/torch210-cxx11-cu126-x86_64-linux/paged_attention/__init__.py @@ -0,0 +1,26 @@ +import ctypes +import sys + +import importlib +from pathlib import Path +from types import ModuleType + +def _import_from_path(file_path: Path) -> ModuleType: + # We cannot use the module name as-is, after adding it to `sys.modules`, + # it would also be used for other imports. So, we make a module name that + # depends on the path for it to be unique using the hex-encoded hash of + # the path. + path_hash = "{:x}".format(ctypes.c_size_t(hash(file_path.absolute())).value) + module_name = path_hash + spec = importlib.util.spec_from_file_location(module_name, file_path) + if spec is None: + raise ImportError(f"Cannot load spec for {module_name} from {file_path}") + module = importlib.util.module_from_spec(spec) + if module is None: + raise ImportError(f"Cannot load module {module_name} from spec") + sys.modules[module_name] = module + spec.loader.exec_module(module) # type: ignore + return module + + +globals().update(vars(_import_from_path(Path(__file__).parent.parent / "__init__.py"))) diff --git a/build/torch28-cxx11-cu126-x86_64-linux/paged_attention/platforms.py b/build/torch210-cxx11-cu126-x86_64-linux/platforms.py similarity index 100% rename from build/torch28-cxx11-cu126-x86_64-linux/paged_attention/platforms.py rename to build/torch210-cxx11-cu126-x86_64-linux/platforms.py diff --git a/build/torch210-cxx11-cu128-x86_64-linux/__init__.py b/build/torch210-cxx11-cu128-x86_64-linux/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..9de56043369487facc1f163df6bd319c9806e5ca --- /dev/null +++ b/build/torch210-cxx11-cu128-x86_64-linux/__init__.py @@ -0,0 +1,21 @@ +from ._custom_ops import ( + convert_fp8, + copy_blocks, + paged_attention_v1, + paged_attention_v2, + reshape_and_cache, + reshape_and_cache_flash, + swap_blocks, +) +from ._ops import ops + +__all__ = [ + "convert_fp8", + "copy_blocks", + "ops", + "paged_attention_v1", + "paged_attention_v2", + "reshape_and_cache", + "reshape_and_cache_flash", + "swap_blocks", +] diff --git a/build/torch28-cxx11-cu128-x86_64-linux/paged_attention/_custom_ops.py b/build/torch210-cxx11-cu128-x86_64-linux/_custom_ops.py similarity index 100% rename from build/torch28-cxx11-cu128-x86_64-linux/paged_attention/_custom_ops.py rename to build/torch210-cxx11-cu128-x86_64-linux/_custom_ops.py diff --git a/build/torch210-cxx11-cu128-x86_64-linux/_ops.py b/build/torch210-cxx11-cu128-x86_64-linux/_ops.py new file mode 100644 index 0000000000000000000000000000000000000000..2bd4e7e759659af7e4cc1eac77bd7d389685c212 --- /dev/null +++ b/build/torch210-cxx11-cu128-x86_64-linux/_ops.py @@ -0,0 +1,9 @@ +import torch +from . import _paged_attention_61fdee0 +ops = torch.ops._paged_attention_61fdee0 + +def add_op_namespace_prefix(op_name: str): + """ + Prefix op by namespace. + """ + return f"_paged_attention_61fdee0::{op_name}" \ No newline at end of file diff --git a/build/torch210-cxx11-cu128-x86_64-linux/_paged_attention_61fdee0.abi3.so b/build/torch210-cxx11-cu128-x86_64-linux/_paged_attention_61fdee0.abi3.so new file mode 100644 index 0000000000000000000000000000000000000000..b002c242b6810eb57a1674cd3c1cb793fae069f5 --- /dev/null +++ b/build/torch210-cxx11-cu128-x86_64-linux/_paged_attention_61fdee0.abi3.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0fc9eab1173a3561eb41273b124f0be3410761691653ab2e15fcbbf13dddf1b2 +size 167726072 diff --git a/build/torch210-cxx11-cu128-x86_64-linux/metadata.json b/build/torch210-cxx11-cu128-x86_64-linux/metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..76bafa5f33b6818aa6bb4cab04be811b87519b44 --- /dev/null +++ b/build/torch210-cxx11-cu128-x86_64-linux/metadata.json @@ -0,0 +1 @@ +{"python-depends":[]} \ No newline at end of file diff --git a/build/torch210-cxx11-cu128-x86_64-linux/paged_attention/__init__.py b/build/torch210-cxx11-cu128-x86_64-linux/paged_attention/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..03dbc1afe1cf156661a2b1b22003cd5f599a0309 --- /dev/null +++ b/build/torch210-cxx11-cu128-x86_64-linux/paged_attention/__init__.py @@ -0,0 +1,26 @@ +import ctypes +import sys + +import importlib +from pathlib import Path +from types import ModuleType + +def _import_from_path(file_path: Path) -> ModuleType: + # We cannot use the module name as-is, after adding it to `sys.modules`, + # it would also be used for other imports. So, we make a module name that + # depends on the path for it to be unique using the hex-encoded hash of + # the path. + path_hash = "{:x}".format(ctypes.c_size_t(hash(file_path.absolute())).value) + module_name = path_hash + spec = importlib.util.spec_from_file_location(module_name, file_path) + if spec is None: + raise ImportError(f"Cannot load spec for {module_name} from {file_path}") + module = importlib.util.module_from_spec(spec) + if module is None: + raise ImportError(f"Cannot load module {module_name} from spec") + sys.modules[module_name] = module + spec.loader.exec_module(module) # type: ignore + return module + + +globals().update(vars(_import_from_path(Path(__file__).parent.parent / "__init__.py"))) diff --git a/build/torch28-cxx11-cu128-x86_64-linux/paged_attention/platforms.py b/build/torch210-cxx11-cu128-x86_64-linux/platforms.py similarity index 100% rename from build/torch28-cxx11-cu128-x86_64-linux/paged_attention/platforms.py rename to build/torch210-cxx11-cu128-x86_64-linux/platforms.py diff --git a/build/torch210-cxx11-cu130-x86_64-linux/__init__.py b/build/torch210-cxx11-cu130-x86_64-linux/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..9de56043369487facc1f163df6bd319c9806e5ca --- /dev/null +++ b/build/torch210-cxx11-cu130-x86_64-linux/__init__.py @@ -0,0 +1,21 @@ +from ._custom_ops import ( + convert_fp8, + copy_blocks, + paged_attention_v1, + paged_attention_v2, + reshape_and_cache, + reshape_and_cache_flash, + swap_blocks, +) +from ._ops import ops + +__all__ = [ + "convert_fp8", + "copy_blocks", + "ops", + "paged_attention_v1", + "paged_attention_v2", + "reshape_and_cache", + "reshape_and_cache_flash", + "swap_blocks", +] diff --git a/build/torch28-cxx11-cu129-x86_64-linux/paged_attention/_custom_ops.py b/build/torch210-cxx11-cu130-x86_64-linux/_custom_ops.py similarity index 100% rename from build/torch28-cxx11-cu129-x86_64-linux/paged_attention/_custom_ops.py rename to build/torch210-cxx11-cu130-x86_64-linux/_custom_ops.py diff --git a/build/torch210-cxx11-cu130-x86_64-linux/_ops.py b/build/torch210-cxx11-cu130-x86_64-linux/_ops.py new file mode 100644 index 0000000000000000000000000000000000000000..2bd4e7e759659af7e4cc1eac77bd7d389685c212 --- /dev/null +++ b/build/torch210-cxx11-cu130-x86_64-linux/_ops.py @@ -0,0 +1,9 @@ +import torch +from . import _paged_attention_61fdee0 +ops = torch.ops._paged_attention_61fdee0 + +def add_op_namespace_prefix(op_name: str): + """ + Prefix op by namespace. + """ + return f"_paged_attention_61fdee0::{op_name}" \ No newline at end of file diff --git a/build/torch210-cxx11-cu130-x86_64-linux/_paged_attention_61fdee0.abi3.so b/build/torch210-cxx11-cu130-x86_64-linux/_paged_attention_61fdee0.abi3.so new file mode 100644 index 0000000000000000000000000000000000000000..6849c076874f1ab122b62fd07be84861a483c523 --- /dev/null +++ b/build/torch210-cxx11-cu130-x86_64-linux/_paged_attention_61fdee0.abi3.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:02541ec822b88d3a774823154414c689c258080c719887103f26f82db12a2ca6 +size 86563760 diff --git a/build/torch210-cxx11-cu130-x86_64-linux/metadata.json b/build/torch210-cxx11-cu130-x86_64-linux/metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..76bafa5f33b6818aa6bb4cab04be811b87519b44 --- /dev/null +++ b/build/torch210-cxx11-cu130-x86_64-linux/metadata.json @@ -0,0 +1 @@ +{"python-depends":[]} \ No newline at end of file diff --git a/build/torch210-cxx11-cu130-x86_64-linux/paged_attention/__init__.py b/build/torch210-cxx11-cu130-x86_64-linux/paged_attention/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..03dbc1afe1cf156661a2b1b22003cd5f599a0309 --- /dev/null +++ b/build/torch210-cxx11-cu130-x86_64-linux/paged_attention/__init__.py @@ -0,0 +1,26 @@ +import ctypes +import sys + +import importlib +from pathlib import Path +from types import ModuleType + +def _import_from_path(file_path: Path) -> ModuleType: + # We cannot use the module name as-is, after adding it to `sys.modules`, + # it would also be used for other imports. So, we make a module name that + # depends on the path for it to be unique using the hex-encoded hash of + # the path. + path_hash = "{:x}".format(ctypes.c_size_t(hash(file_path.absolute())).value) + module_name = path_hash + spec = importlib.util.spec_from_file_location(module_name, file_path) + if spec is None: + raise ImportError(f"Cannot load spec for {module_name} from {file_path}") + module = importlib.util.module_from_spec(spec) + if module is None: + raise ImportError(f"Cannot load module {module_name} from spec") + sys.modules[module_name] = module + spec.loader.exec_module(module) # type: ignore + return module + + +globals().update(vars(_import_from_path(Path(__file__).parent.parent / "__init__.py"))) diff --git a/build/torch28-cxx11-cu129-x86_64-linux/paged_attention/platforms.py b/build/torch210-cxx11-cu130-x86_64-linux/platforms.py similarity index 100% rename from build/torch28-cxx11-cu129-x86_64-linux/paged_attention/platforms.py rename to build/torch210-cxx11-cu130-x86_64-linux/platforms.py diff --git a/build/torch210-cxx11-rocm70-x86_64-linux/__init__.py b/build/torch210-cxx11-rocm70-x86_64-linux/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..9de56043369487facc1f163df6bd319c9806e5ca --- /dev/null +++ b/build/torch210-cxx11-rocm70-x86_64-linux/__init__.py @@ -0,0 +1,21 @@ +from ._custom_ops import ( + convert_fp8, + copy_blocks, + paged_attention_v1, + paged_attention_v2, + reshape_and_cache, + reshape_and_cache_flash, + swap_blocks, +) +from ._ops import ops + +__all__ = [ + "convert_fp8", + "copy_blocks", + "ops", + "paged_attention_v1", + "paged_attention_v2", + "reshape_and_cache", + "reshape_and_cache_flash", + "swap_blocks", +] diff --git a/build/torch28-cxx11-rocm63-x86_64-linux/paged_attention/_custom_ops.py b/build/torch210-cxx11-rocm70-x86_64-linux/_custom_ops.py similarity index 100% rename from build/torch28-cxx11-rocm63-x86_64-linux/paged_attention/_custom_ops.py rename to build/torch210-cxx11-rocm70-x86_64-linux/_custom_ops.py diff --git a/build/torch210-cxx11-rocm70-x86_64-linux/_ops.py b/build/torch210-cxx11-rocm70-x86_64-linux/_ops.py new file mode 100644 index 0000000000000000000000000000000000000000..2bd4e7e759659af7e4cc1eac77bd7d389685c212 --- /dev/null +++ b/build/torch210-cxx11-rocm70-x86_64-linux/_ops.py @@ -0,0 +1,9 @@ +import torch +from . import _paged_attention_61fdee0 +ops = torch.ops._paged_attention_61fdee0 + +def add_op_namespace_prefix(op_name: str): + """ + Prefix op by namespace. + """ + return f"_paged_attention_61fdee0::{op_name}" \ No newline at end of file diff --git a/build/torch210-cxx11-rocm70-x86_64-linux/_paged_attention_61fdee0.abi3.so b/build/torch210-cxx11-rocm70-x86_64-linux/_paged_attention_61fdee0.abi3.so new file mode 100644 index 0000000000000000000000000000000000000000..593324309e1d9817f864ec42f802326998af0960 --- /dev/null +++ b/build/torch210-cxx11-rocm70-x86_64-linux/_paged_attention_61fdee0.abi3.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:393f597797a88bc946476a4309c1a90bb5742e4566bdc311f04bbdd3d80648bf +size 118183688 diff --git a/build/torch210-cxx11-rocm70-x86_64-linux/metadata.json b/build/torch210-cxx11-rocm70-x86_64-linux/metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..76bafa5f33b6818aa6bb4cab04be811b87519b44 --- /dev/null +++ b/build/torch210-cxx11-rocm70-x86_64-linux/metadata.json @@ -0,0 +1 @@ +{"python-depends":[]} \ No newline at end of file diff --git a/build/torch210-cxx11-rocm70-x86_64-linux/paged_attention/__init__.py b/build/torch210-cxx11-rocm70-x86_64-linux/paged_attention/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..03dbc1afe1cf156661a2b1b22003cd5f599a0309 --- /dev/null +++ b/build/torch210-cxx11-rocm70-x86_64-linux/paged_attention/__init__.py @@ -0,0 +1,26 @@ +import ctypes +import sys + +import importlib +from pathlib import Path +from types import ModuleType + +def _import_from_path(file_path: Path) -> ModuleType: + # We cannot use the module name as-is, after adding it to `sys.modules`, + # it would also be used for other imports. So, we make a module name that + # depends on the path for it to be unique using the hex-encoded hash of + # the path. + path_hash = "{:x}".format(ctypes.c_size_t(hash(file_path.absolute())).value) + module_name = path_hash + spec = importlib.util.spec_from_file_location(module_name, file_path) + if spec is None: + raise ImportError(f"Cannot load spec for {module_name} from {file_path}") + module = importlib.util.module_from_spec(spec) + if module is None: + raise ImportError(f"Cannot load module {module_name} from spec") + sys.modules[module_name] = module + spec.loader.exec_module(module) # type: ignore + return module + + +globals().update(vars(_import_from_path(Path(__file__).parent.parent / "__init__.py"))) diff --git a/build/torch28-cxx11-rocm63-x86_64-linux/paged_attention/platforms.py b/build/torch210-cxx11-rocm70-x86_64-linux/platforms.py similarity index 100% rename from build/torch28-cxx11-rocm63-x86_64-linux/paged_attention/platforms.py rename to build/torch210-cxx11-rocm70-x86_64-linux/platforms.py diff --git a/build/torch210-cxx11-rocm71-x86_64-linux/__init__.py b/build/torch210-cxx11-rocm71-x86_64-linux/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..9de56043369487facc1f163df6bd319c9806e5ca --- /dev/null +++ b/build/torch210-cxx11-rocm71-x86_64-linux/__init__.py @@ -0,0 +1,21 @@ +from ._custom_ops import ( + convert_fp8, + copy_blocks, + paged_attention_v1, + paged_attention_v2, + reshape_and_cache, + reshape_and_cache_flash, + swap_blocks, +) +from ._ops import ops + +__all__ = [ + "convert_fp8", + "copy_blocks", + "ops", + "paged_attention_v1", + "paged_attention_v2", + "reshape_and_cache", + "reshape_and_cache_flash", + "swap_blocks", +] diff --git a/build/torch28-cxx11-rocm64-x86_64-linux/paged_attention/_custom_ops.py b/build/torch210-cxx11-rocm71-x86_64-linux/_custom_ops.py similarity index 100% rename from build/torch28-cxx11-rocm64-x86_64-linux/paged_attention/_custom_ops.py rename to build/torch210-cxx11-rocm71-x86_64-linux/_custom_ops.py diff --git a/build/torch210-cxx11-rocm71-x86_64-linux/_ops.py b/build/torch210-cxx11-rocm71-x86_64-linux/_ops.py new file mode 100644 index 0000000000000000000000000000000000000000..2bd4e7e759659af7e4cc1eac77bd7d389685c212 --- /dev/null +++ b/build/torch210-cxx11-rocm71-x86_64-linux/_ops.py @@ -0,0 +1,9 @@ +import torch +from . import _paged_attention_61fdee0 +ops = torch.ops._paged_attention_61fdee0 + +def add_op_namespace_prefix(op_name: str): + """ + Prefix op by namespace. + """ + return f"_paged_attention_61fdee0::{op_name}" \ No newline at end of file diff --git a/build/torch210-cxx11-rocm71-x86_64-linux/_paged_attention_61fdee0.abi3.so b/build/torch210-cxx11-rocm71-x86_64-linux/_paged_attention_61fdee0.abi3.so new file mode 100644 index 0000000000000000000000000000000000000000..f1ef8e7e1d227a46779dfc83ea8fd9b51b113b1c --- /dev/null +++ b/build/torch210-cxx11-rocm71-x86_64-linux/_paged_attention_61fdee0.abi3.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f2b2140dfa254ee167124932eda2da2ad9697c55a51b2c9a069a8293067e1710 +size 118089384 diff --git a/build/torch210-cxx11-rocm71-x86_64-linux/metadata.json b/build/torch210-cxx11-rocm71-x86_64-linux/metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..76bafa5f33b6818aa6bb4cab04be811b87519b44 --- /dev/null +++ b/build/torch210-cxx11-rocm71-x86_64-linux/metadata.json @@ -0,0 +1 @@ +{"python-depends":[]} \ No newline at end of file diff --git a/build/torch210-cxx11-rocm71-x86_64-linux/paged_attention/__init__.py b/build/torch210-cxx11-rocm71-x86_64-linux/paged_attention/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..03dbc1afe1cf156661a2b1b22003cd5f599a0309 --- /dev/null +++ b/build/torch210-cxx11-rocm71-x86_64-linux/paged_attention/__init__.py @@ -0,0 +1,26 @@ +import ctypes +import sys + +import importlib +from pathlib import Path +from types import ModuleType + +def _import_from_path(file_path: Path) -> ModuleType: + # We cannot use the module name as-is, after adding it to `sys.modules`, + # it would also be used for other imports. So, we make a module name that + # depends on the path for it to be unique using the hex-encoded hash of + # the path. + path_hash = "{:x}".format(ctypes.c_size_t(hash(file_path.absolute())).value) + module_name = path_hash + spec = importlib.util.spec_from_file_location(module_name, file_path) + if spec is None: + raise ImportError(f"Cannot load spec for {module_name} from {file_path}") + module = importlib.util.module_from_spec(spec) + if module is None: + raise ImportError(f"Cannot load module {module_name} from spec") + sys.modules[module_name] = module + spec.loader.exec_module(module) # type: ignore + return module + + +globals().update(vars(_import_from_path(Path(__file__).parent.parent / "__init__.py"))) diff --git a/build/torch28-cxx11-rocm64-x86_64-linux/paged_attention/platforms.py b/build/torch210-cxx11-rocm71-x86_64-linux/platforms.py similarity index 100% rename from build/torch28-cxx11-rocm64-x86_64-linux/paged_attention/platforms.py rename to build/torch210-cxx11-rocm71-x86_64-linux/platforms.py diff --git a/build/torch28-cxx11-cu126-x86_64-linux/__init__.py b/build/torch28-cxx11-cu126-x86_64-linux/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..9de56043369487facc1f163df6bd319c9806e5ca --- /dev/null +++ b/build/torch28-cxx11-cu126-x86_64-linux/__init__.py @@ -0,0 +1,21 @@ +from ._custom_ops import ( + convert_fp8, + copy_blocks, + paged_attention_v1, + paged_attention_v2, + reshape_and_cache, + reshape_and_cache_flash, + swap_blocks, +) +from ._ops import ops + +__all__ = [ + "convert_fp8", + "copy_blocks", + "ops", + "paged_attention_v1", + "paged_attention_v2", + "reshape_and_cache", + "reshape_and_cache_flash", + "swap_blocks", +] diff --git a/build/torch29-cxx11-cu126-x86_64-linux/paged_attention/_custom_ops.py b/build/torch28-cxx11-cu126-x86_64-linux/_custom_ops.py similarity index 100% rename from build/torch29-cxx11-cu126-x86_64-linux/paged_attention/_custom_ops.py rename to build/torch28-cxx11-cu126-x86_64-linux/_custom_ops.py diff --git a/build/torch28-cxx11-cu126-x86_64-linux/_ops.py b/build/torch28-cxx11-cu126-x86_64-linux/_ops.py new file mode 100644 index 0000000000000000000000000000000000000000..2bd4e7e759659af7e4cc1eac77bd7d389685c212 --- /dev/null +++ b/build/torch28-cxx11-cu126-x86_64-linux/_ops.py @@ -0,0 +1,9 @@ +import torch +from . import _paged_attention_61fdee0 +ops = torch.ops._paged_attention_61fdee0 + +def add_op_namespace_prefix(op_name: str): + """ + Prefix op by namespace. + """ + return f"_paged_attention_61fdee0::{op_name}" \ No newline at end of file diff --git a/build/torch28-cxx11-cu126-x86_64-linux/_paged_attention_61fdee0.abi3.so b/build/torch28-cxx11-cu126-x86_64-linux/_paged_attention_61fdee0.abi3.so new file mode 100644 index 0000000000000000000000000000000000000000..8b26436fcf6269996928a344ec03f9c25c2ff76c --- /dev/null +++ b/build/torch28-cxx11-cu126-x86_64-linux/_paged_attention_61fdee0.abi3.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:38d093ba15e819a09622c0f4f87b895bebf7cef5c9e0a24a85f8c6fc66e2e797 +size 140143504 diff --git a/build/torch28-cxx11-cu126-x86_64-linux/metadata.json b/build/torch28-cxx11-cu126-x86_64-linux/metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..76bafa5f33b6818aa6bb4cab04be811b87519b44 --- /dev/null +++ b/build/torch28-cxx11-cu126-x86_64-linux/metadata.json @@ -0,0 +1 @@ +{"python-depends":[]} \ No newline at end of file diff --git a/build/torch28-cxx11-cu126-x86_64-linux/paged_attention/__init__.py b/build/torch28-cxx11-cu126-x86_64-linux/paged_attention/__init__.py index 9de56043369487facc1f163df6bd319c9806e5ca..03dbc1afe1cf156661a2b1b22003cd5f599a0309 100644 --- a/build/torch28-cxx11-cu126-x86_64-linux/paged_attention/__init__.py +++ b/build/torch28-cxx11-cu126-x86_64-linux/paged_attention/__init__.py @@ -1,21 +1,26 @@ -from ._custom_ops import ( - convert_fp8, - copy_blocks, - paged_attention_v1, - paged_attention_v2, - reshape_and_cache, - reshape_and_cache_flash, - swap_blocks, -) -from ._ops import ops +import ctypes +import sys -__all__ = [ - "convert_fp8", - "copy_blocks", - "ops", - "paged_attention_v1", - "paged_attention_v2", - "reshape_and_cache", - "reshape_and_cache_flash", - "swap_blocks", -] +import importlib +from pathlib import Path +from types import ModuleType + +def _import_from_path(file_path: Path) -> ModuleType: + # We cannot use the module name as-is, after adding it to `sys.modules`, + # it would also be used for other imports. So, we make a module name that + # depends on the path for it to be unique using the hex-encoded hash of + # the path. + path_hash = "{:x}".format(ctypes.c_size_t(hash(file_path.absolute())).value) + module_name = path_hash + spec = importlib.util.spec_from_file_location(module_name, file_path) + if spec is None: + raise ImportError(f"Cannot load spec for {module_name} from {file_path}") + module = importlib.util.module_from_spec(spec) + if module is None: + raise ImportError(f"Cannot load module {module_name} from spec") + sys.modules[module_name] = module + spec.loader.exec_module(module) # type: ignore + return module + + +globals().update(vars(_import_from_path(Path(__file__).parent.parent / "__init__.py"))) diff --git a/build/torch28-cxx11-cu126-x86_64-linux/paged_attention/_ops.py b/build/torch28-cxx11-cu126-x86_64-linux/paged_attention/_ops.py deleted file mode 100644 index d09b198f2169310837bcd6b8ea88308879b5775d..0000000000000000000000000000000000000000 --- a/build/torch28-cxx11-cu126-x86_64-linux/paged_attention/_ops.py +++ /dev/null @@ -1,9 +0,0 @@ -import torch -from . import _paged_attention_287831d -ops = torch.ops._paged_attention_287831d - -def add_op_namespace_prefix(op_name: str): - """ - Prefix op by namespace. - """ - return f"_paged_attention_287831d::{op_name}" \ No newline at end of file diff --git a/build/torch28-cxx11-cu126-x86_64-linux/paged_attention/_paged_attention_287831d.abi3.so b/build/torch28-cxx11-cu126-x86_64-linux/paged_attention/_paged_attention_287831d.abi3.so deleted file mode 100644 index cc815e53f08fc6f9626837f54ef0ad5c683bd951..0000000000000000000000000000000000000000 --- a/build/torch28-cxx11-cu126-x86_64-linux/paged_attention/_paged_attention_287831d.abi3.so +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:d7df0c8b8817d956e843be64c94018f7a4f11059f4efd6e0e8e09171afdaa0f4 -size 85751728 diff --git a/build/torch29-cxx11-cu126-x86_64-linux/paged_attention/platforms.py b/build/torch28-cxx11-cu126-x86_64-linux/platforms.py similarity index 100% rename from build/torch29-cxx11-cu126-x86_64-linux/paged_attention/platforms.py rename to build/torch28-cxx11-cu126-x86_64-linux/platforms.py diff --git a/build/torch28-cxx11-cu128-x86_64-linux/__init__.py b/build/torch28-cxx11-cu128-x86_64-linux/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..9de56043369487facc1f163df6bd319c9806e5ca --- /dev/null +++ b/build/torch28-cxx11-cu128-x86_64-linux/__init__.py @@ -0,0 +1,21 @@ +from ._custom_ops import ( + convert_fp8, + copy_blocks, + paged_attention_v1, + paged_attention_v2, + reshape_and_cache, + reshape_and_cache_flash, + swap_blocks, +) +from ._ops import ops + +__all__ = [ + "convert_fp8", + "copy_blocks", + "ops", + "paged_attention_v1", + "paged_attention_v2", + "reshape_and_cache", + "reshape_and_cache_flash", + "swap_blocks", +] diff --git a/build/torch29-cxx11-cu128-x86_64-linux/paged_attention/_custom_ops.py b/build/torch28-cxx11-cu128-x86_64-linux/_custom_ops.py similarity index 100% rename from build/torch29-cxx11-cu128-x86_64-linux/paged_attention/_custom_ops.py rename to build/torch28-cxx11-cu128-x86_64-linux/_custom_ops.py diff --git a/build/torch28-cxx11-cu128-x86_64-linux/_ops.py b/build/torch28-cxx11-cu128-x86_64-linux/_ops.py new file mode 100644 index 0000000000000000000000000000000000000000..2bd4e7e759659af7e4cc1eac77bd7d389685c212 --- /dev/null +++ b/build/torch28-cxx11-cu128-x86_64-linux/_ops.py @@ -0,0 +1,9 @@ +import torch +from . import _paged_attention_61fdee0 +ops = torch.ops._paged_attention_61fdee0 + +def add_op_namespace_prefix(op_name: str): + """ + Prefix op by namespace. + """ + return f"_paged_attention_61fdee0::{op_name}" \ No newline at end of file diff --git a/build/torch28-cxx11-cu128-x86_64-linux/_paged_attention_61fdee0.abi3.so b/build/torch28-cxx11-cu128-x86_64-linux/_paged_attention_61fdee0.abi3.so new file mode 100644 index 0000000000000000000000000000000000000000..048ba728e45b778f3fcdaf63b46ef3d1080429d3 --- /dev/null +++ b/build/torch28-cxx11-cu128-x86_64-linux/_paged_attention_61fdee0.abi3.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:98bf417805044e0a2aecc82d925892da63df504f3fdde94017bfe4c2e334ca20 +size 167710768 diff --git a/build/torch28-cxx11-cu128-x86_64-linux/metadata.json b/build/torch28-cxx11-cu128-x86_64-linux/metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..76bafa5f33b6818aa6bb4cab04be811b87519b44 --- /dev/null +++ b/build/torch28-cxx11-cu128-x86_64-linux/metadata.json @@ -0,0 +1 @@ +{"python-depends":[]} \ No newline at end of file diff --git a/build/torch28-cxx11-cu128-x86_64-linux/paged_attention/__init__.py b/build/torch28-cxx11-cu128-x86_64-linux/paged_attention/__init__.py index 9de56043369487facc1f163df6bd319c9806e5ca..03dbc1afe1cf156661a2b1b22003cd5f599a0309 100644 --- a/build/torch28-cxx11-cu128-x86_64-linux/paged_attention/__init__.py +++ b/build/torch28-cxx11-cu128-x86_64-linux/paged_attention/__init__.py @@ -1,21 +1,26 @@ -from ._custom_ops import ( - convert_fp8, - copy_blocks, - paged_attention_v1, - paged_attention_v2, - reshape_and_cache, - reshape_and_cache_flash, - swap_blocks, -) -from ._ops import ops +import ctypes +import sys -__all__ = [ - "convert_fp8", - "copy_blocks", - "ops", - "paged_attention_v1", - "paged_attention_v2", - "reshape_and_cache", - "reshape_and_cache_flash", - "swap_blocks", -] +import importlib +from pathlib import Path +from types import ModuleType + +def _import_from_path(file_path: Path) -> ModuleType: + # We cannot use the module name as-is, after adding it to `sys.modules`, + # it would also be used for other imports. So, we make a module name that + # depends on the path for it to be unique using the hex-encoded hash of + # the path. + path_hash = "{:x}".format(ctypes.c_size_t(hash(file_path.absolute())).value) + module_name = path_hash + spec = importlib.util.spec_from_file_location(module_name, file_path) + if spec is None: + raise ImportError(f"Cannot load spec for {module_name} from {file_path}") + module = importlib.util.module_from_spec(spec) + if module is None: + raise ImportError(f"Cannot load module {module_name} from spec") + sys.modules[module_name] = module + spec.loader.exec_module(module) # type: ignore + return module + + +globals().update(vars(_import_from_path(Path(__file__).parent.parent / "__init__.py"))) diff --git a/build/torch28-cxx11-cu128-x86_64-linux/paged_attention/_ops.py b/build/torch28-cxx11-cu128-x86_64-linux/paged_attention/_ops.py deleted file mode 100644 index d09b198f2169310837bcd6b8ea88308879b5775d..0000000000000000000000000000000000000000 --- a/build/torch28-cxx11-cu128-x86_64-linux/paged_attention/_ops.py +++ /dev/null @@ -1,9 +0,0 @@ -import torch -from . import _paged_attention_287831d -ops = torch.ops._paged_attention_287831d - -def add_op_namespace_prefix(op_name: str): - """ - Prefix op by namespace. - """ - return f"_paged_attention_287831d::{op_name}" \ No newline at end of file diff --git a/build/torch28-cxx11-cu128-x86_64-linux/paged_attention/_paged_attention_287831d.abi3.so b/build/torch28-cxx11-cu128-x86_64-linux/paged_attention/_paged_attention_287831d.abi3.so deleted file mode 100644 index d4a560ead15f25ea9e44b98558bb9eb531274ba2..0000000000000000000000000000000000000000 --- a/build/torch28-cxx11-cu128-x86_64-linux/paged_attention/_paged_attention_287831d.abi3.so +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:a60fe3114de0950f07a243dd41c6da8805d293fd649de6fc580ac585ffb19ca2 -size 102693952 diff --git a/build/torch29-cxx11-cu128-x86_64-linux/paged_attention/platforms.py b/build/torch28-cxx11-cu128-x86_64-linux/platforms.py similarity index 100% rename from build/torch29-cxx11-cu128-x86_64-linux/paged_attention/platforms.py rename to build/torch28-cxx11-cu128-x86_64-linux/platforms.py diff --git a/build/torch28-cxx11-cu129-x86_64-linux/__init__.py b/build/torch28-cxx11-cu129-x86_64-linux/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..9de56043369487facc1f163df6bd319c9806e5ca --- /dev/null +++ b/build/torch28-cxx11-cu129-x86_64-linux/__init__.py @@ -0,0 +1,21 @@ +from ._custom_ops import ( + convert_fp8, + copy_blocks, + paged_attention_v1, + paged_attention_v2, + reshape_and_cache, + reshape_and_cache_flash, + swap_blocks, +) +from ._ops import ops + +__all__ = [ + "convert_fp8", + "copy_blocks", + "ops", + "paged_attention_v1", + "paged_attention_v2", + "reshape_and_cache", + "reshape_and_cache_flash", + "swap_blocks", +] diff --git a/build/torch29-cxx11-cu130-x86_64-linux/paged_attention/_custom_ops.py b/build/torch28-cxx11-cu129-x86_64-linux/_custom_ops.py similarity index 100% rename from build/torch29-cxx11-cu130-x86_64-linux/paged_attention/_custom_ops.py rename to build/torch28-cxx11-cu129-x86_64-linux/_custom_ops.py diff --git a/build/torch28-cxx11-cu129-x86_64-linux/_ops.py b/build/torch28-cxx11-cu129-x86_64-linux/_ops.py new file mode 100644 index 0000000000000000000000000000000000000000..2bd4e7e759659af7e4cc1eac77bd7d389685c212 --- /dev/null +++ b/build/torch28-cxx11-cu129-x86_64-linux/_ops.py @@ -0,0 +1,9 @@ +import torch +from . import _paged_attention_61fdee0 +ops = torch.ops._paged_attention_61fdee0 + +def add_op_namespace_prefix(op_name: str): + """ + Prefix op by namespace. + """ + return f"_paged_attention_61fdee0::{op_name}" \ No newline at end of file diff --git a/build/torch28-cxx11-cu129-x86_64-linux/_paged_attention_61fdee0.abi3.so b/build/torch28-cxx11-cu129-x86_64-linux/_paged_attention_61fdee0.abi3.so new file mode 100644 index 0000000000000000000000000000000000000000..94fa2f9eb60163286175a28c037cafe893f42f70 --- /dev/null +++ b/build/torch28-cxx11-cu129-x86_64-linux/_paged_attention_61fdee0.abi3.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f7c79bc2b1b87298cef91da8c07913ac2e7e1def5bcab75793e2f516680a37bd +size 182084600 diff --git a/build/torch28-cxx11-cu129-x86_64-linux/metadata.json b/build/torch28-cxx11-cu129-x86_64-linux/metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..76bafa5f33b6818aa6bb4cab04be811b87519b44 --- /dev/null +++ b/build/torch28-cxx11-cu129-x86_64-linux/metadata.json @@ -0,0 +1 @@ +{"python-depends":[]} \ No newline at end of file diff --git a/build/torch28-cxx11-cu129-x86_64-linux/paged_attention/__init__.py b/build/torch28-cxx11-cu129-x86_64-linux/paged_attention/__init__.py index 9de56043369487facc1f163df6bd319c9806e5ca..03dbc1afe1cf156661a2b1b22003cd5f599a0309 100644 --- a/build/torch28-cxx11-cu129-x86_64-linux/paged_attention/__init__.py +++ b/build/torch28-cxx11-cu129-x86_64-linux/paged_attention/__init__.py @@ -1,21 +1,26 @@ -from ._custom_ops import ( - convert_fp8, - copy_blocks, - paged_attention_v1, - paged_attention_v2, - reshape_and_cache, - reshape_and_cache_flash, - swap_blocks, -) -from ._ops import ops +import ctypes +import sys -__all__ = [ - "convert_fp8", - "copy_blocks", - "ops", - "paged_attention_v1", - "paged_attention_v2", - "reshape_and_cache", - "reshape_and_cache_flash", - "swap_blocks", -] +import importlib +from pathlib import Path +from types import ModuleType + +def _import_from_path(file_path: Path) -> ModuleType: + # We cannot use the module name as-is, after adding it to `sys.modules`, + # it would also be used for other imports. So, we make a module name that + # depends on the path for it to be unique using the hex-encoded hash of + # the path. + path_hash = "{:x}".format(ctypes.c_size_t(hash(file_path.absolute())).value) + module_name = path_hash + spec = importlib.util.spec_from_file_location(module_name, file_path) + if spec is None: + raise ImportError(f"Cannot load spec for {module_name} from {file_path}") + module = importlib.util.module_from_spec(spec) + if module is None: + raise ImportError(f"Cannot load module {module_name} from spec") + sys.modules[module_name] = module + spec.loader.exec_module(module) # type: ignore + return module + + +globals().update(vars(_import_from_path(Path(__file__).parent.parent / "__init__.py"))) diff --git a/build/torch28-cxx11-cu129-x86_64-linux/paged_attention/_ops.py b/build/torch28-cxx11-cu129-x86_64-linux/paged_attention/_ops.py deleted file mode 100644 index d09b198f2169310837bcd6b8ea88308879b5775d..0000000000000000000000000000000000000000 --- a/build/torch28-cxx11-cu129-x86_64-linux/paged_attention/_ops.py +++ /dev/null @@ -1,9 +0,0 @@ -import torch -from . import _paged_attention_287831d -ops = torch.ops._paged_attention_287831d - -def add_op_namespace_prefix(op_name: str): - """ - Prefix op by namespace. - """ - return f"_paged_attention_287831d::{op_name}" \ No newline at end of file diff --git a/build/torch28-cxx11-cu129-x86_64-linux/paged_attention/_paged_attention_287831d.abi3.so b/build/torch28-cxx11-cu129-x86_64-linux/paged_attention/_paged_attention_287831d.abi3.so deleted file mode 100644 index d0bd04d89b3f4bab5737456feafdb8071033fc92..0000000000000000000000000000000000000000 --- a/build/torch28-cxx11-cu129-x86_64-linux/paged_attention/_paged_attention_287831d.abi3.so +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:0bea4028251eaee2c98c67b9b20dbf575b49ab4c747bb31f6c8a06a0870ba969 -size 111214600 diff --git a/build/torch29-cxx11-cu130-x86_64-linux/paged_attention/platforms.py b/build/torch28-cxx11-cu129-x86_64-linux/platforms.py similarity index 100% rename from build/torch29-cxx11-cu130-x86_64-linux/paged_attention/platforms.py rename to build/torch28-cxx11-cu129-x86_64-linux/platforms.py diff --git a/build/torch28-cxx11-rocm63-x86_64-linux/__init__.py b/build/torch28-cxx11-rocm63-x86_64-linux/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..9de56043369487facc1f163df6bd319c9806e5ca --- /dev/null +++ b/build/torch28-cxx11-rocm63-x86_64-linux/__init__.py @@ -0,0 +1,21 @@ +from ._custom_ops import ( + convert_fp8, + copy_blocks, + paged_attention_v1, + paged_attention_v2, + reshape_and_cache, + reshape_and_cache_flash, + swap_blocks, +) +from ._ops import ops + +__all__ = [ + "convert_fp8", + "copy_blocks", + "ops", + "paged_attention_v1", + "paged_attention_v2", + "reshape_and_cache", + "reshape_and_cache_flash", + "swap_blocks", +] diff --git a/build/torch29-cxx11-rocm63-x86_64-linux/paged_attention/_custom_ops.py b/build/torch28-cxx11-rocm63-x86_64-linux/_custom_ops.py similarity index 100% rename from build/torch29-cxx11-rocm63-x86_64-linux/paged_attention/_custom_ops.py rename to build/torch28-cxx11-rocm63-x86_64-linux/_custom_ops.py diff --git a/build/torch28-cxx11-rocm63-x86_64-linux/_ops.py b/build/torch28-cxx11-rocm63-x86_64-linux/_ops.py new file mode 100644 index 0000000000000000000000000000000000000000..2bd4e7e759659af7e4cc1eac77bd7d389685c212 --- /dev/null +++ b/build/torch28-cxx11-rocm63-x86_64-linux/_ops.py @@ -0,0 +1,9 @@ +import torch +from . import _paged_attention_61fdee0 +ops = torch.ops._paged_attention_61fdee0 + +def add_op_namespace_prefix(op_name: str): + """ + Prefix op by namespace. + """ + return f"_paged_attention_61fdee0::{op_name}" \ No newline at end of file diff --git a/build/torch28-cxx11-rocm63-x86_64-linux/_paged_attention_61fdee0.abi3.so b/build/torch28-cxx11-rocm63-x86_64-linux/_paged_attention_61fdee0.abi3.so new file mode 100644 index 0000000000000000000000000000000000000000..90d976a09d0b6b4cd4406dd73131fdb78fe1f092 --- /dev/null +++ b/build/torch28-cxx11-rocm63-x86_64-linux/_paged_attention_61fdee0.abi3.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0eefc3ba00e45ffc81f9c6e33ef2664e78c8daf9256367a44de16a8210a28488 +size 120192848 diff --git a/build/torch28-cxx11-rocm63-x86_64-linux/metadata.json b/build/torch28-cxx11-rocm63-x86_64-linux/metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..76bafa5f33b6818aa6bb4cab04be811b87519b44 --- /dev/null +++ b/build/torch28-cxx11-rocm63-x86_64-linux/metadata.json @@ -0,0 +1 @@ +{"python-depends":[]} \ No newline at end of file diff --git a/build/torch28-cxx11-rocm63-x86_64-linux/paged_attention/__init__.py b/build/torch28-cxx11-rocm63-x86_64-linux/paged_attention/__init__.py index 9de56043369487facc1f163df6bd319c9806e5ca..03dbc1afe1cf156661a2b1b22003cd5f599a0309 100644 --- a/build/torch28-cxx11-rocm63-x86_64-linux/paged_attention/__init__.py +++ b/build/torch28-cxx11-rocm63-x86_64-linux/paged_attention/__init__.py @@ -1,21 +1,26 @@ -from ._custom_ops import ( - convert_fp8, - copy_blocks, - paged_attention_v1, - paged_attention_v2, - reshape_and_cache, - reshape_and_cache_flash, - swap_blocks, -) -from ._ops import ops +import ctypes +import sys -__all__ = [ - "convert_fp8", - "copy_blocks", - "ops", - "paged_attention_v1", - "paged_attention_v2", - "reshape_and_cache", - "reshape_and_cache_flash", - "swap_blocks", -] +import importlib +from pathlib import Path +from types import ModuleType + +def _import_from_path(file_path: Path) -> ModuleType: + # We cannot use the module name as-is, after adding it to `sys.modules`, + # it would also be used for other imports. So, we make a module name that + # depends on the path for it to be unique using the hex-encoded hash of + # the path. + path_hash = "{:x}".format(ctypes.c_size_t(hash(file_path.absolute())).value) + module_name = path_hash + spec = importlib.util.spec_from_file_location(module_name, file_path) + if spec is None: + raise ImportError(f"Cannot load spec for {module_name} from {file_path}") + module = importlib.util.module_from_spec(spec) + if module is None: + raise ImportError(f"Cannot load module {module_name} from spec") + sys.modules[module_name] = module + spec.loader.exec_module(module) # type: ignore + return module + + +globals().update(vars(_import_from_path(Path(__file__).parent.parent / "__init__.py"))) diff --git a/build/torch28-cxx11-rocm63-x86_64-linux/paged_attention/_ops.py b/build/torch28-cxx11-rocm63-x86_64-linux/paged_attention/_ops.py deleted file mode 100644 index d09b198f2169310837bcd6b8ea88308879b5775d..0000000000000000000000000000000000000000 --- a/build/torch28-cxx11-rocm63-x86_64-linux/paged_attention/_ops.py +++ /dev/null @@ -1,9 +0,0 @@ -import torch -from . import _paged_attention_287831d -ops = torch.ops._paged_attention_287831d - -def add_op_namespace_prefix(op_name: str): - """ - Prefix op by namespace. - """ - return f"_paged_attention_287831d::{op_name}" \ No newline at end of file diff --git a/build/torch28-cxx11-rocm63-x86_64-linux/paged_attention/_paged_attention_287831d.abi3.so b/build/torch28-cxx11-rocm63-x86_64-linux/paged_attention/_paged_attention_287831d.abi3.so deleted file mode 100644 index 3a63d605d0ac845d3b372edcfb0a2cb3b191c35b..0000000000000000000000000000000000000000 --- a/build/torch28-cxx11-rocm63-x86_64-linux/paged_attention/_paged_attention_287831d.abi3.so +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:52c29c39d300ade26ca567fe96db040df2bed151dd35adf24c74a41469fee54b -size 120179024 diff --git a/build/torch29-cxx11-rocm63-x86_64-linux/paged_attention/platforms.py b/build/torch28-cxx11-rocm63-x86_64-linux/platforms.py similarity index 100% rename from build/torch29-cxx11-rocm63-x86_64-linux/paged_attention/platforms.py rename to build/torch28-cxx11-rocm63-x86_64-linux/platforms.py diff --git a/build/torch28-cxx11-rocm64-x86_64-linux/__init__.py b/build/torch28-cxx11-rocm64-x86_64-linux/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..9de56043369487facc1f163df6bd319c9806e5ca --- /dev/null +++ b/build/torch28-cxx11-rocm64-x86_64-linux/__init__.py @@ -0,0 +1,21 @@ +from ._custom_ops import ( + convert_fp8, + copy_blocks, + paged_attention_v1, + paged_attention_v2, + reshape_and_cache, + reshape_and_cache_flash, + swap_blocks, +) +from ._ops import ops + +__all__ = [ + "convert_fp8", + "copy_blocks", + "ops", + "paged_attention_v1", + "paged_attention_v2", + "reshape_and_cache", + "reshape_and_cache_flash", + "swap_blocks", +] diff --git a/build/torch29-cxx11-rocm64-x86_64-linux/paged_attention/_custom_ops.py b/build/torch28-cxx11-rocm64-x86_64-linux/_custom_ops.py similarity index 100% rename from build/torch29-cxx11-rocm64-x86_64-linux/paged_attention/_custom_ops.py rename to build/torch28-cxx11-rocm64-x86_64-linux/_custom_ops.py diff --git a/build/torch28-cxx11-rocm64-x86_64-linux/_ops.py b/build/torch28-cxx11-rocm64-x86_64-linux/_ops.py new file mode 100644 index 0000000000000000000000000000000000000000..2bd4e7e759659af7e4cc1eac77bd7d389685c212 --- /dev/null +++ b/build/torch28-cxx11-rocm64-x86_64-linux/_ops.py @@ -0,0 +1,9 @@ +import torch +from . import _paged_attention_61fdee0 +ops = torch.ops._paged_attention_61fdee0 + +def add_op_namespace_prefix(op_name: str): + """ + Prefix op by namespace. + """ + return f"_paged_attention_61fdee0::{op_name}" \ No newline at end of file diff --git a/build/torch28-cxx11-rocm64-x86_64-linux/_paged_attention_61fdee0.abi3.so b/build/torch28-cxx11-rocm64-x86_64-linux/_paged_attention_61fdee0.abi3.so new file mode 100644 index 0000000000000000000000000000000000000000..99cf2efeb2aaf47ddb5784ae61d11fdc42deb037 --- /dev/null +++ b/build/torch28-cxx11-rocm64-x86_64-linux/_paged_attention_61fdee0.abi3.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ea189ed02545d6db789499705def489ae3f9a7d743bd3e6d9370edd815c4495b +size 121030544 diff --git a/build/torch28-cxx11-rocm64-x86_64-linux/metadata.json b/build/torch28-cxx11-rocm64-x86_64-linux/metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..76bafa5f33b6818aa6bb4cab04be811b87519b44 --- /dev/null +++ b/build/torch28-cxx11-rocm64-x86_64-linux/metadata.json @@ -0,0 +1 @@ +{"python-depends":[]} \ No newline at end of file diff --git a/build/torch28-cxx11-rocm64-x86_64-linux/paged_attention/__init__.py b/build/torch28-cxx11-rocm64-x86_64-linux/paged_attention/__init__.py index 9de56043369487facc1f163df6bd319c9806e5ca..03dbc1afe1cf156661a2b1b22003cd5f599a0309 100644 --- a/build/torch28-cxx11-rocm64-x86_64-linux/paged_attention/__init__.py +++ b/build/torch28-cxx11-rocm64-x86_64-linux/paged_attention/__init__.py @@ -1,21 +1,26 @@ -from ._custom_ops import ( - convert_fp8, - copy_blocks, - paged_attention_v1, - paged_attention_v2, - reshape_and_cache, - reshape_and_cache_flash, - swap_blocks, -) -from ._ops import ops +import ctypes +import sys -__all__ = [ - "convert_fp8", - "copy_blocks", - "ops", - "paged_attention_v1", - "paged_attention_v2", - "reshape_and_cache", - "reshape_and_cache_flash", - "swap_blocks", -] +import importlib +from pathlib import Path +from types import ModuleType + +def _import_from_path(file_path: Path) -> ModuleType: + # We cannot use the module name as-is, after adding it to `sys.modules`, + # it would also be used for other imports. So, we make a module name that + # depends on the path for it to be unique using the hex-encoded hash of + # the path. + path_hash = "{:x}".format(ctypes.c_size_t(hash(file_path.absolute())).value) + module_name = path_hash + spec = importlib.util.spec_from_file_location(module_name, file_path) + if spec is None: + raise ImportError(f"Cannot load spec for {module_name} from {file_path}") + module = importlib.util.module_from_spec(spec) + if module is None: + raise ImportError(f"Cannot load module {module_name} from spec") + sys.modules[module_name] = module + spec.loader.exec_module(module) # type: ignore + return module + + +globals().update(vars(_import_from_path(Path(__file__).parent.parent / "__init__.py"))) diff --git a/build/torch28-cxx11-rocm64-x86_64-linux/paged_attention/_ops.py b/build/torch28-cxx11-rocm64-x86_64-linux/paged_attention/_ops.py deleted file mode 100644 index d09b198f2169310837bcd6b8ea88308879b5775d..0000000000000000000000000000000000000000 --- a/build/torch28-cxx11-rocm64-x86_64-linux/paged_attention/_ops.py +++ /dev/null @@ -1,9 +0,0 @@ -import torch -from . import _paged_attention_287831d -ops = torch.ops._paged_attention_287831d - -def add_op_namespace_prefix(op_name: str): - """ - Prefix op by namespace. - """ - return f"_paged_attention_287831d::{op_name}" \ No newline at end of file diff --git a/build/torch28-cxx11-rocm64-x86_64-linux/paged_attention/_paged_attention_287831d.abi3.so b/build/torch28-cxx11-rocm64-x86_64-linux/paged_attention/_paged_attention_287831d.abi3.so deleted file mode 100644 index 61362a8dfa894caa1e5cb20049c00f1f7e34d5fd..0000000000000000000000000000000000000000 --- a/build/torch28-cxx11-rocm64-x86_64-linux/paged_attention/_paged_attention_287831d.abi3.so +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:3ef6061f268d5e65216889615ed121e438d29c93ddaffef959b3c66d5219bb55 -size 121012632 diff --git a/build/torch29-cxx11-rocm64-x86_64-linux/paged_attention/platforms.py b/build/torch28-cxx11-rocm64-x86_64-linux/platforms.py similarity index 100% rename from build/torch29-cxx11-rocm64-x86_64-linux/paged_attention/platforms.py rename to build/torch28-cxx11-rocm64-x86_64-linux/platforms.py diff --git a/build/torch29-cxx11-cu126-x86_64-linux/__init__.py b/build/torch29-cxx11-cu126-x86_64-linux/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..9de56043369487facc1f163df6bd319c9806e5ca --- /dev/null +++ b/build/torch29-cxx11-cu126-x86_64-linux/__init__.py @@ -0,0 +1,21 @@ +from ._custom_ops import ( + convert_fp8, + copy_blocks, + paged_attention_v1, + paged_attention_v2, + reshape_and_cache, + reshape_and_cache_flash, + swap_blocks, +) +from ._ops import ops + +__all__ = [ + "convert_fp8", + "copy_blocks", + "ops", + "paged_attention_v1", + "paged_attention_v2", + "reshape_and_cache", + "reshape_and_cache_flash", + "swap_blocks", +] diff --git a/build/torch29-cxx11-cu126-x86_64-linux/_custom_ops.py b/build/torch29-cxx11-cu126-x86_64-linux/_custom_ops.py new file mode 100644 index 0000000000000000000000000000000000000000..a0c0b8db085468dee5100c98d14106a9ee917bf2 --- /dev/null +++ b/build/torch29-cxx11-cu126-x86_64-linux/_custom_ops.py @@ -0,0 +1,173 @@ +from typing import List, Optional + +import torch + +from ._ops import ops + + +# page attention ops +def paged_attention_v1( + out: torch.Tensor, + query: torch.Tensor, + key_cache: torch.Tensor, + value_cache: torch.Tensor, + num_kv_heads: int, + scale: float, + block_tables: torch.Tensor, + seq_lens: torch.Tensor, + block_size: int, + max_seq_len: int, + alibi_slopes: Optional[torch.Tensor], + kv_cache_dtype: str, + k_scale: float, + v_scale: float, + tp_rank: int = 0, + blocksparse_local_blocks: int = 0, + blocksparse_vert_stride: int = 0, + blocksparse_block_size: int = 64, + blocksparse_head_sliding_step: int = 0, +) -> None: + ops.paged_attention_v1( + out, + query, + key_cache, + value_cache, + num_kv_heads, + scale, + block_tables, + seq_lens, + block_size, + max_seq_len, + alibi_slopes, + kv_cache_dtype, + k_scale, + v_scale, + tp_rank, + blocksparse_local_blocks, + blocksparse_vert_stride, + blocksparse_block_size, + blocksparse_head_sliding_step, + ) + + +def paged_attention_v2( + out: torch.Tensor, + exp_sum: torch.Tensor, + max_logits: torch.Tensor, + tmp_out: torch.Tensor, + query: torch.Tensor, + key_cache: torch.Tensor, + value_cache: torch.Tensor, + num_kv_heads: int, + scale: float, + block_tables: torch.Tensor, + seq_lens: torch.Tensor, + block_size: int, + max_seq_len: int, + alibi_slopes: Optional[torch.Tensor], + kv_cache_dtype: str, + k_scale: float, + v_scale: float, + tp_rank: int = 0, + blocksparse_local_blocks: int = 0, + blocksparse_vert_stride: int = 0, + blocksparse_block_size: int = 64, + blocksparse_head_sliding_step: int = 0, +) -> None: + ops.paged_attention_v2( + out, + exp_sum, + max_logits, + tmp_out, + query, + key_cache, + value_cache, + num_kv_heads, + scale, + block_tables, + seq_lens, + block_size, + max_seq_len, + alibi_slopes, + kv_cache_dtype, + k_scale, + v_scale, + tp_rank, + blocksparse_local_blocks, + blocksparse_vert_stride, + blocksparse_block_size, + blocksparse_head_sliding_step, + ) + + +def reshape_and_cache( + key: torch.Tensor, + value: torch.Tensor, + key_cache: torch.Tensor, + value_cache: torch.Tensor, + slot_mapping: torch.Tensor, + kv_cache_dtype: str, + k_scale: float, + v_scale: float, +) -> None: + ops.reshape_and_cache( + key, + value, + key_cache, + value_cache, + slot_mapping, + kv_cache_dtype, + k_scale, + v_scale, + ) + + +def reshape_and_cache_flash( + key: torch.Tensor, + value: torch.Tensor, + key_cache: torch.Tensor, + value_cache: torch.Tensor, + slot_mapping: torch.Tensor, + kv_cache_dtype: str, + k_scale: torch.Tensor, + v_scale: torch.Tensor, +) -> None: + ops.reshape_and_cache_flash( + key, + value, + key_cache, + value_cache, + slot_mapping, + kv_cache_dtype, + k_scale, + v_scale, + ) + + +def copy_blocks( + key_caches: List[torch.Tensor], + value_caches: List[torch.Tensor], + block_mapping: torch.Tensor, +) -> None: + ops.copy_blocks(key_caches, value_caches, block_mapping) + + +def swap_blocks( + src: torch.Tensor, dst: torch.Tensor, block_mapping: torch.Tensor +) -> None: + ops.swap_blocks(src, dst, block_mapping) + + +def convert_fp8( + output: torch.Tensor, input: torch.Tensor, scale: float = 1.0, kv_dtype: str = "fp8" +) -> None: + ops.convert_fp8(output, input, scale, kv_dtype) + + +__all__ = [ + "convert_fp8", + "paged_attention_v1", + "paged_attention_v2", + "reshape_and_cache", + "copy_blocks", +] diff --git a/build/torch29-cxx11-cu126-x86_64-linux/_ops.py b/build/torch29-cxx11-cu126-x86_64-linux/_ops.py new file mode 100644 index 0000000000000000000000000000000000000000..2bd4e7e759659af7e4cc1eac77bd7d389685c212 --- /dev/null +++ b/build/torch29-cxx11-cu126-x86_64-linux/_ops.py @@ -0,0 +1,9 @@ +import torch +from . import _paged_attention_61fdee0 +ops = torch.ops._paged_attention_61fdee0 + +def add_op_namespace_prefix(op_name: str): + """ + Prefix op by namespace. + """ + return f"_paged_attention_61fdee0::{op_name}" \ No newline at end of file diff --git a/build/torch29-cxx11-cu126-x86_64-linux/_paged_attention_61fdee0.abi3.so b/build/torch29-cxx11-cu126-x86_64-linux/_paged_attention_61fdee0.abi3.so new file mode 100644 index 0000000000000000000000000000000000000000..28a0a658747e9b163fec25f424e7ab6bebd4c1e9 --- /dev/null +++ b/build/torch29-cxx11-cu126-x86_64-linux/_paged_attention_61fdee0.abi3.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:26b70838282cecf41b4f44961b1b14b0198b6b7c069fa443adf0afda19c2f6a1 +size 140143480 diff --git a/build/torch29-cxx11-cu126-x86_64-linux/metadata.json b/build/torch29-cxx11-cu126-x86_64-linux/metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..76bafa5f33b6818aa6bb4cab04be811b87519b44 --- /dev/null +++ b/build/torch29-cxx11-cu126-x86_64-linux/metadata.json @@ -0,0 +1 @@ +{"python-depends":[]} \ No newline at end of file diff --git a/build/torch29-cxx11-cu126-x86_64-linux/paged_attention/__init__.py b/build/torch29-cxx11-cu126-x86_64-linux/paged_attention/__init__.py index 9de56043369487facc1f163df6bd319c9806e5ca..03dbc1afe1cf156661a2b1b22003cd5f599a0309 100644 --- a/build/torch29-cxx11-cu126-x86_64-linux/paged_attention/__init__.py +++ b/build/torch29-cxx11-cu126-x86_64-linux/paged_attention/__init__.py @@ -1,21 +1,26 @@ -from ._custom_ops import ( - convert_fp8, - copy_blocks, - paged_attention_v1, - paged_attention_v2, - reshape_and_cache, - reshape_and_cache_flash, - swap_blocks, -) -from ._ops import ops +import ctypes +import sys -__all__ = [ - "convert_fp8", - "copy_blocks", - "ops", - "paged_attention_v1", - "paged_attention_v2", - "reshape_and_cache", - "reshape_and_cache_flash", - "swap_blocks", -] +import importlib +from pathlib import Path +from types import ModuleType + +def _import_from_path(file_path: Path) -> ModuleType: + # We cannot use the module name as-is, after adding it to `sys.modules`, + # it would also be used for other imports. So, we make a module name that + # depends on the path for it to be unique using the hex-encoded hash of + # the path. + path_hash = "{:x}".format(ctypes.c_size_t(hash(file_path.absolute())).value) + module_name = path_hash + spec = importlib.util.spec_from_file_location(module_name, file_path) + if spec is None: + raise ImportError(f"Cannot load spec for {module_name} from {file_path}") + module = importlib.util.module_from_spec(spec) + if module is None: + raise ImportError(f"Cannot load module {module_name} from spec") + sys.modules[module_name] = module + spec.loader.exec_module(module) # type: ignore + return module + + +globals().update(vars(_import_from_path(Path(__file__).parent.parent / "__init__.py"))) diff --git a/build/torch29-cxx11-cu126-x86_64-linux/paged_attention/_ops.py b/build/torch29-cxx11-cu126-x86_64-linux/paged_attention/_ops.py deleted file mode 100644 index d09b198f2169310837bcd6b8ea88308879b5775d..0000000000000000000000000000000000000000 --- a/build/torch29-cxx11-cu126-x86_64-linux/paged_attention/_ops.py +++ /dev/null @@ -1,9 +0,0 @@ -import torch -from . import _paged_attention_287831d -ops = torch.ops._paged_attention_287831d - -def add_op_namespace_prefix(op_name: str): - """ - Prefix op by namespace. - """ - return f"_paged_attention_287831d::{op_name}" \ No newline at end of file diff --git a/build/torch29-cxx11-cu126-x86_64-linux/paged_attention/_paged_attention_287831d.abi3.so b/build/torch29-cxx11-cu126-x86_64-linux/paged_attention/_paged_attention_287831d.abi3.so deleted file mode 100644 index 5b43244a5d8468200333911b0ea4708dd5c35c38..0000000000000000000000000000000000000000 --- a/build/torch29-cxx11-cu126-x86_64-linux/paged_attention/_paged_attention_287831d.abi3.so +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:cf84af5418afac998537d6c01c88ad3bd083fa1912a1c90ae70809196cb0cc45 -size 85751704 diff --git a/build/torch29-cxx11-cu126-x86_64-linux/platforms.py b/build/torch29-cxx11-cu126-x86_64-linux/platforms.py new file mode 100644 index 0000000000000000000000000000000000000000..6277d5f50ff3ddc265bb39fa1c4d17e0341b7767 --- /dev/null +++ b/build/torch29-cxx11-cu126-x86_64-linux/platforms.py @@ -0,0 +1,92 @@ +import os +import random +from abc import ABC, abstractmethod +from functools import lru_cache, wraps +from typing import Callable, ParamSpec, TypeVar + +import numpy as np +import torch + +IS_ROCM = torch.version.hip is not None +IS_MPS = torch.backends.mps.is_available() + + +class Platform(ABC): + @classmethod + def seed_everything(cls, seed: int) -> None: + """ + Set the seed of each random module. + `torch.manual_seed` will set seed on all devices. + + Loosely based on: https://github.com/Lightning-AI/pytorch-lightning/blob/2.4.0/src/lightning/fabric/utilities/seed.py#L20 + """ + random.seed(seed) + np.random.seed(seed) + torch.manual_seed(seed) + + @abstractmethod + def get_device_name(self, device_id: int = 0) -> str: ... + + @abstractmethod + def is_cuda(self) -> bool: ... + + @abstractmethod + def is_rocm(self) -> bool: ... + + @abstractmethod + def is_mps(self) -> bool: ... + + +class CudaPlatform(Platform): + @classmethod + @lru_cache(maxsize=8) + def get_device_name(cls, device_id: int = 0) -> str: + return torch.cuda.get_device_name(0) + + def is_cuda(self) -> bool: + return True + + def is_rocm(self) -> bool: + return False + + def is_mps(self) -> bool: + return False + + +class RocmPlatform(Platform): + @classmethod + @lru_cache(maxsize=8) + def get_device_name(cls, device_id: int = 0) -> str: + return torch.cuda.get_device_name(device_id) + + def is_cuda(self) -> bool: + return False + + def is_rocm(self) -> bool: + return True + + def is_mps(self) -> bool: + return False + + +class MpsPlatform(Platform): + @classmethod + @lru_cache(maxsize=8) + def get_device_name(cls, device_id: int = 0) -> str: + return torch.cuda.get_device_name(device_id) + + def is_cuda(self) -> bool: + return False + + def is_rocm(self) -> bool: + return False + + def is_mps(self) -> bool: + return True + +current_platform = ( + RocmPlatform() if IS_ROCM else + MpsPlatform() if IS_MPS else + CudaPlatform() if torch.cuda.is_available() else + None +) diff --git a/build/torch29-cxx11-cu128-x86_64-linux/__init__.py b/build/torch29-cxx11-cu128-x86_64-linux/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..9de56043369487facc1f163df6bd319c9806e5ca --- /dev/null +++ b/build/torch29-cxx11-cu128-x86_64-linux/__init__.py @@ -0,0 +1,21 @@ +from ._custom_ops import ( + convert_fp8, + copy_blocks, + paged_attention_v1, + paged_attention_v2, + reshape_and_cache, + reshape_and_cache_flash, + swap_blocks, +) +from ._ops import ops + +__all__ = [ + "convert_fp8", + "copy_blocks", + "ops", + "paged_attention_v1", + "paged_attention_v2", + "reshape_and_cache", + "reshape_and_cache_flash", + "swap_blocks", +] diff --git a/build/torch29-cxx11-cu128-x86_64-linux/_custom_ops.py b/build/torch29-cxx11-cu128-x86_64-linux/_custom_ops.py new file mode 100644 index 0000000000000000000000000000000000000000..a0c0b8db085468dee5100c98d14106a9ee917bf2 --- /dev/null +++ b/build/torch29-cxx11-cu128-x86_64-linux/_custom_ops.py @@ -0,0 +1,173 @@ +from typing import List, Optional + +import torch + +from ._ops import ops + + +# page attention ops +def paged_attention_v1( + out: torch.Tensor, + query: torch.Tensor, + key_cache: torch.Tensor, + value_cache: torch.Tensor, + num_kv_heads: int, + scale: float, + block_tables: torch.Tensor, + seq_lens: torch.Tensor, + block_size: int, + max_seq_len: int, + alibi_slopes: Optional[torch.Tensor], + kv_cache_dtype: str, + k_scale: float, + v_scale: float, + tp_rank: int = 0, + blocksparse_local_blocks: int = 0, + blocksparse_vert_stride: int = 0, + blocksparse_block_size: int = 64, + blocksparse_head_sliding_step: int = 0, +) -> None: + ops.paged_attention_v1( + out, + query, + key_cache, + value_cache, + num_kv_heads, + scale, + block_tables, + seq_lens, + block_size, + max_seq_len, + alibi_slopes, + kv_cache_dtype, + k_scale, + v_scale, + tp_rank, + blocksparse_local_blocks, + blocksparse_vert_stride, + blocksparse_block_size, + blocksparse_head_sliding_step, + ) + + +def paged_attention_v2( + out: torch.Tensor, + exp_sum: torch.Tensor, + max_logits: torch.Tensor, + tmp_out: torch.Tensor, + query: torch.Tensor, + key_cache: torch.Tensor, + value_cache: torch.Tensor, + num_kv_heads: int, + scale: float, + block_tables: torch.Tensor, + seq_lens: torch.Tensor, + block_size: int, + max_seq_len: int, + alibi_slopes: Optional[torch.Tensor], + kv_cache_dtype: str, + k_scale: float, + v_scale: float, + tp_rank: int = 0, + blocksparse_local_blocks: int = 0, + blocksparse_vert_stride: int = 0, + blocksparse_block_size: int = 64, + blocksparse_head_sliding_step: int = 0, +) -> None: + ops.paged_attention_v2( + out, + exp_sum, + max_logits, + tmp_out, + query, + key_cache, + value_cache, + num_kv_heads, + scale, + block_tables, + seq_lens, + block_size, + max_seq_len, + alibi_slopes, + kv_cache_dtype, + k_scale, + v_scale, + tp_rank, + blocksparse_local_blocks, + blocksparse_vert_stride, + blocksparse_block_size, + blocksparse_head_sliding_step, + ) + + +def reshape_and_cache( + key: torch.Tensor, + value: torch.Tensor, + key_cache: torch.Tensor, + value_cache: torch.Tensor, + slot_mapping: torch.Tensor, + kv_cache_dtype: str, + k_scale: float, + v_scale: float, +) -> None: + ops.reshape_and_cache( + key, + value, + key_cache, + value_cache, + slot_mapping, + kv_cache_dtype, + k_scale, + v_scale, + ) + + +def reshape_and_cache_flash( + key: torch.Tensor, + value: torch.Tensor, + key_cache: torch.Tensor, + value_cache: torch.Tensor, + slot_mapping: torch.Tensor, + kv_cache_dtype: str, + k_scale: torch.Tensor, + v_scale: torch.Tensor, +) -> None: + ops.reshape_and_cache_flash( + key, + value, + key_cache, + value_cache, + slot_mapping, + kv_cache_dtype, + k_scale, + v_scale, + ) + + +def copy_blocks( + key_caches: List[torch.Tensor], + value_caches: List[torch.Tensor], + block_mapping: torch.Tensor, +) -> None: + ops.copy_blocks(key_caches, value_caches, block_mapping) + + +def swap_blocks( + src: torch.Tensor, dst: torch.Tensor, block_mapping: torch.Tensor +) -> None: + ops.swap_blocks(src, dst, block_mapping) + + +def convert_fp8( + output: torch.Tensor, input: torch.Tensor, scale: float = 1.0, kv_dtype: str = "fp8" +) -> None: + ops.convert_fp8(output, input, scale, kv_dtype) + + +__all__ = [ + "convert_fp8", + "paged_attention_v1", + "paged_attention_v2", + "reshape_and_cache", + "copy_blocks", +] diff --git a/build/torch29-cxx11-cu128-x86_64-linux/_ops.py b/build/torch29-cxx11-cu128-x86_64-linux/_ops.py new file mode 100644 index 0000000000000000000000000000000000000000..2bd4e7e759659af7e4cc1eac77bd7d389685c212 --- /dev/null +++ b/build/torch29-cxx11-cu128-x86_64-linux/_ops.py @@ -0,0 +1,9 @@ +import torch +from . import _paged_attention_61fdee0 +ops = torch.ops._paged_attention_61fdee0 + +def add_op_namespace_prefix(op_name: str): + """ + Prefix op by namespace. + """ + return f"_paged_attention_61fdee0::{op_name}" \ No newline at end of file diff --git a/build/torch29-cxx11-cu128-x86_64-linux/_paged_attention_61fdee0.abi3.so b/build/torch29-cxx11-cu128-x86_64-linux/_paged_attention_61fdee0.abi3.so new file mode 100644 index 0000000000000000000000000000000000000000..9c0c2c30784913e0078b51bf5817b4cbb466b474 --- /dev/null +++ b/build/torch29-cxx11-cu128-x86_64-linux/_paged_attention_61fdee0.abi3.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:48596eca3f19db876f2bc877069621745b5d587d018585c2c5be482dee5dd9ea +size 167710744 diff --git a/build/torch29-cxx11-cu128-x86_64-linux/metadata.json b/build/torch29-cxx11-cu128-x86_64-linux/metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..76bafa5f33b6818aa6bb4cab04be811b87519b44 --- /dev/null +++ b/build/torch29-cxx11-cu128-x86_64-linux/metadata.json @@ -0,0 +1 @@ +{"python-depends":[]} \ No newline at end of file diff --git a/build/torch29-cxx11-cu128-x86_64-linux/paged_attention/__init__.py b/build/torch29-cxx11-cu128-x86_64-linux/paged_attention/__init__.py index 9de56043369487facc1f163df6bd319c9806e5ca..03dbc1afe1cf156661a2b1b22003cd5f599a0309 100644 --- a/build/torch29-cxx11-cu128-x86_64-linux/paged_attention/__init__.py +++ b/build/torch29-cxx11-cu128-x86_64-linux/paged_attention/__init__.py @@ -1,21 +1,26 @@ -from ._custom_ops import ( - convert_fp8, - copy_blocks, - paged_attention_v1, - paged_attention_v2, - reshape_and_cache, - reshape_and_cache_flash, - swap_blocks, -) -from ._ops import ops +import ctypes +import sys -__all__ = [ - "convert_fp8", - "copy_blocks", - "ops", - "paged_attention_v1", - "paged_attention_v2", - "reshape_and_cache", - "reshape_and_cache_flash", - "swap_blocks", -] +import importlib +from pathlib import Path +from types import ModuleType + +def _import_from_path(file_path: Path) -> ModuleType: + # We cannot use the module name as-is, after adding it to `sys.modules`, + # it would also be used for other imports. So, we make a module name that + # depends on the path for it to be unique using the hex-encoded hash of + # the path. + path_hash = "{:x}".format(ctypes.c_size_t(hash(file_path.absolute())).value) + module_name = path_hash + spec = importlib.util.spec_from_file_location(module_name, file_path) + if spec is None: + raise ImportError(f"Cannot load spec for {module_name} from {file_path}") + module = importlib.util.module_from_spec(spec) + if module is None: + raise ImportError(f"Cannot load module {module_name} from spec") + sys.modules[module_name] = module + spec.loader.exec_module(module) # type: ignore + return module + + +globals().update(vars(_import_from_path(Path(__file__).parent.parent / "__init__.py"))) diff --git a/build/torch29-cxx11-cu128-x86_64-linux/paged_attention/_ops.py b/build/torch29-cxx11-cu128-x86_64-linux/paged_attention/_ops.py deleted file mode 100644 index d09b198f2169310837bcd6b8ea88308879b5775d..0000000000000000000000000000000000000000 --- a/build/torch29-cxx11-cu128-x86_64-linux/paged_attention/_ops.py +++ /dev/null @@ -1,9 +0,0 @@ -import torch -from . import _paged_attention_287831d -ops = torch.ops._paged_attention_287831d - -def add_op_namespace_prefix(op_name: str): - """ - Prefix op by namespace. - """ - return f"_paged_attention_287831d::{op_name}" \ No newline at end of file diff --git a/build/torch29-cxx11-cu128-x86_64-linux/paged_attention/_paged_attention_287831d.abi3.so b/build/torch29-cxx11-cu128-x86_64-linux/paged_attention/_paged_attention_287831d.abi3.so deleted file mode 100644 index 7c05d6141ccb9dc15ec6eff9eaf2b022672c4f92..0000000000000000000000000000000000000000 --- a/build/torch29-cxx11-cu128-x86_64-linux/paged_attention/_paged_attention_287831d.abi3.so +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:39a0b3ae487a77fd16fae849a19f4c542a7ca8b8d423a64e32692e77b036eee6 -size 102693928 diff --git a/build/torch29-cxx11-cu128-x86_64-linux/platforms.py b/build/torch29-cxx11-cu128-x86_64-linux/platforms.py new file mode 100644 index 0000000000000000000000000000000000000000..6277d5f50ff3ddc265bb39fa1c4d17e0341b7767 --- /dev/null +++ b/build/torch29-cxx11-cu128-x86_64-linux/platforms.py @@ -0,0 +1,92 @@ +import os +import random +from abc import ABC, abstractmethod +from functools import lru_cache, wraps +from typing import Callable, ParamSpec, TypeVar + +import numpy as np +import torch + +IS_ROCM = torch.version.hip is not None +IS_MPS = torch.backends.mps.is_available() + + +class Platform(ABC): + @classmethod + def seed_everything(cls, seed: int) -> None: + """ + Set the seed of each random module. + `torch.manual_seed` will set seed on all devices. + + Loosely based on: https://github.com/Lightning-AI/pytorch-lightning/blob/2.4.0/src/lightning/fabric/utilities/seed.py#L20 + """ + random.seed(seed) + np.random.seed(seed) + torch.manual_seed(seed) + + @abstractmethod + def get_device_name(self, device_id: int = 0) -> str: ... + + @abstractmethod + def is_cuda(self) -> bool: ... + + @abstractmethod + def is_rocm(self) -> bool: ... + + @abstractmethod + def is_mps(self) -> bool: ... + + +class CudaPlatform(Platform): + @classmethod + @lru_cache(maxsize=8) + def get_device_name(cls, device_id: int = 0) -> str: + return torch.cuda.get_device_name(0) + + def is_cuda(self) -> bool: + return True + + def is_rocm(self) -> bool: + return False + + def is_mps(self) -> bool: + return False + + +class RocmPlatform(Platform): + @classmethod + @lru_cache(maxsize=8) + def get_device_name(cls, device_id: int = 0) -> str: + return torch.cuda.get_device_name(device_id) + + def is_cuda(self) -> bool: + return False + + def is_rocm(self) -> bool: + return True + + def is_mps(self) -> bool: + return False + + +class MpsPlatform(Platform): + @classmethod + @lru_cache(maxsize=8) + def get_device_name(cls, device_id: int = 0) -> str: + return torch.cuda.get_device_name(device_id) + + def is_cuda(self) -> bool: + return False + + def is_rocm(self) -> bool: + return False + + def is_mps(self) -> bool: + return True + +current_platform = ( + RocmPlatform() if IS_ROCM else + MpsPlatform() if IS_MPS else + CudaPlatform() if torch.cuda.is_available() else + None +) diff --git a/build/torch29-cxx11-cu130-x86_64-linux/__init__.py b/build/torch29-cxx11-cu130-x86_64-linux/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..9de56043369487facc1f163df6bd319c9806e5ca --- /dev/null +++ b/build/torch29-cxx11-cu130-x86_64-linux/__init__.py @@ -0,0 +1,21 @@ +from ._custom_ops import ( + convert_fp8, + copy_blocks, + paged_attention_v1, + paged_attention_v2, + reshape_and_cache, + reshape_and_cache_flash, + swap_blocks, +) +from ._ops import ops + +__all__ = [ + "convert_fp8", + "copy_blocks", + "ops", + "paged_attention_v1", + "paged_attention_v2", + "reshape_and_cache", + "reshape_and_cache_flash", + "swap_blocks", +] diff --git a/build/torch29-cxx11-cu130-x86_64-linux/_custom_ops.py b/build/torch29-cxx11-cu130-x86_64-linux/_custom_ops.py new file mode 100644 index 0000000000000000000000000000000000000000..a0c0b8db085468dee5100c98d14106a9ee917bf2 --- /dev/null +++ b/build/torch29-cxx11-cu130-x86_64-linux/_custom_ops.py @@ -0,0 +1,173 @@ +from typing import List, Optional + +import torch + +from ._ops import ops + + +# page attention ops +def paged_attention_v1( + out: torch.Tensor, + query: torch.Tensor, + key_cache: torch.Tensor, + value_cache: torch.Tensor, + num_kv_heads: int, + scale: float, + block_tables: torch.Tensor, + seq_lens: torch.Tensor, + block_size: int, + max_seq_len: int, + alibi_slopes: Optional[torch.Tensor], + kv_cache_dtype: str, + k_scale: float, + v_scale: float, + tp_rank: int = 0, + blocksparse_local_blocks: int = 0, + blocksparse_vert_stride: int = 0, + blocksparse_block_size: int = 64, + blocksparse_head_sliding_step: int = 0, +) -> None: + ops.paged_attention_v1( + out, + query, + key_cache, + value_cache, + num_kv_heads, + scale, + block_tables, + seq_lens, + block_size, + max_seq_len, + alibi_slopes, + kv_cache_dtype, + k_scale, + v_scale, + tp_rank, + blocksparse_local_blocks, + blocksparse_vert_stride, + blocksparse_block_size, + blocksparse_head_sliding_step, + ) + + +def paged_attention_v2( + out: torch.Tensor, + exp_sum: torch.Tensor, + max_logits: torch.Tensor, + tmp_out: torch.Tensor, + query: torch.Tensor, + key_cache: torch.Tensor, + value_cache: torch.Tensor, + num_kv_heads: int, + scale: float, + block_tables: torch.Tensor, + seq_lens: torch.Tensor, + block_size: int, + max_seq_len: int, + alibi_slopes: Optional[torch.Tensor], + kv_cache_dtype: str, + k_scale: float, + v_scale: float, + tp_rank: int = 0, + blocksparse_local_blocks: int = 0, + blocksparse_vert_stride: int = 0, + blocksparse_block_size: int = 64, + blocksparse_head_sliding_step: int = 0, +) -> None: + ops.paged_attention_v2( + out, + exp_sum, + max_logits, + tmp_out, + query, + key_cache, + value_cache, + num_kv_heads, + scale, + block_tables, + seq_lens, + block_size, + max_seq_len, + alibi_slopes, + kv_cache_dtype, + k_scale, + v_scale, + tp_rank, + blocksparse_local_blocks, + blocksparse_vert_stride, + blocksparse_block_size, + blocksparse_head_sliding_step, + ) + + +def reshape_and_cache( + key: torch.Tensor, + value: torch.Tensor, + key_cache: torch.Tensor, + value_cache: torch.Tensor, + slot_mapping: torch.Tensor, + kv_cache_dtype: str, + k_scale: float, + v_scale: float, +) -> None: + ops.reshape_and_cache( + key, + value, + key_cache, + value_cache, + slot_mapping, + kv_cache_dtype, + k_scale, + v_scale, + ) + + +def reshape_and_cache_flash( + key: torch.Tensor, + value: torch.Tensor, + key_cache: torch.Tensor, + value_cache: torch.Tensor, + slot_mapping: torch.Tensor, + kv_cache_dtype: str, + k_scale: torch.Tensor, + v_scale: torch.Tensor, +) -> None: + ops.reshape_and_cache_flash( + key, + value, + key_cache, + value_cache, + slot_mapping, + kv_cache_dtype, + k_scale, + v_scale, + ) + + +def copy_blocks( + key_caches: List[torch.Tensor], + value_caches: List[torch.Tensor], + block_mapping: torch.Tensor, +) -> None: + ops.copy_blocks(key_caches, value_caches, block_mapping) + + +def swap_blocks( + src: torch.Tensor, dst: torch.Tensor, block_mapping: torch.Tensor +) -> None: + ops.swap_blocks(src, dst, block_mapping) + + +def convert_fp8( + output: torch.Tensor, input: torch.Tensor, scale: float = 1.0, kv_dtype: str = "fp8" +) -> None: + ops.convert_fp8(output, input, scale, kv_dtype) + + +__all__ = [ + "convert_fp8", + "paged_attention_v1", + "paged_attention_v2", + "reshape_and_cache", + "copy_blocks", +] diff --git a/build/torch29-cxx11-cu130-x86_64-linux/_ops.py b/build/torch29-cxx11-cu130-x86_64-linux/_ops.py new file mode 100644 index 0000000000000000000000000000000000000000..2bd4e7e759659af7e4cc1eac77bd7d389685c212 --- /dev/null +++ b/build/torch29-cxx11-cu130-x86_64-linux/_ops.py @@ -0,0 +1,9 @@ +import torch +from . import _paged_attention_61fdee0 +ops = torch.ops._paged_attention_61fdee0 + +def add_op_namespace_prefix(op_name: str): + """ + Prefix op by namespace. + """ + return f"_paged_attention_61fdee0::{op_name}" \ No newline at end of file diff --git a/build/torch29-cxx11-cu130-x86_64-linux/_paged_attention_61fdee0.abi3.so b/build/torch29-cxx11-cu130-x86_64-linux/_paged_attention_61fdee0.abi3.so new file mode 100644 index 0000000000000000000000000000000000000000..9acb324c71fae6ac0520b1d6d484a1ecba2edcf4 --- /dev/null +++ b/build/torch29-cxx11-cu130-x86_64-linux/_paged_attention_61fdee0.abi3.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:30fb532980def29df5a671c493d3578586098e136c46cdd89c83120d24997433 +size 86544192 diff --git a/build/torch29-cxx11-cu130-x86_64-linux/metadata.json b/build/torch29-cxx11-cu130-x86_64-linux/metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..76bafa5f33b6818aa6bb4cab04be811b87519b44 --- /dev/null +++ b/build/torch29-cxx11-cu130-x86_64-linux/metadata.json @@ -0,0 +1 @@ +{"python-depends":[]} \ No newline at end of file diff --git a/build/torch29-cxx11-cu130-x86_64-linux/paged_attention/__init__.py b/build/torch29-cxx11-cu130-x86_64-linux/paged_attention/__init__.py index 9de56043369487facc1f163df6bd319c9806e5ca..03dbc1afe1cf156661a2b1b22003cd5f599a0309 100644 --- a/build/torch29-cxx11-cu130-x86_64-linux/paged_attention/__init__.py +++ b/build/torch29-cxx11-cu130-x86_64-linux/paged_attention/__init__.py @@ -1,21 +1,26 @@ -from ._custom_ops import ( - convert_fp8, - copy_blocks, - paged_attention_v1, - paged_attention_v2, - reshape_and_cache, - reshape_and_cache_flash, - swap_blocks, -) -from ._ops import ops +import ctypes +import sys -__all__ = [ - "convert_fp8", - "copy_blocks", - "ops", - "paged_attention_v1", - "paged_attention_v2", - "reshape_and_cache", - "reshape_and_cache_flash", - "swap_blocks", -] +import importlib +from pathlib import Path +from types import ModuleType + +def _import_from_path(file_path: Path) -> ModuleType: + # We cannot use the module name as-is, after adding it to `sys.modules`, + # it would also be used for other imports. So, we make a module name that + # depends on the path for it to be unique using the hex-encoded hash of + # the path. + path_hash = "{:x}".format(ctypes.c_size_t(hash(file_path.absolute())).value) + module_name = path_hash + spec = importlib.util.spec_from_file_location(module_name, file_path) + if spec is None: + raise ImportError(f"Cannot load spec for {module_name} from {file_path}") + module = importlib.util.module_from_spec(spec) + if module is None: + raise ImportError(f"Cannot load module {module_name} from spec") + sys.modules[module_name] = module + spec.loader.exec_module(module) # type: ignore + return module + + +globals().update(vars(_import_from_path(Path(__file__).parent.parent / "__init__.py"))) diff --git a/build/torch29-cxx11-cu130-x86_64-linux/paged_attention/_ops.py b/build/torch29-cxx11-cu130-x86_64-linux/paged_attention/_ops.py deleted file mode 100644 index d09b198f2169310837bcd6b8ea88308879b5775d..0000000000000000000000000000000000000000 --- a/build/torch29-cxx11-cu130-x86_64-linux/paged_attention/_ops.py +++ /dev/null @@ -1,9 +0,0 @@ -import torch -from . import _paged_attention_287831d -ops = torch.ops._paged_attention_287831d - -def add_op_namespace_prefix(op_name: str): - """ - Prefix op by namespace. - """ - return f"_paged_attention_287831d::{op_name}" \ No newline at end of file diff --git a/build/torch29-cxx11-cu130-x86_64-linux/paged_attention/_paged_attention_287831d.abi3.so b/build/torch29-cxx11-cu130-x86_64-linux/paged_attention/_paged_attention_287831d.abi3.so deleted file mode 100644 index 9c026ee568dcd5109945fe7a8eddceeba1a15275..0000000000000000000000000000000000000000 --- a/build/torch29-cxx11-cu130-x86_64-linux/paged_attention/_paged_attention_287831d.abi3.so +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:231953b324c6cfd0847c9322d311e7096af3909ced06344b6a5485baed936767 -size 63028488 diff --git a/build/torch29-cxx11-cu130-x86_64-linux/platforms.py b/build/torch29-cxx11-cu130-x86_64-linux/platforms.py new file mode 100644 index 0000000000000000000000000000000000000000..6277d5f50ff3ddc265bb39fa1c4d17e0341b7767 --- /dev/null +++ b/build/torch29-cxx11-cu130-x86_64-linux/platforms.py @@ -0,0 +1,92 @@ +import os +import random +from abc import ABC, abstractmethod +from functools import lru_cache, wraps +from typing import Callable, ParamSpec, TypeVar + +import numpy as np +import torch + +IS_ROCM = torch.version.hip is not None +IS_MPS = torch.backends.mps.is_available() + + +class Platform(ABC): + @classmethod + def seed_everything(cls, seed: int) -> None: + """ + Set the seed of each random module. + `torch.manual_seed` will set seed on all devices. + + Loosely based on: https://github.com/Lightning-AI/pytorch-lightning/blob/2.4.0/src/lightning/fabric/utilities/seed.py#L20 + """ + random.seed(seed) + np.random.seed(seed) + torch.manual_seed(seed) + + @abstractmethod + def get_device_name(self, device_id: int = 0) -> str: ... + + @abstractmethod + def is_cuda(self) -> bool: ... + + @abstractmethod + def is_rocm(self) -> bool: ... + + @abstractmethod + def is_mps(self) -> bool: ... + + +class CudaPlatform(Platform): + @classmethod + @lru_cache(maxsize=8) + def get_device_name(cls, device_id: int = 0) -> str: + return torch.cuda.get_device_name(0) + + def is_cuda(self) -> bool: + return True + + def is_rocm(self) -> bool: + return False + + def is_mps(self) -> bool: + return False + + +class RocmPlatform(Platform): + @classmethod + @lru_cache(maxsize=8) + def get_device_name(cls, device_id: int = 0) -> str: + return torch.cuda.get_device_name(device_id) + + def is_cuda(self) -> bool: + return False + + def is_rocm(self) -> bool: + return True + + def is_mps(self) -> bool: + return False + + +class MpsPlatform(Platform): + @classmethod + @lru_cache(maxsize=8) + def get_device_name(cls, device_id: int = 0) -> str: + return torch.cuda.get_device_name(device_id) + + def is_cuda(self) -> bool: + return False + + def is_rocm(self) -> bool: + return False + + def is_mps(self) -> bool: + return True + +current_platform = ( + RocmPlatform() if IS_ROCM else + MpsPlatform() if IS_MPS else + CudaPlatform() if torch.cuda.is_available() else + None +) diff --git a/build/torch29-cxx11-rocm63-x86_64-linux/__init__.py b/build/torch29-cxx11-rocm63-x86_64-linux/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..9de56043369487facc1f163df6bd319c9806e5ca --- /dev/null +++ b/build/torch29-cxx11-rocm63-x86_64-linux/__init__.py @@ -0,0 +1,21 @@ +from ._custom_ops import ( + convert_fp8, + copy_blocks, + paged_attention_v1, + paged_attention_v2, + reshape_and_cache, + reshape_and_cache_flash, + swap_blocks, +) +from ._ops import ops + +__all__ = [ + "convert_fp8", + "copy_blocks", + "ops", + "paged_attention_v1", + "paged_attention_v2", + "reshape_and_cache", + "reshape_and_cache_flash", + "swap_blocks", +] diff --git a/build/torch29-cxx11-rocm63-x86_64-linux/_custom_ops.py b/build/torch29-cxx11-rocm63-x86_64-linux/_custom_ops.py new file mode 100644 index 0000000000000000000000000000000000000000..a0c0b8db085468dee5100c98d14106a9ee917bf2 --- /dev/null +++ b/build/torch29-cxx11-rocm63-x86_64-linux/_custom_ops.py @@ -0,0 +1,173 @@ +from typing import List, Optional + +import torch + +from ._ops import ops + + +# page attention ops +def paged_attention_v1( + out: torch.Tensor, + query: torch.Tensor, + key_cache: torch.Tensor, + value_cache: torch.Tensor, + num_kv_heads: int, + scale: float, + block_tables: torch.Tensor, + seq_lens: torch.Tensor, + block_size: int, + max_seq_len: int, + alibi_slopes: Optional[torch.Tensor], + kv_cache_dtype: str, + k_scale: float, + v_scale: float, + tp_rank: int = 0, + blocksparse_local_blocks: int = 0, + blocksparse_vert_stride: int = 0, + blocksparse_block_size: int = 64, + blocksparse_head_sliding_step: int = 0, +) -> None: + ops.paged_attention_v1( + out, + query, + key_cache, + value_cache, + num_kv_heads, + scale, + block_tables, + seq_lens, + block_size, + max_seq_len, + alibi_slopes, + kv_cache_dtype, + k_scale, + v_scale, + tp_rank, + blocksparse_local_blocks, + blocksparse_vert_stride, + blocksparse_block_size, + blocksparse_head_sliding_step, + ) + + +def paged_attention_v2( + out: torch.Tensor, + exp_sum: torch.Tensor, + max_logits: torch.Tensor, + tmp_out: torch.Tensor, + query: torch.Tensor, + key_cache: torch.Tensor, + value_cache: torch.Tensor, + num_kv_heads: int, + scale: float, + block_tables: torch.Tensor, + seq_lens: torch.Tensor, + block_size: int, + max_seq_len: int, + alibi_slopes: Optional[torch.Tensor], + kv_cache_dtype: str, + k_scale: float, + v_scale: float, + tp_rank: int = 0, + blocksparse_local_blocks: int = 0, + blocksparse_vert_stride: int = 0, + blocksparse_block_size: int = 64, + blocksparse_head_sliding_step: int = 0, +) -> None: + ops.paged_attention_v2( + out, + exp_sum, + max_logits, + tmp_out, + query, + key_cache, + value_cache, + num_kv_heads, + scale, + block_tables, + seq_lens, + block_size, + max_seq_len, + alibi_slopes, + kv_cache_dtype, + k_scale, + v_scale, + tp_rank, + blocksparse_local_blocks, + blocksparse_vert_stride, + blocksparse_block_size, + blocksparse_head_sliding_step, + ) + + +def reshape_and_cache( + key: torch.Tensor, + value: torch.Tensor, + key_cache: torch.Tensor, + value_cache: torch.Tensor, + slot_mapping: torch.Tensor, + kv_cache_dtype: str, + k_scale: float, + v_scale: float, +) -> None: + ops.reshape_and_cache( + key, + value, + key_cache, + value_cache, + slot_mapping, + kv_cache_dtype, + k_scale, + v_scale, + ) + + +def reshape_and_cache_flash( + key: torch.Tensor, + value: torch.Tensor, + key_cache: torch.Tensor, + value_cache: torch.Tensor, + slot_mapping: torch.Tensor, + kv_cache_dtype: str, + k_scale: torch.Tensor, + v_scale: torch.Tensor, +) -> None: + ops.reshape_and_cache_flash( + key, + value, + key_cache, + value_cache, + slot_mapping, + kv_cache_dtype, + k_scale, + v_scale, + ) + + +def copy_blocks( + key_caches: List[torch.Tensor], + value_caches: List[torch.Tensor], + block_mapping: torch.Tensor, +) -> None: + ops.copy_blocks(key_caches, value_caches, block_mapping) + + +def swap_blocks( + src: torch.Tensor, dst: torch.Tensor, block_mapping: torch.Tensor +) -> None: + ops.swap_blocks(src, dst, block_mapping) + + +def convert_fp8( + output: torch.Tensor, input: torch.Tensor, scale: float = 1.0, kv_dtype: str = "fp8" +) -> None: + ops.convert_fp8(output, input, scale, kv_dtype) + + +__all__ = [ + "convert_fp8", + "paged_attention_v1", + "paged_attention_v2", + "reshape_and_cache", + "copy_blocks", +] diff --git a/build/torch29-cxx11-rocm63-x86_64-linux/_ops.py b/build/torch29-cxx11-rocm63-x86_64-linux/_ops.py new file mode 100644 index 0000000000000000000000000000000000000000..2bd4e7e759659af7e4cc1eac77bd7d389685c212 --- /dev/null +++ b/build/torch29-cxx11-rocm63-x86_64-linux/_ops.py @@ -0,0 +1,9 @@ +import torch +from . import _paged_attention_61fdee0 +ops = torch.ops._paged_attention_61fdee0 + +def add_op_namespace_prefix(op_name: str): + """ + Prefix op by namespace. + """ + return f"_paged_attention_61fdee0::{op_name}" \ No newline at end of file diff --git a/build/torch29-cxx11-rocm63-x86_64-linux/_paged_attention_61fdee0.abi3.so b/build/torch29-cxx11-rocm63-x86_64-linux/_paged_attention_61fdee0.abi3.so new file mode 100644 index 0000000000000000000000000000000000000000..47ca67b92a0cfb6d6ac5e6809cc3e3af7535eaa8 --- /dev/null +++ b/build/torch29-cxx11-rocm63-x86_64-linux/_paged_attention_61fdee0.abi3.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:50d231ac13d7dfed51429bf1ccbe506e4edcd7cb53bec926f2c50b583c722bf5 +size 120196920 diff --git a/build/torch29-cxx11-rocm63-x86_64-linux/metadata.json b/build/torch29-cxx11-rocm63-x86_64-linux/metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..76bafa5f33b6818aa6bb4cab04be811b87519b44 --- /dev/null +++ b/build/torch29-cxx11-rocm63-x86_64-linux/metadata.json @@ -0,0 +1 @@ +{"python-depends":[]} \ No newline at end of file diff --git a/build/torch29-cxx11-rocm63-x86_64-linux/paged_attention/__init__.py b/build/torch29-cxx11-rocm63-x86_64-linux/paged_attention/__init__.py index 9de56043369487facc1f163df6bd319c9806e5ca..03dbc1afe1cf156661a2b1b22003cd5f599a0309 100644 --- a/build/torch29-cxx11-rocm63-x86_64-linux/paged_attention/__init__.py +++ b/build/torch29-cxx11-rocm63-x86_64-linux/paged_attention/__init__.py @@ -1,21 +1,26 @@ -from ._custom_ops import ( - convert_fp8, - copy_blocks, - paged_attention_v1, - paged_attention_v2, - reshape_and_cache, - reshape_and_cache_flash, - swap_blocks, -) -from ._ops import ops +import ctypes +import sys -__all__ = [ - "convert_fp8", - "copy_blocks", - "ops", - "paged_attention_v1", - "paged_attention_v2", - "reshape_and_cache", - "reshape_and_cache_flash", - "swap_blocks", -] +import importlib +from pathlib import Path +from types import ModuleType + +def _import_from_path(file_path: Path) -> ModuleType: + # We cannot use the module name as-is, after adding it to `sys.modules`, + # it would also be used for other imports. So, we make a module name that + # depends on the path for it to be unique using the hex-encoded hash of + # the path. + path_hash = "{:x}".format(ctypes.c_size_t(hash(file_path.absolute())).value) + module_name = path_hash + spec = importlib.util.spec_from_file_location(module_name, file_path) + if spec is None: + raise ImportError(f"Cannot load spec for {module_name} from {file_path}") + module = importlib.util.module_from_spec(spec) + if module is None: + raise ImportError(f"Cannot load module {module_name} from spec") + sys.modules[module_name] = module + spec.loader.exec_module(module) # type: ignore + return module + + +globals().update(vars(_import_from_path(Path(__file__).parent.parent / "__init__.py"))) diff --git a/build/torch29-cxx11-rocm63-x86_64-linux/paged_attention/_ops.py b/build/torch29-cxx11-rocm63-x86_64-linux/paged_attention/_ops.py deleted file mode 100644 index d09b198f2169310837bcd6b8ea88308879b5775d..0000000000000000000000000000000000000000 --- a/build/torch29-cxx11-rocm63-x86_64-linux/paged_attention/_ops.py +++ /dev/null @@ -1,9 +0,0 @@ -import torch -from . import _paged_attention_287831d -ops = torch.ops._paged_attention_287831d - -def add_op_namespace_prefix(op_name: str): - """ - Prefix op by namespace. - """ - return f"_paged_attention_287831d::{op_name}" \ No newline at end of file diff --git a/build/torch29-cxx11-rocm63-x86_64-linux/paged_attention/_paged_attention_287831d.abi3.so b/build/torch29-cxx11-rocm63-x86_64-linux/paged_attention/_paged_attention_287831d.abi3.so deleted file mode 100644 index fadde2636c54c5339bdf392960d0154c2a47cf42..0000000000000000000000000000000000000000 --- a/build/torch29-cxx11-rocm63-x86_64-linux/paged_attention/_paged_attention_287831d.abi3.so +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:406d35b2ad310b4d62739fb18b48931e7631d608ab10be496e65784860d12bae -size 120179000 diff --git a/build/torch29-cxx11-rocm63-x86_64-linux/platforms.py b/build/torch29-cxx11-rocm63-x86_64-linux/platforms.py new file mode 100644 index 0000000000000000000000000000000000000000..6277d5f50ff3ddc265bb39fa1c4d17e0341b7767 --- /dev/null +++ b/build/torch29-cxx11-rocm63-x86_64-linux/platforms.py @@ -0,0 +1,92 @@ +import os +import random +from abc import ABC, abstractmethod +from functools import lru_cache, wraps +from typing import Callable, ParamSpec, TypeVar + +import numpy as np +import torch + +IS_ROCM = torch.version.hip is not None +IS_MPS = torch.backends.mps.is_available() + + +class Platform(ABC): + @classmethod + def seed_everything(cls, seed: int) -> None: + """ + Set the seed of each random module. + `torch.manual_seed` will set seed on all devices. + + Loosely based on: https://github.com/Lightning-AI/pytorch-lightning/blob/2.4.0/src/lightning/fabric/utilities/seed.py#L20 + """ + random.seed(seed) + np.random.seed(seed) + torch.manual_seed(seed) + + @abstractmethod + def get_device_name(self, device_id: int = 0) -> str: ... + + @abstractmethod + def is_cuda(self) -> bool: ... + + @abstractmethod + def is_rocm(self) -> bool: ... + + @abstractmethod + def is_mps(self) -> bool: ... + + +class CudaPlatform(Platform): + @classmethod + @lru_cache(maxsize=8) + def get_device_name(cls, device_id: int = 0) -> str: + return torch.cuda.get_device_name(0) + + def is_cuda(self) -> bool: + return True + + def is_rocm(self) -> bool: + return False + + def is_mps(self) -> bool: + return False + + +class RocmPlatform(Platform): + @classmethod + @lru_cache(maxsize=8) + def get_device_name(cls, device_id: int = 0) -> str: + return torch.cuda.get_device_name(device_id) + + def is_cuda(self) -> bool: + return False + + def is_rocm(self) -> bool: + return True + + def is_mps(self) -> bool: + return False + + +class MpsPlatform(Platform): + @classmethod + @lru_cache(maxsize=8) + def get_device_name(cls, device_id: int = 0) -> str: + return torch.cuda.get_device_name(device_id) + + def is_cuda(self) -> bool: + return False + + def is_rocm(self) -> bool: + return False + + def is_mps(self) -> bool: + return True + +current_platform = ( + RocmPlatform() if IS_ROCM else + MpsPlatform() if IS_MPS else + CudaPlatform() if torch.cuda.is_available() else + None +) diff --git a/build/torch29-cxx11-rocm64-x86_64-linux/__init__.py b/build/torch29-cxx11-rocm64-x86_64-linux/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..9de56043369487facc1f163df6bd319c9806e5ca --- /dev/null +++ b/build/torch29-cxx11-rocm64-x86_64-linux/__init__.py @@ -0,0 +1,21 @@ +from ._custom_ops import ( + convert_fp8, + copy_blocks, + paged_attention_v1, + paged_attention_v2, + reshape_and_cache, + reshape_and_cache_flash, + swap_blocks, +) +from ._ops import ops + +__all__ = [ + "convert_fp8", + "copy_blocks", + "ops", + "paged_attention_v1", + "paged_attention_v2", + "reshape_and_cache", + "reshape_and_cache_flash", + "swap_blocks", +] diff --git a/build/torch29-cxx11-rocm64-x86_64-linux/_custom_ops.py b/build/torch29-cxx11-rocm64-x86_64-linux/_custom_ops.py new file mode 100644 index 0000000000000000000000000000000000000000..a0c0b8db085468dee5100c98d14106a9ee917bf2 --- /dev/null +++ b/build/torch29-cxx11-rocm64-x86_64-linux/_custom_ops.py @@ -0,0 +1,173 @@ +from typing import List, Optional + +import torch + +from ._ops import ops + + +# page attention ops +def paged_attention_v1( + out: torch.Tensor, + query: torch.Tensor, + key_cache: torch.Tensor, + value_cache: torch.Tensor, + num_kv_heads: int, + scale: float, + block_tables: torch.Tensor, + seq_lens: torch.Tensor, + block_size: int, + max_seq_len: int, + alibi_slopes: Optional[torch.Tensor], + kv_cache_dtype: str, + k_scale: float, + v_scale: float, + tp_rank: int = 0, + blocksparse_local_blocks: int = 0, + blocksparse_vert_stride: int = 0, + blocksparse_block_size: int = 64, + blocksparse_head_sliding_step: int = 0, +) -> None: + ops.paged_attention_v1( + out, + query, + key_cache, + value_cache, + num_kv_heads, + scale, + block_tables, + seq_lens, + block_size, + max_seq_len, + alibi_slopes, + kv_cache_dtype, + k_scale, + v_scale, + tp_rank, + blocksparse_local_blocks, + blocksparse_vert_stride, + blocksparse_block_size, + blocksparse_head_sliding_step, + ) + + +def paged_attention_v2( + out: torch.Tensor, + exp_sum: torch.Tensor, + max_logits: torch.Tensor, + tmp_out: torch.Tensor, + query: torch.Tensor, + key_cache: torch.Tensor, + value_cache: torch.Tensor, + num_kv_heads: int, + scale: float, + block_tables: torch.Tensor, + seq_lens: torch.Tensor, + block_size: int, + max_seq_len: int, + alibi_slopes: Optional[torch.Tensor], + kv_cache_dtype: str, + k_scale: float, + v_scale: float, + tp_rank: int = 0, + blocksparse_local_blocks: int = 0, + blocksparse_vert_stride: int = 0, + blocksparse_block_size: int = 64, + blocksparse_head_sliding_step: int = 0, +) -> None: + ops.paged_attention_v2( + out, + exp_sum, + max_logits, + tmp_out, + query, + key_cache, + value_cache, + num_kv_heads, + scale, + block_tables, + seq_lens, + block_size, + max_seq_len, + alibi_slopes, + kv_cache_dtype, + k_scale, + v_scale, + tp_rank, + blocksparse_local_blocks, + blocksparse_vert_stride, + blocksparse_block_size, + blocksparse_head_sliding_step, + ) + + +def reshape_and_cache( + key: torch.Tensor, + value: torch.Tensor, + key_cache: torch.Tensor, + value_cache: torch.Tensor, + slot_mapping: torch.Tensor, + kv_cache_dtype: str, + k_scale: float, + v_scale: float, +) -> None: + ops.reshape_and_cache( + key, + value, + key_cache, + value_cache, + slot_mapping, + kv_cache_dtype, + k_scale, + v_scale, + ) + + +def reshape_and_cache_flash( + key: torch.Tensor, + value: torch.Tensor, + key_cache: torch.Tensor, + value_cache: torch.Tensor, + slot_mapping: torch.Tensor, + kv_cache_dtype: str, + k_scale: torch.Tensor, + v_scale: torch.Tensor, +) -> None: + ops.reshape_and_cache_flash( + key, + value, + key_cache, + value_cache, + slot_mapping, + kv_cache_dtype, + k_scale, + v_scale, + ) + + +def copy_blocks( + key_caches: List[torch.Tensor], + value_caches: List[torch.Tensor], + block_mapping: torch.Tensor, +) -> None: + ops.copy_blocks(key_caches, value_caches, block_mapping) + + +def swap_blocks( + src: torch.Tensor, dst: torch.Tensor, block_mapping: torch.Tensor +) -> None: + ops.swap_blocks(src, dst, block_mapping) + + +def convert_fp8( + output: torch.Tensor, input: torch.Tensor, scale: float = 1.0, kv_dtype: str = "fp8" +) -> None: + ops.convert_fp8(output, input, scale, kv_dtype) + + +__all__ = [ + "convert_fp8", + "paged_attention_v1", + "paged_attention_v2", + "reshape_and_cache", + "copy_blocks", +] diff --git a/build/torch29-cxx11-rocm64-x86_64-linux/_ops.py b/build/torch29-cxx11-rocm64-x86_64-linux/_ops.py new file mode 100644 index 0000000000000000000000000000000000000000..2bd4e7e759659af7e4cc1eac77bd7d389685c212 --- /dev/null +++ b/build/torch29-cxx11-rocm64-x86_64-linux/_ops.py @@ -0,0 +1,9 @@ +import torch +from . import _paged_attention_61fdee0 +ops = torch.ops._paged_attention_61fdee0 + +def add_op_namespace_prefix(op_name: str): + """ + Prefix op by namespace. + """ + return f"_paged_attention_61fdee0::{op_name}" \ No newline at end of file diff --git a/build/torch29-cxx11-rocm64-x86_64-linux/_paged_attention_61fdee0.abi3.so b/build/torch29-cxx11-rocm64-x86_64-linux/_paged_attention_61fdee0.abi3.so new file mode 100644 index 0000000000000000000000000000000000000000..19541075b7b2ca270911729c2469b41ee8c42f5e --- /dev/null +++ b/build/torch29-cxx11-rocm64-x86_64-linux/_paged_attention_61fdee0.abi3.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a337ea826cba2881731cc25eab2cc6340dd35fbfe8e6f6f9be2180cff9faf9c4 +size 121030520 diff --git a/build/torch29-cxx11-rocm64-x86_64-linux/metadata.json b/build/torch29-cxx11-rocm64-x86_64-linux/metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..76bafa5f33b6818aa6bb4cab04be811b87519b44 --- /dev/null +++ b/build/torch29-cxx11-rocm64-x86_64-linux/metadata.json @@ -0,0 +1 @@ +{"python-depends":[]} \ No newline at end of file diff --git a/build/torch29-cxx11-rocm64-x86_64-linux/paged_attention/__init__.py b/build/torch29-cxx11-rocm64-x86_64-linux/paged_attention/__init__.py index 9de56043369487facc1f163df6bd319c9806e5ca..03dbc1afe1cf156661a2b1b22003cd5f599a0309 100644 --- a/build/torch29-cxx11-rocm64-x86_64-linux/paged_attention/__init__.py +++ b/build/torch29-cxx11-rocm64-x86_64-linux/paged_attention/__init__.py @@ -1,21 +1,26 @@ -from ._custom_ops import ( - convert_fp8, - copy_blocks, - paged_attention_v1, - paged_attention_v2, - reshape_and_cache, - reshape_and_cache_flash, - swap_blocks, -) -from ._ops import ops +import ctypes +import sys -__all__ = [ - "convert_fp8", - "copy_blocks", - "ops", - "paged_attention_v1", - "paged_attention_v2", - "reshape_and_cache", - "reshape_and_cache_flash", - "swap_blocks", -] +import importlib +from pathlib import Path +from types import ModuleType + +def _import_from_path(file_path: Path) -> ModuleType: + # We cannot use the module name as-is, after adding it to `sys.modules`, + # it would also be used for other imports. So, we make a module name that + # depends on the path for it to be unique using the hex-encoded hash of + # the path. + path_hash = "{:x}".format(ctypes.c_size_t(hash(file_path.absolute())).value) + module_name = path_hash + spec = importlib.util.spec_from_file_location(module_name, file_path) + if spec is None: + raise ImportError(f"Cannot load spec for {module_name} from {file_path}") + module = importlib.util.module_from_spec(spec) + if module is None: + raise ImportError(f"Cannot load module {module_name} from spec") + sys.modules[module_name] = module + spec.loader.exec_module(module) # type: ignore + return module + + +globals().update(vars(_import_from_path(Path(__file__).parent.parent / "__init__.py"))) diff --git a/build/torch29-cxx11-rocm64-x86_64-linux/paged_attention/_ops.py b/build/torch29-cxx11-rocm64-x86_64-linux/paged_attention/_ops.py deleted file mode 100644 index d09b198f2169310837bcd6b8ea88308879b5775d..0000000000000000000000000000000000000000 --- a/build/torch29-cxx11-rocm64-x86_64-linux/paged_attention/_ops.py +++ /dev/null @@ -1,9 +0,0 @@ -import torch -from . import _paged_attention_287831d -ops = torch.ops._paged_attention_287831d - -def add_op_namespace_prefix(op_name: str): - """ - Prefix op by namespace. - """ - return f"_paged_attention_287831d::{op_name}" \ No newline at end of file diff --git a/build/torch29-cxx11-rocm64-x86_64-linux/paged_attention/_paged_attention_287831d.abi3.so b/build/torch29-cxx11-rocm64-x86_64-linux/paged_attention/_paged_attention_287831d.abi3.so deleted file mode 100644 index bdbe2f956dd1c1ba8f0a78095e58b46899558204..0000000000000000000000000000000000000000 --- a/build/torch29-cxx11-rocm64-x86_64-linux/paged_attention/_paged_attention_287831d.abi3.so +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:0b142366acfa55e943523691850460892ec311161d9dbba4b3ac4dcaa8a96794 -size 121016696 diff --git a/build/torch29-cxx11-rocm64-x86_64-linux/platforms.py b/build/torch29-cxx11-rocm64-x86_64-linux/platforms.py new file mode 100644 index 0000000000000000000000000000000000000000..6277d5f50ff3ddc265bb39fa1c4d17e0341b7767 --- /dev/null +++ b/build/torch29-cxx11-rocm64-x86_64-linux/platforms.py @@ -0,0 +1,92 @@ +import os +import random +from abc import ABC, abstractmethod +from functools import lru_cache, wraps +from typing import Callable, ParamSpec, TypeVar + +import numpy as np +import torch + +IS_ROCM = torch.version.hip is not None +IS_MPS = torch.backends.mps.is_available() + + +class Platform(ABC): + @classmethod + def seed_everything(cls, seed: int) -> None: + """ + Set the seed of each random module. + `torch.manual_seed` will set seed on all devices. + + Loosely based on: https://github.com/Lightning-AI/pytorch-lightning/blob/2.4.0/src/lightning/fabric/utilities/seed.py#L20 + """ + random.seed(seed) + np.random.seed(seed) + torch.manual_seed(seed) + + @abstractmethod + def get_device_name(self, device_id: int = 0) -> str: ... + + @abstractmethod + def is_cuda(self) -> bool: ... + + @abstractmethod + def is_rocm(self) -> bool: ... + + @abstractmethod + def is_mps(self) -> bool: ... + + +class CudaPlatform(Platform): + @classmethod + @lru_cache(maxsize=8) + def get_device_name(cls, device_id: int = 0) -> str: + return torch.cuda.get_device_name(0) + + def is_cuda(self) -> bool: + return True + + def is_rocm(self) -> bool: + return False + + def is_mps(self) -> bool: + return False + + +class RocmPlatform(Platform): + @classmethod + @lru_cache(maxsize=8) + def get_device_name(cls, device_id: int = 0) -> str: + return torch.cuda.get_device_name(device_id) + + def is_cuda(self) -> bool: + return False + + def is_rocm(self) -> bool: + return True + + def is_mps(self) -> bool: + return False + + +class MpsPlatform(Platform): + @classmethod + @lru_cache(maxsize=8) + def get_device_name(cls, device_id: int = 0) -> str: + return torch.cuda.get_device_name(device_id) + + def is_cuda(self) -> bool: + return False + + def is_rocm(self) -> bool: + return False + + def is_mps(self) -> bool: + return True + +current_platform = ( + RocmPlatform() if IS_ROCM else + MpsPlatform() if IS_MPS else + CudaPlatform() if torch.cuda.is_available() else + None +)