diff --git a/.gitattributes b/.gitattributes index e1ad7cbaf16a76cddd75d6d2aa6024c46b96a7f9..3c706352df3e264b5466ecf45bc76a6e232a4a2c 100644 --- a/.gitattributes +++ b/.gitattributes @@ -51,3 +51,12 @@ build/torch28-cxx11-cu129-x86_64-linux/rwkv/_rwkv_0aa7cb0.abi3.so filter=lfs dif build/torch29-cxx11-cu126-x86_64-linux/rwkv/_rwkv_0aa7cb0.abi3.so filter=lfs diff=lfs merge=lfs -text build/torch29-cxx11-cu128-x86_64-linux/rwkv/_rwkv_0aa7cb0.abi3.so filter=lfs diff=lfs merge=lfs -text build/torch29-cxx11-cu130-x86_64-linux/rwkv/_rwkv_0aa7cb0.abi3.so filter=lfs diff=lfs merge=lfs -text +build/torch210-cxx11-cu126-x86_64-linux/_rwkv_d5e72cf.abi3.so filter=lfs diff=lfs merge=lfs -text +build/torch210-cxx11-cu128-x86_64-linux/_rwkv_d5e72cf.abi3.so filter=lfs diff=lfs merge=lfs -text +build/torch210-cxx11-cu130-x86_64-linux/_rwkv_d5e72cf.abi3.so filter=lfs diff=lfs merge=lfs -text +build/torch28-cxx11-cu126-x86_64-linux/_rwkv_d5e72cf.abi3.so filter=lfs diff=lfs merge=lfs -text +build/torch28-cxx11-cu128-x86_64-linux/_rwkv_d5e72cf.abi3.so filter=lfs diff=lfs merge=lfs -text +build/torch28-cxx11-cu129-x86_64-linux/_rwkv_d5e72cf.abi3.so filter=lfs diff=lfs merge=lfs -text +build/torch29-cxx11-cu126-x86_64-linux/_rwkv_d5e72cf.abi3.so filter=lfs diff=lfs merge=lfs -text +build/torch29-cxx11-cu128-x86_64-linux/_rwkv_d5e72cf.abi3.so filter=lfs diff=lfs merge=lfs -text +build/torch29-cxx11-cu130-x86_64-linux/_rwkv_d5e72cf.abi3.so filter=lfs diff=lfs merge=lfs -text diff --git a/build/torch210-cxx11-cu126-x86_64-linux/__init__.py b/build/torch210-cxx11-cu126-x86_64-linux/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..d4ea22dbc160d2b75b32c3c8c5d882d25c77b494 --- /dev/null +++ b/build/torch210-cxx11-cu126-x86_64-linux/__init__.py @@ -0,0 +1,170 @@ +from ._ops import ops +from typing import Tuple, Any + +# Use a broad Tensor alias to avoid importing torch at import time. +from torch import Tensor + +def forward(w: Tensor, u: Tensor, k: Tensor, v: Tensor, y: Tensor) -> None: + """RWKV WKV forward pass (float32). + + Runs the CUDA kernel and writes the result into ``y`` in-place. + + Args: + w: Decay weights, shape ``[C]``, dtype ``torch.float32``. + u: Input tensor, shape ``[B, T, C]``, dtype ``torch.float32``. + k: Key tensor, shape ``[B, T, C]``, dtype ``torch.float32``. + v: Value tensor, shape ``[B, T, C]``, dtype ``torch.float32``. + y: Output tensor, shape ``[B, T, C]``, dtype ``torch.float32`` (written in-place). + + Notes: + - All tensors must be on the same CUDA device. + - Shapes must agree on ``B``, ``T`` and ``C``. + """ + _validate_device_match((w, u, k, v, y)) + ops.forward(w, u, k, v, y) + + +def forward_bf16(w: Tensor, u: Tensor, k: Tensor, v: Tensor, y: Tensor) -> None: + """RWKV WKV forward pass (bfloat16 inputs/outputs, float32 ``w``). + + Runs the CUDA kernel and writes the result into ``y`` in-place. + + Args: + w: Decay weights, shape ``[C]``, dtype ``torch.float32``. + u: Input tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16``. + k: Key tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16``. + v: Value tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16``. + y: Output tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16`` (written in-place). + + Notes: + - All tensors must be on the same CUDA device. + - Shapes must agree on ``B``, ``T`` and ``C``. + """ + _validate_device_match((w, u, k, v, y)) + ops.forward_bf16(w, u, k, v, y) + + +def forward_with_state(w: Tensor, u: Tensor, k: Tensor, v: Tensor, y: Tensor, s: Tensor) -> None: + """RWKV WKV forward pass with persistent state (float32). + + Runs the CUDA kernel using and updating state ``s`` and writes the result into ``y``. + + Args: + w: Decay weights, shape ``[C]``, dtype ``torch.float32``. + u: Input tensor, shape ``[B, T, C]``, dtype ``torch.float32``. + k: Key tensor, shape ``[B, T, C]``, dtype ``torch.float32``. + v: Value tensor, shape ``[B, T, C]``, dtype ``torch.float32``. + y: Output tensor, shape ``[B, T, C]``, dtype ``torch.float32`` (written in-place). + s: Stateful tensor, shape ``[B, C]``, dtype ``torch.float32`` (updated in-place). + + Notes: + - All tensors must be on the same CUDA device. + - Shapes must agree on ``B`` and ``C``; ``y`` shares ``[B, T, C]`` with inputs. + """ + _validate_device_match((w, u, k, v, y, s)) + ops.forward_with_state(w, u, k, v, y, s) + + +def forward_with_state_bf16(w: Tensor, u: Tensor, k: Tensor, v: Tensor, y: Tensor, s: Tensor) -> None: + """RWKV WKV forward pass with persistent state (bfloat16 inputs/outputs, float32 ``w`` and ``s``). + + Runs the CUDA kernel using and updating state ``s`` and writes the result into ``y``. + + Args: + w: Decay weights, shape ``[C]``, dtype ``torch.float32``. + u: Input tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16``. + k: Key tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16``. + v: Value tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16``. + y: Output tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16`` (written in-place). + s: Stateful tensor, shape ``[B, C]``, dtype ``torch.float32`` (updated in-place). + + Notes: + - All tensors must be on the same CUDA device. + - Shapes must agree on ``B`` and ``C``; ``y`` shares ``[B, T, C]`` with inputs. + """ + _validate_device_match((w, u, k, v, y, s)) + ops.forward_with_state_bf16(w, u, k, v, y, s) + + +def backward( + w: Tensor, + u: Tensor, + k: Tensor, + v: Tensor, + y: Tensor, + gy: Tensor, + gw: Tensor, + gu: Tensor, + gk: Tensor, + gv: Tensor, +) -> None: + """RWKV WKV backward pass (float32). + + Writes gradients into the provided tensors in-place. + + Args: + w: Decay weights, shape ``[C]``, dtype ``torch.float32``. + u, k, v, y: Forward-pass tensors, shape ``[B, T, C]``, dtype ``torch.float32``. + gy: Gradient of ``y``, shape ``[B, T, C]``, dtype ``torch.float32``. + gw: Gradient for ``w``, shape ``[C]``, dtype ``torch.float32`` (written in-place). + gu, gk, gv: Gradients for ``u``, ``k``, ``v`` respectively, shape ``[B, T, C]``, dtype ``torch.float32`` (written in-place). + + Notes: + - All tensors must be on the same CUDA device. + - Shapes must agree on ``B``, ``T`` and ``C``. + """ + _validate_device_match((w, u, k, v, y, gy, gw, gu, gk, gv)) + ops.backward(w, u, k, v, y, gy, gw, gu, gk, gv) + + +def backward_bf16( + w: Tensor, + u: Tensor, + k: Tensor, + v: Tensor, + y: Tensor, + gy: Tensor, + gw: Tensor, + gu: Tensor, + gk: Tensor, + gv: Tensor, +) -> None: + """RWKV WKV backward pass (bfloat16 inputs/outputs/gradients, float32 ``w``). + + Writes gradients into the provided tensors in-place. + + Args: + w: Decay weights, shape ``[C]``, dtype ``torch.float32``. + u, k, v, y: Forward-pass tensors, shape ``[B, T, C]``, dtype ``torch.bfloat16``. + gy: Gradient of ``y``, shape ``[B, T, C]``, dtype ``torch.bfloat16``. + gw: Gradient for ``w``, shape ``[C]``, dtype ``torch.bfloat16`` (written in-place). + gu, gk, gv: Gradients for ``u``, ``k``, ``v`` respectively, shape ``[B, T, C]``, dtype ``torch.bfloat16`` (written in-place). + + Notes: + - All tensors must be on the same CUDA device. + - Shapes must agree on ``B``, ``T`` and ``C``. + """ + _validate_device_match((w, u, k, v, y, gy, gw, gu, gk, gv)) + ops.backward_bf16(w, u, k, v, y, gy, gw, gu, gk, gv) + + +def _validate_device_match(tensors: Tuple[Tensor, ...]) -> None: + """Minimal runtime validation that all tensors live on the same CUDA device.""" + if not tensors: + return + device = tensors[0].device + if not device.type == "cuda": + raise RuntimeError("RWKV CUDA ops require CUDA tensors") + for t in tensors[1:]: + if t.device != device: + raise RuntimeError("All tensors must be on the same CUDA device") + + +__all__ = [ + "forward", + "forward_bf16", + "forward_with_state", + "forward_with_state_bf16", + "backward", + "backward_bf16", +] \ No newline at end of file diff --git a/build/torch28-cxx11-cu128-x86_64-linux/rwkv/_ops.py b/build/torch210-cxx11-cu126-x86_64-linux/_ops.py similarity index 51% rename from build/torch28-cxx11-cu128-x86_64-linux/rwkv/_ops.py rename to build/torch210-cxx11-cu126-x86_64-linux/_ops.py index c54b92a73953ef7168a8a45b1d3caa2e64acc4cb..38c698ed4f2208d589ead1ef3852c6019d3eb01c 100644 --- a/build/torch28-cxx11-cu128-x86_64-linux/rwkv/_ops.py +++ b/build/torch210-cxx11-cu126-x86_64-linux/_ops.py @@ -1,9 +1,9 @@ import torch -from . import _rwkv_0aa7cb0 -ops = torch.ops._rwkv_0aa7cb0 +from . import _rwkv_d5e72cf +ops = torch.ops._rwkv_d5e72cf def add_op_namespace_prefix(op_name: str): """ Prefix op by namespace. """ - return f"_rwkv_0aa7cb0::{op_name}" \ No newline at end of file + return f"_rwkv_d5e72cf::{op_name}" \ No newline at end of file diff --git a/build/torch210-cxx11-cu126-x86_64-linux/_rwkv_d5e72cf.abi3.so b/build/torch210-cxx11-cu126-x86_64-linux/_rwkv_d5e72cf.abi3.so new file mode 100644 index 0000000000000000000000000000000000000000..21e1f67b7b7783ba134b08031c558c66511e9eef --- /dev/null +++ b/build/torch210-cxx11-cu126-x86_64-linux/_rwkv_d5e72cf.abi3.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4f5cdcbad9720748c4cf71224cc978424700cdccaad1d462173cf72eafe6e42c +size 2116384 diff --git a/build/torch210-cxx11-cu126-x86_64-linux/metadata.json b/build/torch210-cxx11-cu126-x86_64-linux/metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..76bafa5f33b6818aa6bb4cab04be811b87519b44 --- /dev/null +++ b/build/torch210-cxx11-cu126-x86_64-linux/metadata.json @@ -0,0 +1 @@ +{"python-depends":[]} \ No newline at end of file diff --git a/build/torch210-cxx11-cu126-x86_64-linux/rwkv/__init__.py b/build/torch210-cxx11-cu126-x86_64-linux/rwkv/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..03dbc1afe1cf156661a2b1b22003cd5f599a0309 --- /dev/null +++ b/build/torch210-cxx11-cu126-x86_64-linux/rwkv/__init__.py @@ -0,0 +1,26 @@ +import ctypes +import sys + +import importlib +from pathlib import Path +from types import ModuleType + +def _import_from_path(file_path: Path) -> ModuleType: + # We cannot use the module name as-is, after adding it to `sys.modules`, + # it would also be used for other imports. So, we make a module name that + # depends on the path for it to be unique using the hex-encoded hash of + # the path. + path_hash = "{:x}".format(ctypes.c_size_t(hash(file_path.absolute())).value) + module_name = path_hash + spec = importlib.util.spec_from_file_location(module_name, file_path) + if spec is None: + raise ImportError(f"Cannot load spec for {module_name} from {file_path}") + module = importlib.util.module_from_spec(spec) + if module is None: + raise ImportError(f"Cannot load module {module_name} from spec") + sys.modules[module_name] = module + spec.loader.exec_module(module) # type: ignore + return module + + +globals().update(vars(_import_from_path(Path(__file__).parent.parent / "__init__.py"))) diff --git a/build/torch210-cxx11-cu128-x86_64-linux/__init__.py b/build/torch210-cxx11-cu128-x86_64-linux/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..d4ea22dbc160d2b75b32c3c8c5d882d25c77b494 --- /dev/null +++ b/build/torch210-cxx11-cu128-x86_64-linux/__init__.py @@ -0,0 +1,170 @@ +from ._ops import ops +from typing import Tuple, Any + +# Use a broad Tensor alias to avoid importing torch at import time. +from torch import Tensor + +def forward(w: Tensor, u: Tensor, k: Tensor, v: Tensor, y: Tensor) -> None: + """RWKV WKV forward pass (float32). + + Runs the CUDA kernel and writes the result into ``y`` in-place. + + Args: + w: Decay weights, shape ``[C]``, dtype ``torch.float32``. + u: Input tensor, shape ``[B, T, C]``, dtype ``torch.float32``. + k: Key tensor, shape ``[B, T, C]``, dtype ``torch.float32``. + v: Value tensor, shape ``[B, T, C]``, dtype ``torch.float32``. + y: Output tensor, shape ``[B, T, C]``, dtype ``torch.float32`` (written in-place). + + Notes: + - All tensors must be on the same CUDA device. + - Shapes must agree on ``B``, ``T`` and ``C``. + """ + _validate_device_match((w, u, k, v, y)) + ops.forward(w, u, k, v, y) + + +def forward_bf16(w: Tensor, u: Tensor, k: Tensor, v: Tensor, y: Tensor) -> None: + """RWKV WKV forward pass (bfloat16 inputs/outputs, float32 ``w``). + + Runs the CUDA kernel and writes the result into ``y`` in-place. + + Args: + w: Decay weights, shape ``[C]``, dtype ``torch.float32``. + u: Input tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16``. + k: Key tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16``. + v: Value tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16``. + y: Output tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16`` (written in-place). + + Notes: + - All tensors must be on the same CUDA device. + - Shapes must agree on ``B``, ``T`` and ``C``. + """ + _validate_device_match((w, u, k, v, y)) + ops.forward_bf16(w, u, k, v, y) + + +def forward_with_state(w: Tensor, u: Tensor, k: Tensor, v: Tensor, y: Tensor, s: Tensor) -> None: + """RWKV WKV forward pass with persistent state (float32). + + Runs the CUDA kernel using and updating state ``s`` and writes the result into ``y``. + + Args: + w: Decay weights, shape ``[C]``, dtype ``torch.float32``. + u: Input tensor, shape ``[B, T, C]``, dtype ``torch.float32``. + k: Key tensor, shape ``[B, T, C]``, dtype ``torch.float32``. + v: Value tensor, shape ``[B, T, C]``, dtype ``torch.float32``. + y: Output tensor, shape ``[B, T, C]``, dtype ``torch.float32`` (written in-place). + s: Stateful tensor, shape ``[B, C]``, dtype ``torch.float32`` (updated in-place). + + Notes: + - All tensors must be on the same CUDA device. + - Shapes must agree on ``B`` and ``C``; ``y`` shares ``[B, T, C]`` with inputs. + """ + _validate_device_match((w, u, k, v, y, s)) + ops.forward_with_state(w, u, k, v, y, s) + + +def forward_with_state_bf16(w: Tensor, u: Tensor, k: Tensor, v: Tensor, y: Tensor, s: Tensor) -> None: + """RWKV WKV forward pass with persistent state (bfloat16 inputs/outputs, float32 ``w`` and ``s``). + + Runs the CUDA kernel using and updating state ``s`` and writes the result into ``y``. + + Args: + w: Decay weights, shape ``[C]``, dtype ``torch.float32``. + u: Input tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16``. + k: Key tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16``. + v: Value tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16``. + y: Output tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16`` (written in-place). + s: Stateful tensor, shape ``[B, C]``, dtype ``torch.float32`` (updated in-place). + + Notes: + - All tensors must be on the same CUDA device. + - Shapes must agree on ``B`` and ``C``; ``y`` shares ``[B, T, C]`` with inputs. + """ + _validate_device_match((w, u, k, v, y, s)) + ops.forward_with_state_bf16(w, u, k, v, y, s) + + +def backward( + w: Tensor, + u: Tensor, + k: Tensor, + v: Tensor, + y: Tensor, + gy: Tensor, + gw: Tensor, + gu: Tensor, + gk: Tensor, + gv: Tensor, +) -> None: + """RWKV WKV backward pass (float32). + + Writes gradients into the provided tensors in-place. + + Args: + w: Decay weights, shape ``[C]``, dtype ``torch.float32``. + u, k, v, y: Forward-pass tensors, shape ``[B, T, C]``, dtype ``torch.float32``. + gy: Gradient of ``y``, shape ``[B, T, C]``, dtype ``torch.float32``. + gw: Gradient for ``w``, shape ``[C]``, dtype ``torch.float32`` (written in-place). + gu, gk, gv: Gradients for ``u``, ``k``, ``v`` respectively, shape ``[B, T, C]``, dtype ``torch.float32`` (written in-place). + + Notes: + - All tensors must be on the same CUDA device. + - Shapes must agree on ``B``, ``T`` and ``C``. + """ + _validate_device_match((w, u, k, v, y, gy, gw, gu, gk, gv)) + ops.backward(w, u, k, v, y, gy, gw, gu, gk, gv) + + +def backward_bf16( + w: Tensor, + u: Tensor, + k: Tensor, + v: Tensor, + y: Tensor, + gy: Tensor, + gw: Tensor, + gu: Tensor, + gk: Tensor, + gv: Tensor, +) -> None: + """RWKV WKV backward pass (bfloat16 inputs/outputs/gradients, float32 ``w``). + + Writes gradients into the provided tensors in-place. + + Args: + w: Decay weights, shape ``[C]``, dtype ``torch.float32``. + u, k, v, y: Forward-pass tensors, shape ``[B, T, C]``, dtype ``torch.bfloat16``. + gy: Gradient of ``y``, shape ``[B, T, C]``, dtype ``torch.bfloat16``. + gw: Gradient for ``w``, shape ``[C]``, dtype ``torch.bfloat16`` (written in-place). + gu, gk, gv: Gradients for ``u``, ``k``, ``v`` respectively, shape ``[B, T, C]``, dtype ``torch.bfloat16`` (written in-place). + + Notes: + - All tensors must be on the same CUDA device. + - Shapes must agree on ``B``, ``T`` and ``C``. + """ + _validate_device_match((w, u, k, v, y, gy, gw, gu, gk, gv)) + ops.backward_bf16(w, u, k, v, y, gy, gw, gu, gk, gv) + + +def _validate_device_match(tensors: Tuple[Tensor, ...]) -> None: + """Minimal runtime validation that all tensors live on the same CUDA device.""" + if not tensors: + return + device = tensors[0].device + if not device.type == "cuda": + raise RuntimeError("RWKV CUDA ops require CUDA tensors") + for t in tensors[1:]: + if t.device != device: + raise RuntimeError("All tensors must be on the same CUDA device") + + +__all__ = [ + "forward", + "forward_bf16", + "forward_with_state", + "forward_with_state_bf16", + "backward", + "backward_bf16", +] \ No newline at end of file diff --git a/build/torch28-cxx11-cu129-x86_64-linux/rwkv/_ops.py b/build/torch210-cxx11-cu128-x86_64-linux/_ops.py similarity index 51% rename from build/torch28-cxx11-cu129-x86_64-linux/rwkv/_ops.py rename to build/torch210-cxx11-cu128-x86_64-linux/_ops.py index c54b92a73953ef7168a8a45b1d3caa2e64acc4cb..38c698ed4f2208d589ead1ef3852c6019d3eb01c 100644 --- a/build/torch28-cxx11-cu129-x86_64-linux/rwkv/_ops.py +++ b/build/torch210-cxx11-cu128-x86_64-linux/_ops.py @@ -1,9 +1,9 @@ import torch -from . import _rwkv_0aa7cb0 -ops = torch.ops._rwkv_0aa7cb0 +from . import _rwkv_d5e72cf +ops = torch.ops._rwkv_d5e72cf def add_op_namespace_prefix(op_name: str): """ Prefix op by namespace. """ - return f"_rwkv_0aa7cb0::{op_name}" \ No newline at end of file + return f"_rwkv_d5e72cf::{op_name}" \ No newline at end of file diff --git a/build/torch210-cxx11-cu128-x86_64-linux/_rwkv_d5e72cf.abi3.so b/build/torch210-cxx11-cu128-x86_64-linux/_rwkv_d5e72cf.abi3.so new file mode 100644 index 0000000000000000000000000000000000000000..86bab240ff09a23df86640b7882f33b46b56af20 --- /dev/null +++ b/build/torch210-cxx11-cu128-x86_64-linux/_rwkv_d5e72cf.abi3.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9653aedd35c1c268cf5f4cd8ba5a4e1f240f3e19c80a23c9ffbe901fcef83bca +size 2318744 diff --git a/build/torch210-cxx11-cu128-x86_64-linux/metadata.json b/build/torch210-cxx11-cu128-x86_64-linux/metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..76bafa5f33b6818aa6bb4cab04be811b87519b44 --- /dev/null +++ b/build/torch210-cxx11-cu128-x86_64-linux/metadata.json @@ -0,0 +1 @@ +{"python-depends":[]} \ No newline at end of file diff --git a/build/torch210-cxx11-cu128-x86_64-linux/rwkv/__init__.py b/build/torch210-cxx11-cu128-x86_64-linux/rwkv/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..03dbc1afe1cf156661a2b1b22003cd5f599a0309 --- /dev/null +++ b/build/torch210-cxx11-cu128-x86_64-linux/rwkv/__init__.py @@ -0,0 +1,26 @@ +import ctypes +import sys + +import importlib +from pathlib import Path +from types import ModuleType + +def _import_from_path(file_path: Path) -> ModuleType: + # We cannot use the module name as-is, after adding it to `sys.modules`, + # it would also be used for other imports. So, we make a module name that + # depends on the path for it to be unique using the hex-encoded hash of + # the path. + path_hash = "{:x}".format(ctypes.c_size_t(hash(file_path.absolute())).value) + module_name = path_hash + spec = importlib.util.spec_from_file_location(module_name, file_path) + if spec is None: + raise ImportError(f"Cannot load spec for {module_name} from {file_path}") + module = importlib.util.module_from_spec(spec) + if module is None: + raise ImportError(f"Cannot load module {module_name} from spec") + sys.modules[module_name] = module + spec.loader.exec_module(module) # type: ignore + return module + + +globals().update(vars(_import_from_path(Path(__file__).parent.parent / "__init__.py"))) diff --git a/build/torch210-cxx11-cu130-x86_64-linux/__init__.py b/build/torch210-cxx11-cu130-x86_64-linux/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..d4ea22dbc160d2b75b32c3c8c5d882d25c77b494 --- /dev/null +++ b/build/torch210-cxx11-cu130-x86_64-linux/__init__.py @@ -0,0 +1,170 @@ +from ._ops import ops +from typing import Tuple, Any + +# Use a broad Tensor alias to avoid importing torch at import time. +from torch import Tensor + +def forward(w: Tensor, u: Tensor, k: Tensor, v: Tensor, y: Tensor) -> None: + """RWKV WKV forward pass (float32). + + Runs the CUDA kernel and writes the result into ``y`` in-place. + + Args: + w: Decay weights, shape ``[C]``, dtype ``torch.float32``. + u: Input tensor, shape ``[B, T, C]``, dtype ``torch.float32``. + k: Key tensor, shape ``[B, T, C]``, dtype ``torch.float32``. + v: Value tensor, shape ``[B, T, C]``, dtype ``torch.float32``. + y: Output tensor, shape ``[B, T, C]``, dtype ``torch.float32`` (written in-place). + + Notes: + - All tensors must be on the same CUDA device. + - Shapes must agree on ``B``, ``T`` and ``C``. + """ + _validate_device_match((w, u, k, v, y)) + ops.forward(w, u, k, v, y) + + +def forward_bf16(w: Tensor, u: Tensor, k: Tensor, v: Tensor, y: Tensor) -> None: + """RWKV WKV forward pass (bfloat16 inputs/outputs, float32 ``w``). + + Runs the CUDA kernel and writes the result into ``y`` in-place. + + Args: + w: Decay weights, shape ``[C]``, dtype ``torch.float32``. + u: Input tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16``. + k: Key tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16``. + v: Value tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16``. + y: Output tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16`` (written in-place). + + Notes: + - All tensors must be on the same CUDA device. + - Shapes must agree on ``B``, ``T`` and ``C``. + """ + _validate_device_match((w, u, k, v, y)) + ops.forward_bf16(w, u, k, v, y) + + +def forward_with_state(w: Tensor, u: Tensor, k: Tensor, v: Tensor, y: Tensor, s: Tensor) -> None: + """RWKV WKV forward pass with persistent state (float32). + + Runs the CUDA kernel using and updating state ``s`` and writes the result into ``y``. + + Args: + w: Decay weights, shape ``[C]``, dtype ``torch.float32``. + u: Input tensor, shape ``[B, T, C]``, dtype ``torch.float32``. + k: Key tensor, shape ``[B, T, C]``, dtype ``torch.float32``. + v: Value tensor, shape ``[B, T, C]``, dtype ``torch.float32``. + y: Output tensor, shape ``[B, T, C]``, dtype ``torch.float32`` (written in-place). + s: Stateful tensor, shape ``[B, C]``, dtype ``torch.float32`` (updated in-place). + + Notes: + - All tensors must be on the same CUDA device. + - Shapes must agree on ``B`` and ``C``; ``y`` shares ``[B, T, C]`` with inputs. + """ + _validate_device_match((w, u, k, v, y, s)) + ops.forward_with_state(w, u, k, v, y, s) + + +def forward_with_state_bf16(w: Tensor, u: Tensor, k: Tensor, v: Tensor, y: Tensor, s: Tensor) -> None: + """RWKV WKV forward pass with persistent state (bfloat16 inputs/outputs, float32 ``w`` and ``s``). + + Runs the CUDA kernel using and updating state ``s`` and writes the result into ``y``. + + Args: + w: Decay weights, shape ``[C]``, dtype ``torch.float32``. + u: Input tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16``. + k: Key tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16``. + v: Value tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16``. + y: Output tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16`` (written in-place). + s: Stateful tensor, shape ``[B, C]``, dtype ``torch.float32`` (updated in-place). + + Notes: + - All tensors must be on the same CUDA device. + - Shapes must agree on ``B`` and ``C``; ``y`` shares ``[B, T, C]`` with inputs. + """ + _validate_device_match((w, u, k, v, y, s)) + ops.forward_with_state_bf16(w, u, k, v, y, s) + + +def backward( + w: Tensor, + u: Tensor, + k: Tensor, + v: Tensor, + y: Tensor, + gy: Tensor, + gw: Tensor, + gu: Tensor, + gk: Tensor, + gv: Tensor, +) -> None: + """RWKV WKV backward pass (float32). + + Writes gradients into the provided tensors in-place. + + Args: + w: Decay weights, shape ``[C]``, dtype ``torch.float32``. + u, k, v, y: Forward-pass tensors, shape ``[B, T, C]``, dtype ``torch.float32``. + gy: Gradient of ``y``, shape ``[B, T, C]``, dtype ``torch.float32``. + gw: Gradient for ``w``, shape ``[C]``, dtype ``torch.float32`` (written in-place). + gu, gk, gv: Gradients for ``u``, ``k``, ``v`` respectively, shape ``[B, T, C]``, dtype ``torch.float32`` (written in-place). + + Notes: + - All tensors must be on the same CUDA device. + - Shapes must agree on ``B``, ``T`` and ``C``. + """ + _validate_device_match((w, u, k, v, y, gy, gw, gu, gk, gv)) + ops.backward(w, u, k, v, y, gy, gw, gu, gk, gv) + + +def backward_bf16( + w: Tensor, + u: Tensor, + k: Tensor, + v: Tensor, + y: Tensor, + gy: Tensor, + gw: Tensor, + gu: Tensor, + gk: Tensor, + gv: Tensor, +) -> None: + """RWKV WKV backward pass (bfloat16 inputs/outputs/gradients, float32 ``w``). + + Writes gradients into the provided tensors in-place. + + Args: + w: Decay weights, shape ``[C]``, dtype ``torch.float32``. + u, k, v, y: Forward-pass tensors, shape ``[B, T, C]``, dtype ``torch.bfloat16``. + gy: Gradient of ``y``, shape ``[B, T, C]``, dtype ``torch.bfloat16``. + gw: Gradient for ``w``, shape ``[C]``, dtype ``torch.bfloat16`` (written in-place). + gu, gk, gv: Gradients for ``u``, ``k``, ``v`` respectively, shape ``[B, T, C]``, dtype ``torch.bfloat16`` (written in-place). + + Notes: + - All tensors must be on the same CUDA device. + - Shapes must agree on ``B``, ``T`` and ``C``. + """ + _validate_device_match((w, u, k, v, y, gy, gw, gu, gk, gv)) + ops.backward_bf16(w, u, k, v, y, gy, gw, gu, gk, gv) + + +def _validate_device_match(tensors: Tuple[Tensor, ...]) -> None: + """Minimal runtime validation that all tensors live on the same CUDA device.""" + if not tensors: + return + device = tensors[0].device + if not device.type == "cuda": + raise RuntimeError("RWKV CUDA ops require CUDA tensors") + for t in tensors[1:]: + if t.device != device: + raise RuntimeError("All tensors must be on the same CUDA device") + + +__all__ = [ + "forward", + "forward_bf16", + "forward_with_state", + "forward_with_state_bf16", + "backward", + "backward_bf16", +] \ No newline at end of file diff --git a/build/torch28-cxx11-cu126-x86_64-linux/rwkv/_ops.py b/build/torch210-cxx11-cu130-x86_64-linux/_ops.py similarity index 51% rename from build/torch28-cxx11-cu126-x86_64-linux/rwkv/_ops.py rename to build/torch210-cxx11-cu130-x86_64-linux/_ops.py index c54b92a73953ef7168a8a45b1d3caa2e64acc4cb..38c698ed4f2208d589ead1ef3852c6019d3eb01c 100644 --- a/build/torch28-cxx11-cu126-x86_64-linux/rwkv/_ops.py +++ b/build/torch210-cxx11-cu130-x86_64-linux/_ops.py @@ -1,9 +1,9 @@ import torch -from . import _rwkv_0aa7cb0 -ops = torch.ops._rwkv_0aa7cb0 +from . import _rwkv_d5e72cf +ops = torch.ops._rwkv_d5e72cf def add_op_namespace_prefix(op_name: str): """ Prefix op by namespace. """ - return f"_rwkv_0aa7cb0::{op_name}" \ No newline at end of file + return f"_rwkv_d5e72cf::{op_name}" \ No newline at end of file diff --git a/build/torch210-cxx11-cu130-x86_64-linux/_rwkv_d5e72cf.abi3.so b/build/torch210-cxx11-cu130-x86_64-linux/_rwkv_d5e72cf.abi3.so new file mode 100644 index 0000000000000000000000000000000000000000..656338a14f43ffaa58fa8cf9c629806ff51d5d50 --- /dev/null +++ b/build/torch210-cxx11-cu130-x86_64-linux/_rwkv_d5e72cf.abi3.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7baf2eff9e1002104d19e31763d294d524e34264cd18867f98c419f0607cad08 +size 2348224 diff --git a/build/torch210-cxx11-cu130-x86_64-linux/metadata.json b/build/torch210-cxx11-cu130-x86_64-linux/metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..76bafa5f33b6818aa6bb4cab04be811b87519b44 --- /dev/null +++ b/build/torch210-cxx11-cu130-x86_64-linux/metadata.json @@ -0,0 +1 @@ +{"python-depends":[]} \ No newline at end of file diff --git a/build/torch210-cxx11-cu130-x86_64-linux/rwkv/__init__.py b/build/torch210-cxx11-cu130-x86_64-linux/rwkv/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..03dbc1afe1cf156661a2b1b22003cd5f599a0309 --- /dev/null +++ b/build/torch210-cxx11-cu130-x86_64-linux/rwkv/__init__.py @@ -0,0 +1,26 @@ +import ctypes +import sys + +import importlib +from pathlib import Path +from types import ModuleType + +def _import_from_path(file_path: Path) -> ModuleType: + # We cannot use the module name as-is, after adding it to `sys.modules`, + # it would also be used for other imports. So, we make a module name that + # depends on the path for it to be unique using the hex-encoded hash of + # the path. + path_hash = "{:x}".format(ctypes.c_size_t(hash(file_path.absolute())).value) + module_name = path_hash + spec = importlib.util.spec_from_file_location(module_name, file_path) + if spec is None: + raise ImportError(f"Cannot load spec for {module_name} from {file_path}") + module = importlib.util.module_from_spec(spec) + if module is None: + raise ImportError(f"Cannot load module {module_name} from spec") + sys.modules[module_name] = module + spec.loader.exec_module(module) # type: ignore + return module + + +globals().update(vars(_import_from_path(Path(__file__).parent.parent / "__init__.py"))) diff --git a/build/torch28-cxx11-cu126-x86_64-linux/__init__.py b/build/torch28-cxx11-cu126-x86_64-linux/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..d4ea22dbc160d2b75b32c3c8c5d882d25c77b494 --- /dev/null +++ b/build/torch28-cxx11-cu126-x86_64-linux/__init__.py @@ -0,0 +1,170 @@ +from ._ops import ops +from typing import Tuple, Any + +# Use a broad Tensor alias to avoid importing torch at import time. +from torch import Tensor + +def forward(w: Tensor, u: Tensor, k: Tensor, v: Tensor, y: Tensor) -> None: + """RWKV WKV forward pass (float32). + + Runs the CUDA kernel and writes the result into ``y`` in-place. + + Args: + w: Decay weights, shape ``[C]``, dtype ``torch.float32``. + u: Input tensor, shape ``[B, T, C]``, dtype ``torch.float32``. + k: Key tensor, shape ``[B, T, C]``, dtype ``torch.float32``. + v: Value tensor, shape ``[B, T, C]``, dtype ``torch.float32``. + y: Output tensor, shape ``[B, T, C]``, dtype ``torch.float32`` (written in-place). + + Notes: + - All tensors must be on the same CUDA device. + - Shapes must agree on ``B``, ``T`` and ``C``. + """ + _validate_device_match((w, u, k, v, y)) + ops.forward(w, u, k, v, y) + + +def forward_bf16(w: Tensor, u: Tensor, k: Tensor, v: Tensor, y: Tensor) -> None: + """RWKV WKV forward pass (bfloat16 inputs/outputs, float32 ``w``). + + Runs the CUDA kernel and writes the result into ``y`` in-place. + + Args: + w: Decay weights, shape ``[C]``, dtype ``torch.float32``. + u: Input tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16``. + k: Key tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16``. + v: Value tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16``. + y: Output tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16`` (written in-place). + + Notes: + - All tensors must be on the same CUDA device. + - Shapes must agree on ``B``, ``T`` and ``C``. + """ + _validate_device_match((w, u, k, v, y)) + ops.forward_bf16(w, u, k, v, y) + + +def forward_with_state(w: Tensor, u: Tensor, k: Tensor, v: Tensor, y: Tensor, s: Tensor) -> None: + """RWKV WKV forward pass with persistent state (float32). + + Runs the CUDA kernel using and updating state ``s`` and writes the result into ``y``. + + Args: + w: Decay weights, shape ``[C]``, dtype ``torch.float32``. + u: Input tensor, shape ``[B, T, C]``, dtype ``torch.float32``. + k: Key tensor, shape ``[B, T, C]``, dtype ``torch.float32``. + v: Value tensor, shape ``[B, T, C]``, dtype ``torch.float32``. + y: Output tensor, shape ``[B, T, C]``, dtype ``torch.float32`` (written in-place). + s: Stateful tensor, shape ``[B, C]``, dtype ``torch.float32`` (updated in-place). + + Notes: + - All tensors must be on the same CUDA device. + - Shapes must agree on ``B`` and ``C``; ``y`` shares ``[B, T, C]`` with inputs. + """ + _validate_device_match((w, u, k, v, y, s)) + ops.forward_with_state(w, u, k, v, y, s) + + +def forward_with_state_bf16(w: Tensor, u: Tensor, k: Tensor, v: Tensor, y: Tensor, s: Tensor) -> None: + """RWKV WKV forward pass with persistent state (bfloat16 inputs/outputs, float32 ``w`` and ``s``). + + Runs the CUDA kernel using and updating state ``s`` and writes the result into ``y``. + + Args: + w: Decay weights, shape ``[C]``, dtype ``torch.float32``. + u: Input tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16``. + k: Key tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16``. + v: Value tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16``. + y: Output tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16`` (written in-place). + s: Stateful tensor, shape ``[B, C]``, dtype ``torch.float32`` (updated in-place). + + Notes: + - All tensors must be on the same CUDA device. + - Shapes must agree on ``B`` and ``C``; ``y`` shares ``[B, T, C]`` with inputs. + """ + _validate_device_match((w, u, k, v, y, s)) + ops.forward_with_state_bf16(w, u, k, v, y, s) + + +def backward( + w: Tensor, + u: Tensor, + k: Tensor, + v: Tensor, + y: Tensor, + gy: Tensor, + gw: Tensor, + gu: Tensor, + gk: Tensor, + gv: Tensor, +) -> None: + """RWKV WKV backward pass (float32). + + Writes gradients into the provided tensors in-place. + + Args: + w: Decay weights, shape ``[C]``, dtype ``torch.float32``. + u, k, v, y: Forward-pass tensors, shape ``[B, T, C]``, dtype ``torch.float32``. + gy: Gradient of ``y``, shape ``[B, T, C]``, dtype ``torch.float32``. + gw: Gradient for ``w``, shape ``[C]``, dtype ``torch.float32`` (written in-place). + gu, gk, gv: Gradients for ``u``, ``k``, ``v`` respectively, shape ``[B, T, C]``, dtype ``torch.float32`` (written in-place). + + Notes: + - All tensors must be on the same CUDA device. + - Shapes must agree on ``B``, ``T`` and ``C``. + """ + _validate_device_match((w, u, k, v, y, gy, gw, gu, gk, gv)) + ops.backward(w, u, k, v, y, gy, gw, gu, gk, gv) + + +def backward_bf16( + w: Tensor, + u: Tensor, + k: Tensor, + v: Tensor, + y: Tensor, + gy: Tensor, + gw: Tensor, + gu: Tensor, + gk: Tensor, + gv: Tensor, +) -> None: + """RWKV WKV backward pass (bfloat16 inputs/outputs/gradients, float32 ``w``). + + Writes gradients into the provided tensors in-place. + + Args: + w: Decay weights, shape ``[C]``, dtype ``torch.float32``. + u, k, v, y: Forward-pass tensors, shape ``[B, T, C]``, dtype ``torch.bfloat16``. + gy: Gradient of ``y``, shape ``[B, T, C]``, dtype ``torch.bfloat16``. + gw: Gradient for ``w``, shape ``[C]``, dtype ``torch.bfloat16`` (written in-place). + gu, gk, gv: Gradients for ``u``, ``k``, ``v`` respectively, shape ``[B, T, C]``, dtype ``torch.bfloat16`` (written in-place). + + Notes: + - All tensors must be on the same CUDA device. + - Shapes must agree on ``B``, ``T`` and ``C``. + """ + _validate_device_match((w, u, k, v, y, gy, gw, gu, gk, gv)) + ops.backward_bf16(w, u, k, v, y, gy, gw, gu, gk, gv) + + +def _validate_device_match(tensors: Tuple[Tensor, ...]) -> None: + """Minimal runtime validation that all tensors live on the same CUDA device.""" + if not tensors: + return + device = tensors[0].device + if not device.type == "cuda": + raise RuntimeError("RWKV CUDA ops require CUDA tensors") + for t in tensors[1:]: + if t.device != device: + raise RuntimeError("All tensors must be on the same CUDA device") + + +__all__ = [ + "forward", + "forward_bf16", + "forward_with_state", + "forward_with_state_bf16", + "backward", + "backward_bf16", +] \ No newline at end of file diff --git a/build/torch29-cxx11-cu126-x86_64-linux/rwkv/_ops.py b/build/torch28-cxx11-cu126-x86_64-linux/_ops.py similarity index 51% rename from build/torch29-cxx11-cu126-x86_64-linux/rwkv/_ops.py rename to build/torch28-cxx11-cu126-x86_64-linux/_ops.py index c54b92a73953ef7168a8a45b1d3caa2e64acc4cb..38c698ed4f2208d589ead1ef3852c6019d3eb01c 100644 --- a/build/torch29-cxx11-cu126-x86_64-linux/rwkv/_ops.py +++ b/build/torch28-cxx11-cu126-x86_64-linux/_ops.py @@ -1,9 +1,9 @@ import torch -from . import _rwkv_0aa7cb0 -ops = torch.ops._rwkv_0aa7cb0 +from . import _rwkv_d5e72cf +ops = torch.ops._rwkv_d5e72cf def add_op_namespace_prefix(op_name: str): """ Prefix op by namespace. """ - return f"_rwkv_0aa7cb0::{op_name}" \ No newline at end of file + return f"_rwkv_d5e72cf::{op_name}" \ No newline at end of file diff --git a/build/torch28-cxx11-cu126-x86_64-linux/_rwkv_d5e72cf.abi3.so b/build/torch28-cxx11-cu126-x86_64-linux/_rwkv_d5e72cf.abi3.so new file mode 100644 index 0000000000000000000000000000000000000000..74c4f69e1f6c5fc9b7faf6b132d6f2fcd9f82025 --- /dev/null +++ b/build/torch28-cxx11-cu126-x86_64-linux/_rwkv_d5e72cf.abi3.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:39086c2154f82e14222211b31e101bc79834a2cafdebaf749c63bd1e6ed4e8a2 +size 2111504 diff --git a/build/torch28-cxx11-cu126-x86_64-linux/metadata.json b/build/torch28-cxx11-cu126-x86_64-linux/metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..76bafa5f33b6818aa6bb4cab04be811b87519b44 --- /dev/null +++ b/build/torch28-cxx11-cu126-x86_64-linux/metadata.json @@ -0,0 +1 @@ +{"python-depends":[]} \ No newline at end of file diff --git a/build/torch28-cxx11-cu126-x86_64-linux/rwkv/__init__.py b/build/torch28-cxx11-cu126-x86_64-linux/rwkv/__init__.py index d4ea22dbc160d2b75b32c3c8c5d882d25c77b494..03dbc1afe1cf156661a2b1b22003cd5f599a0309 100644 --- a/build/torch28-cxx11-cu126-x86_64-linux/rwkv/__init__.py +++ b/build/torch28-cxx11-cu126-x86_64-linux/rwkv/__init__.py @@ -1,170 +1,26 @@ -from ._ops import ops -from typing import Tuple, Any - -# Use a broad Tensor alias to avoid importing torch at import time. -from torch import Tensor - -def forward(w: Tensor, u: Tensor, k: Tensor, v: Tensor, y: Tensor) -> None: - """RWKV WKV forward pass (float32). - - Runs the CUDA kernel and writes the result into ``y`` in-place. - - Args: - w: Decay weights, shape ``[C]``, dtype ``torch.float32``. - u: Input tensor, shape ``[B, T, C]``, dtype ``torch.float32``. - k: Key tensor, shape ``[B, T, C]``, dtype ``torch.float32``. - v: Value tensor, shape ``[B, T, C]``, dtype ``torch.float32``. - y: Output tensor, shape ``[B, T, C]``, dtype ``torch.float32`` (written in-place). - - Notes: - - All tensors must be on the same CUDA device. - - Shapes must agree on ``B``, ``T`` and ``C``. - """ - _validate_device_match((w, u, k, v, y)) - ops.forward(w, u, k, v, y) - - -def forward_bf16(w: Tensor, u: Tensor, k: Tensor, v: Tensor, y: Tensor) -> None: - """RWKV WKV forward pass (bfloat16 inputs/outputs, float32 ``w``). - - Runs the CUDA kernel and writes the result into ``y`` in-place. - - Args: - w: Decay weights, shape ``[C]``, dtype ``torch.float32``. - u: Input tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16``. - k: Key tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16``. - v: Value tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16``. - y: Output tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16`` (written in-place). - - Notes: - - All tensors must be on the same CUDA device. - - Shapes must agree on ``B``, ``T`` and ``C``. - """ - _validate_device_match((w, u, k, v, y)) - ops.forward_bf16(w, u, k, v, y) - - -def forward_with_state(w: Tensor, u: Tensor, k: Tensor, v: Tensor, y: Tensor, s: Tensor) -> None: - """RWKV WKV forward pass with persistent state (float32). - - Runs the CUDA kernel using and updating state ``s`` and writes the result into ``y``. - - Args: - w: Decay weights, shape ``[C]``, dtype ``torch.float32``. - u: Input tensor, shape ``[B, T, C]``, dtype ``torch.float32``. - k: Key tensor, shape ``[B, T, C]``, dtype ``torch.float32``. - v: Value tensor, shape ``[B, T, C]``, dtype ``torch.float32``. - y: Output tensor, shape ``[B, T, C]``, dtype ``torch.float32`` (written in-place). - s: Stateful tensor, shape ``[B, C]``, dtype ``torch.float32`` (updated in-place). - - Notes: - - All tensors must be on the same CUDA device. - - Shapes must agree on ``B`` and ``C``; ``y`` shares ``[B, T, C]`` with inputs. - """ - _validate_device_match((w, u, k, v, y, s)) - ops.forward_with_state(w, u, k, v, y, s) - - -def forward_with_state_bf16(w: Tensor, u: Tensor, k: Tensor, v: Tensor, y: Tensor, s: Tensor) -> None: - """RWKV WKV forward pass with persistent state (bfloat16 inputs/outputs, float32 ``w`` and ``s``). - - Runs the CUDA kernel using and updating state ``s`` and writes the result into ``y``. - - Args: - w: Decay weights, shape ``[C]``, dtype ``torch.float32``. - u: Input tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16``. - k: Key tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16``. - v: Value tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16``. - y: Output tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16`` (written in-place). - s: Stateful tensor, shape ``[B, C]``, dtype ``torch.float32`` (updated in-place). - - Notes: - - All tensors must be on the same CUDA device. - - Shapes must agree on ``B`` and ``C``; ``y`` shares ``[B, T, C]`` with inputs. - """ - _validate_device_match((w, u, k, v, y, s)) - ops.forward_with_state_bf16(w, u, k, v, y, s) - - -def backward( - w: Tensor, - u: Tensor, - k: Tensor, - v: Tensor, - y: Tensor, - gy: Tensor, - gw: Tensor, - gu: Tensor, - gk: Tensor, - gv: Tensor, -) -> None: - """RWKV WKV backward pass (float32). - - Writes gradients into the provided tensors in-place. - - Args: - w: Decay weights, shape ``[C]``, dtype ``torch.float32``. - u, k, v, y: Forward-pass tensors, shape ``[B, T, C]``, dtype ``torch.float32``. - gy: Gradient of ``y``, shape ``[B, T, C]``, dtype ``torch.float32``. - gw: Gradient for ``w``, shape ``[C]``, dtype ``torch.float32`` (written in-place). - gu, gk, gv: Gradients for ``u``, ``k``, ``v`` respectively, shape ``[B, T, C]``, dtype ``torch.float32`` (written in-place). - - Notes: - - All tensors must be on the same CUDA device. - - Shapes must agree on ``B``, ``T`` and ``C``. - """ - _validate_device_match((w, u, k, v, y, gy, gw, gu, gk, gv)) - ops.backward(w, u, k, v, y, gy, gw, gu, gk, gv) - - -def backward_bf16( - w: Tensor, - u: Tensor, - k: Tensor, - v: Tensor, - y: Tensor, - gy: Tensor, - gw: Tensor, - gu: Tensor, - gk: Tensor, - gv: Tensor, -) -> None: - """RWKV WKV backward pass (bfloat16 inputs/outputs/gradients, float32 ``w``). - - Writes gradients into the provided tensors in-place. - - Args: - w: Decay weights, shape ``[C]``, dtype ``torch.float32``. - u, k, v, y: Forward-pass tensors, shape ``[B, T, C]``, dtype ``torch.bfloat16``. - gy: Gradient of ``y``, shape ``[B, T, C]``, dtype ``torch.bfloat16``. - gw: Gradient for ``w``, shape ``[C]``, dtype ``torch.bfloat16`` (written in-place). - gu, gk, gv: Gradients for ``u``, ``k``, ``v`` respectively, shape ``[B, T, C]``, dtype ``torch.bfloat16`` (written in-place). - - Notes: - - All tensors must be on the same CUDA device. - - Shapes must agree on ``B``, ``T`` and ``C``. - """ - _validate_device_match((w, u, k, v, y, gy, gw, gu, gk, gv)) - ops.backward_bf16(w, u, k, v, y, gy, gw, gu, gk, gv) - - -def _validate_device_match(tensors: Tuple[Tensor, ...]) -> None: - """Minimal runtime validation that all tensors live on the same CUDA device.""" - if not tensors: - return - device = tensors[0].device - if not device.type == "cuda": - raise RuntimeError("RWKV CUDA ops require CUDA tensors") - for t in tensors[1:]: - if t.device != device: - raise RuntimeError("All tensors must be on the same CUDA device") - - -__all__ = [ - "forward", - "forward_bf16", - "forward_with_state", - "forward_with_state_bf16", - "backward", - "backward_bf16", -] \ No newline at end of file +import ctypes +import sys + +import importlib +from pathlib import Path +from types import ModuleType + +def _import_from_path(file_path: Path) -> ModuleType: + # We cannot use the module name as-is, after adding it to `sys.modules`, + # it would also be used for other imports. So, we make a module name that + # depends on the path for it to be unique using the hex-encoded hash of + # the path. + path_hash = "{:x}".format(ctypes.c_size_t(hash(file_path.absolute())).value) + module_name = path_hash + spec = importlib.util.spec_from_file_location(module_name, file_path) + if spec is None: + raise ImportError(f"Cannot load spec for {module_name} from {file_path}") + module = importlib.util.module_from_spec(spec) + if module is None: + raise ImportError(f"Cannot load module {module_name} from spec") + sys.modules[module_name] = module + spec.loader.exec_module(module) # type: ignore + return module + + +globals().update(vars(_import_from_path(Path(__file__).parent.parent / "__init__.py"))) diff --git a/build/torch28-cxx11-cu126-x86_64-linux/rwkv/__pycache__/__init__.cpython-313.pyc b/build/torch28-cxx11-cu126-x86_64-linux/rwkv/__pycache__/__init__.cpython-313.pyc deleted file mode 100644 index afb0f9a9b12e0c6ee4dd6f4fbe1bd12c07209d89..0000000000000000000000000000000000000000 Binary files a/build/torch28-cxx11-cu126-x86_64-linux/rwkv/__pycache__/__init__.cpython-313.pyc and /dev/null differ diff --git a/build/torch28-cxx11-cu126-x86_64-linux/rwkv/__pycache__/_ops.cpython-313.pyc b/build/torch28-cxx11-cu126-x86_64-linux/rwkv/__pycache__/_ops.cpython-313.pyc deleted file mode 100644 index 729c48cba29be14ec8309c50afe9acf6bad61753..0000000000000000000000000000000000000000 Binary files a/build/torch28-cxx11-cu126-x86_64-linux/rwkv/__pycache__/_ops.cpython-313.pyc and /dev/null differ diff --git a/build/torch28-cxx11-cu126-x86_64-linux/rwkv/_rwkv_0aa7cb0.abi3.so b/build/torch28-cxx11-cu126-x86_64-linux/rwkv/_rwkv_0aa7cb0.abi3.so deleted file mode 100644 index d073333251d25ee3867d90c92988f5d411a65eb3..0000000000000000000000000000000000000000 --- a/build/torch28-cxx11-cu126-x86_64-linux/rwkv/_rwkv_0aa7cb0.abi3.so +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:05d32a9de8c5172c0eab7fb796430e20c2c358fc9489655afdffa316553cf8c9 -size 2106440 diff --git a/build/torch28-cxx11-cu128-x86_64-linux/__init__.py b/build/torch28-cxx11-cu128-x86_64-linux/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..d4ea22dbc160d2b75b32c3c8c5d882d25c77b494 --- /dev/null +++ b/build/torch28-cxx11-cu128-x86_64-linux/__init__.py @@ -0,0 +1,170 @@ +from ._ops import ops +from typing import Tuple, Any + +# Use a broad Tensor alias to avoid importing torch at import time. +from torch import Tensor + +def forward(w: Tensor, u: Tensor, k: Tensor, v: Tensor, y: Tensor) -> None: + """RWKV WKV forward pass (float32). + + Runs the CUDA kernel and writes the result into ``y`` in-place. + + Args: + w: Decay weights, shape ``[C]``, dtype ``torch.float32``. + u: Input tensor, shape ``[B, T, C]``, dtype ``torch.float32``. + k: Key tensor, shape ``[B, T, C]``, dtype ``torch.float32``. + v: Value tensor, shape ``[B, T, C]``, dtype ``torch.float32``. + y: Output tensor, shape ``[B, T, C]``, dtype ``torch.float32`` (written in-place). + + Notes: + - All tensors must be on the same CUDA device. + - Shapes must agree on ``B``, ``T`` and ``C``. + """ + _validate_device_match((w, u, k, v, y)) + ops.forward(w, u, k, v, y) + + +def forward_bf16(w: Tensor, u: Tensor, k: Tensor, v: Tensor, y: Tensor) -> None: + """RWKV WKV forward pass (bfloat16 inputs/outputs, float32 ``w``). + + Runs the CUDA kernel and writes the result into ``y`` in-place. + + Args: + w: Decay weights, shape ``[C]``, dtype ``torch.float32``. + u: Input tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16``. + k: Key tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16``. + v: Value tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16``. + y: Output tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16`` (written in-place). + + Notes: + - All tensors must be on the same CUDA device. + - Shapes must agree on ``B``, ``T`` and ``C``. + """ + _validate_device_match((w, u, k, v, y)) + ops.forward_bf16(w, u, k, v, y) + + +def forward_with_state(w: Tensor, u: Tensor, k: Tensor, v: Tensor, y: Tensor, s: Tensor) -> None: + """RWKV WKV forward pass with persistent state (float32). + + Runs the CUDA kernel using and updating state ``s`` and writes the result into ``y``. + + Args: + w: Decay weights, shape ``[C]``, dtype ``torch.float32``. + u: Input tensor, shape ``[B, T, C]``, dtype ``torch.float32``. + k: Key tensor, shape ``[B, T, C]``, dtype ``torch.float32``. + v: Value tensor, shape ``[B, T, C]``, dtype ``torch.float32``. + y: Output tensor, shape ``[B, T, C]``, dtype ``torch.float32`` (written in-place). + s: Stateful tensor, shape ``[B, C]``, dtype ``torch.float32`` (updated in-place). + + Notes: + - All tensors must be on the same CUDA device. + - Shapes must agree on ``B`` and ``C``; ``y`` shares ``[B, T, C]`` with inputs. + """ + _validate_device_match((w, u, k, v, y, s)) + ops.forward_with_state(w, u, k, v, y, s) + + +def forward_with_state_bf16(w: Tensor, u: Tensor, k: Tensor, v: Tensor, y: Tensor, s: Tensor) -> None: + """RWKV WKV forward pass with persistent state (bfloat16 inputs/outputs, float32 ``w`` and ``s``). + + Runs the CUDA kernel using and updating state ``s`` and writes the result into ``y``. + + Args: + w: Decay weights, shape ``[C]``, dtype ``torch.float32``. + u: Input tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16``. + k: Key tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16``. + v: Value tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16``. + y: Output tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16`` (written in-place). + s: Stateful tensor, shape ``[B, C]``, dtype ``torch.float32`` (updated in-place). + + Notes: + - All tensors must be on the same CUDA device. + - Shapes must agree on ``B`` and ``C``; ``y`` shares ``[B, T, C]`` with inputs. + """ + _validate_device_match((w, u, k, v, y, s)) + ops.forward_with_state_bf16(w, u, k, v, y, s) + + +def backward( + w: Tensor, + u: Tensor, + k: Tensor, + v: Tensor, + y: Tensor, + gy: Tensor, + gw: Tensor, + gu: Tensor, + gk: Tensor, + gv: Tensor, +) -> None: + """RWKV WKV backward pass (float32). + + Writes gradients into the provided tensors in-place. + + Args: + w: Decay weights, shape ``[C]``, dtype ``torch.float32``. + u, k, v, y: Forward-pass tensors, shape ``[B, T, C]``, dtype ``torch.float32``. + gy: Gradient of ``y``, shape ``[B, T, C]``, dtype ``torch.float32``. + gw: Gradient for ``w``, shape ``[C]``, dtype ``torch.float32`` (written in-place). + gu, gk, gv: Gradients for ``u``, ``k``, ``v`` respectively, shape ``[B, T, C]``, dtype ``torch.float32`` (written in-place). + + Notes: + - All tensors must be on the same CUDA device. + - Shapes must agree on ``B``, ``T`` and ``C``. + """ + _validate_device_match((w, u, k, v, y, gy, gw, gu, gk, gv)) + ops.backward(w, u, k, v, y, gy, gw, gu, gk, gv) + + +def backward_bf16( + w: Tensor, + u: Tensor, + k: Tensor, + v: Tensor, + y: Tensor, + gy: Tensor, + gw: Tensor, + gu: Tensor, + gk: Tensor, + gv: Tensor, +) -> None: + """RWKV WKV backward pass (bfloat16 inputs/outputs/gradients, float32 ``w``). + + Writes gradients into the provided tensors in-place. + + Args: + w: Decay weights, shape ``[C]``, dtype ``torch.float32``. + u, k, v, y: Forward-pass tensors, shape ``[B, T, C]``, dtype ``torch.bfloat16``. + gy: Gradient of ``y``, shape ``[B, T, C]``, dtype ``torch.bfloat16``. + gw: Gradient for ``w``, shape ``[C]``, dtype ``torch.bfloat16`` (written in-place). + gu, gk, gv: Gradients for ``u``, ``k``, ``v`` respectively, shape ``[B, T, C]``, dtype ``torch.bfloat16`` (written in-place). + + Notes: + - All tensors must be on the same CUDA device. + - Shapes must agree on ``B``, ``T`` and ``C``. + """ + _validate_device_match((w, u, k, v, y, gy, gw, gu, gk, gv)) + ops.backward_bf16(w, u, k, v, y, gy, gw, gu, gk, gv) + + +def _validate_device_match(tensors: Tuple[Tensor, ...]) -> None: + """Minimal runtime validation that all tensors live on the same CUDA device.""" + if not tensors: + return + device = tensors[0].device + if not device.type == "cuda": + raise RuntimeError("RWKV CUDA ops require CUDA tensors") + for t in tensors[1:]: + if t.device != device: + raise RuntimeError("All tensors must be on the same CUDA device") + + +__all__ = [ + "forward", + "forward_bf16", + "forward_with_state", + "forward_with_state_bf16", + "backward", + "backward_bf16", +] \ No newline at end of file diff --git a/build/torch28-cxx11-cu128-x86_64-linux/_ops.py b/build/torch28-cxx11-cu128-x86_64-linux/_ops.py new file mode 100644 index 0000000000000000000000000000000000000000..38c698ed4f2208d589ead1ef3852c6019d3eb01c --- /dev/null +++ b/build/torch28-cxx11-cu128-x86_64-linux/_ops.py @@ -0,0 +1,9 @@ +import torch +from . import _rwkv_d5e72cf +ops = torch.ops._rwkv_d5e72cf + +def add_op_namespace_prefix(op_name: str): + """ + Prefix op by namespace. + """ + return f"_rwkv_d5e72cf::{op_name}" \ No newline at end of file diff --git a/build/torch28-cxx11-cu128-x86_64-linux/_rwkv_d5e72cf.abi3.so b/build/torch28-cxx11-cu128-x86_64-linux/_rwkv_d5e72cf.abi3.so new file mode 100644 index 0000000000000000000000000000000000000000..d35758ad54f5a57fa3560f600b60c6e403e40669 --- /dev/null +++ b/build/torch28-cxx11-cu128-x86_64-linux/_rwkv_d5e72cf.abi3.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a65b5086f8d8c9a29a210f5c37e68e97712d68e8884fd3084a2d3005a2c54dd +size 2318032 diff --git a/build/torch28-cxx11-cu128-x86_64-linux/metadata.json b/build/torch28-cxx11-cu128-x86_64-linux/metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..76bafa5f33b6818aa6bb4cab04be811b87519b44 --- /dev/null +++ b/build/torch28-cxx11-cu128-x86_64-linux/metadata.json @@ -0,0 +1 @@ +{"python-depends":[]} \ No newline at end of file diff --git a/build/torch28-cxx11-cu128-x86_64-linux/rwkv/__init__.py b/build/torch28-cxx11-cu128-x86_64-linux/rwkv/__init__.py index d4ea22dbc160d2b75b32c3c8c5d882d25c77b494..03dbc1afe1cf156661a2b1b22003cd5f599a0309 100644 --- a/build/torch28-cxx11-cu128-x86_64-linux/rwkv/__init__.py +++ b/build/torch28-cxx11-cu128-x86_64-linux/rwkv/__init__.py @@ -1,170 +1,26 @@ -from ._ops import ops -from typing import Tuple, Any - -# Use a broad Tensor alias to avoid importing torch at import time. -from torch import Tensor - -def forward(w: Tensor, u: Tensor, k: Tensor, v: Tensor, y: Tensor) -> None: - """RWKV WKV forward pass (float32). - - Runs the CUDA kernel and writes the result into ``y`` in-place. - - Args: - w: Decay weights, shape ``[C]``, dtype ``torch.float32``. - u: Input tensor, shape ``[B, T, C]``, dtype ``torch.float32``. - k: Key tensor, shape ``[B, T, C]``, dtype ``torch.float32``. - v: Value tensor, shape ``[B, T, C]``, dtype ``torch.float32``. - y: Output tensor, shape ``[B, T, C]``, dtype ``torch.float32`` (written in-place). - - Notes: - - All tensors must be on the same CUDA device. - - Shapes must agree on ``B``, ``T`` and ``C``. - """ - _validate_device_match((w, u, k, v, y)) - ops.forward(w, u, k, v, y) - - -def forward_bf16(w: Tensor, u: Tensor, k: Tensor, v: Tensor, y: Tensor) -> None: - """RWKV WKV forward pass (bfloat16 inputs/outputs, float32 ``w``). - - Runs the CUDA kernel and writes the result into ``y`` in-place. - - Args: - w: Decay weights, shape ``[C]``, dtype ``torch.float32``. - u: Input tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16``. - k: Key tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16``. - v: Value tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16``. - y: Output tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16`` (written in-place). - - Notes: - - All tensors must be on the same CUDA device. - - Shapes must agree on ``B``, ``T`` and ``C``. - """ - _validate_device_match((w, u, k, v, y)) - ops.forward_bf16(w, u, k, v, y) - - -def forward_with_state(w: Tensor, u: Tensor, k: Tensor, v: Tensor, y: Tensor, s: Tensor) -> None: - """RWKV WKV forward pass with persistent state (float32). - - Runs the CUDA kernel using and updating state ``s`` and writes the result into ``y``. - - Args: - w: Decay weights, shape ``[C]``, dtype ``torch.float32``. - u: Input tensor, shape ``[B, T, C]``, dtype ``torch.float32``. - k: Key tensor, shape ``[B, T, C]``, dtype ``torch.float32``. - v: Value tensor, shape ``[B, T, C]``, dtype ``torch.float32``. - y: Output tensor, shape ``[B, T, C]``, dtype ``torch.float32`` (written in-place). - s: Stateful tensor, shape ``[B, C]``, dtype ``torch.float32`` (updated in-place). - - Notes: - - All tensors must be on the same CUDA device. - - Shapes must agree on ``B`` and ``C``; ``y`` shares ``[B, T, C]`` with inputs. - """ - _validate_device_match((w, u, k, v, y, s)) - ops.forward_with_state(w, u, k, v, y, s) - - -def forward_with_state_bf16(w: Tensor, u: Tensor, k: Tensor, v: Tensor, y: Tensor, s: Tensor) -> None: - """RWKV WKV forward pass with persistent state (bfloat16 inputs/outputs, float32 ``w`` and ``s``). - - Runs the CUDA kernel using and updating state ``s`` and writes the result into ``y``. - - Args: - w: Decay weights, shape ``[C]``, dtype ``torch.float32``. - u: Input tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16``. - k: Key tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16``. - v: Value tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16``. - y: Output tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16`` (written in-place). - s: Stateful tensor, shape ``[B, C]``, dtype ``torch.float32`` (updated in-place). - - Notes: - - All tensors must be on the same CUDA device. - - Shapes must agree on ``B`` and ``C``; ``y`` shares ``[B, T, C]`` with inputs. - """ - _validate_device_match((w, u, k, v, y, s)) - ops.forward_with_state_bf16(w, u, k, v, y, s) - - -def backward( - w: Tensor, - u: Tensor, - k: Tensor, - v: Tensor, - y: Tensor, - gy: Tensor, - gw: Tensor, - gu: Tensor, - gk: Tensor, - gv: Tensor, -) -> None: - """RWKV WKV backward pass (float32). - - Writes gradients into the provided tensors in-place. - - Args: - w: Decay weights, shape ``[C]``, dtype ``torch.float32``. - u, k, v, y: Forward-pass tensors, shape ``[B, T, C]``, dtype ``torch.float32``. - gy: Gradient of ``y``, shape ``[B, T, C]``, dtype ``torch.float32``. - gw: Gradient for ``w``, shape ``[C]``, dtype ``torch.float32`` (written in-place). - gu, gk, gv: Gradients for ``u``, ``k``, ``v`` respectively, shape ``[B, T, C]``, dtype ``torch.float32`` (written in-place). - - Notes: - - All tensors must be on the same CUDA device. - - Shapes must agree on ``B``, ``T`` and ``C``. - """ - _validate_device_match((w, u, k, v, y, gy, gw, gu, gk, gv)) - ops.backward(w, u, k, v, y, gy, gw, gu, gk, gv) - - -def backward_bf16( - w: Tensor, - u: Tensor, - k: Tensor, - v: Tensor, - y: Tensor, - gy: Tensor, - gw: Tensor, - gu: Tensor, - gk: Tensor, - gv: Tensor, -) -> None: - """RWKV WKV backward pass (bfloat16 inputs/outputs/gradients, float32 ``w``). - - Writes gradients into the provided tensors in-place. - - Args: - w: Decay weights, shape ``[C]``, dtype ``torch.float32``. - u, k, v, y: Forward-pass tensors, shape ``[B, T, C]``, dtype ``torch.bfloat16``. - gy: Gradient of ``y``, shape ``[B, T, C]``, dtype ``torch.bfloat16``. - gw: Gradient for ``w``, shape ``[C]``, dtype ``torch.bfloat16`` (written in-place). - gu, gk, gv: Gradients for ``u``, ``k``, ``v`` respectively, shape ``[B, T, C]``, dtype ``torch.bfloat16`` (written in-place). - - Notes: - - All tensors must be on the same CUDA device. - - Shapes must agree on ``B``, ``T`` and ``C``. - """ - _validate_device_match((w, u, k, v, y, gy, gw, gu, gk, gv)) - ops.backward_bf16(w, u, k, v, y, gy, gw, gu, gk, gv) - - -def _validate_device_match(tensors: Tuple[Tensor, ...]) -> None: - """Minimal runtime validation that all tensors live on the same CUDA device.""" - if not tensors: - return - device = tensors[0].device - if not device.type == "cuda": - raise RuntimeError("RWKV CUDA ops require CUDA tensors") - for t in tensors[1:]: - if t.device != device: - raise RuntimeError("All tensors must be on the same CUDA device") - - -__all__ = [ - "forward", - "forward_bf16", - "forward_with_state", - "forward_with_state_bf16", - "backward", - "backward_bf16", -] \ No newline at end of file +import ctypes +import sys + +import importlib +from pathlib import Path +from types import ModuleType + +def _import_from_path(file_path: Path) -> ModuleType: + # We cannot use the module name as-is, after adding it to `sys.modules`, + # it would also be used for other imports. So, we make a module name that + # depends on the path for it to be unique using the hex-encoded hash of + # the path. + path_hash = "{:x}".format(ctypes.c_size_t(hash(file_path.absolute())).value) + module_name = path_hash + spec = importlib.util.spec_from_file_location(module_name, file_path) + if spec is None: + raise ImportError(f"Cannot load spec for {module_name} from {file_path}") + module = importlib.util.module_from_spec(spec) + if module is None: + raise ImportError(f"Cannot load module {module_name} from spec") + sys.modules[module_name] = module + spec.loader.exec_module(module) # type: ignore + return module + + +globals().update(vars(_import_from_path(Path(__file__).parent.parent / "__init__.py"))) diff --git a/build/torch28-cxx11-cu128-x86_64-linux/rwkv/__pycache__/__init__.cpython-313.pyc b/build/torch28-cxx11-cu128-x86_64-linux/rwkv/__pycache__/__init__.cpython-313.pyc deleted file mode 100644 index 0b5cc392393b5f8caa5d1a61c225dc995209c939..0000000000000000000000000000000000000000 Binary files a/build/torch28-cxx11-cu128-x86_64-linux/rwkv/__pycache__/__init__.cpython-313.pyc and /dev/null differ diff --git a/build/torch28-cxx11-cu128-x86_64-linux/rwkv/__pycache__/_ops.cpython-313.pyc b/build/torch28-cxx11-cu128-x86_64-linux/rwkv/__pycache__/_ops.cpython-313.pyc deleted file mode 100644 index 10e11cb78cb3f5936183558e5bb7970bf9fd447a..0000000000000000000000000000000000000000 Binary files a/build/torch28-cxx11-cu128-x86_64-linux/rwkv/__pycache__/_ops.cpython-313.pyc and /dev/null differ diff --git a/build/torch28-cxx11-cu128-x86_64-linux/rwkv/_rwkv_0aa7cb0.abi3.so b/build/torch28-cxx11-cu128-x86_64-linux/rwkv/_rwkv_0aa7cb0.abi3.so deleted file mode 100644 index 3b610e299e73df625299c1485415c3f2dcaedae6..0000000000000000000000000000000000000000 --- a/build/torch28-cxx11-cu128-x86_64-linux/rwkv/_rwkv_0aa7cb0.abi3.so +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:c41dad15dbf8c85fd000ac0fd4e51a0772cc0bcc91f2abc85e57e43356b717bd -size 2308848 diff --git a/build/torch28-cxx11-cu129-x86_64-linux/__init__.py b/build/torch28-cxx11-cu129-x86_64-linux/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..d4ea22dbc160d2b75b32c3c8c5d882d25c77b494 --- /dev/null +++ b/build/torch28-cxx11-cu129-x86_64-linux/__init__.py @@ -0,0 +1,170 @@ +from ._ops import ops +from typing import Tuple, Any + +# Use a broad Tensor alias to avoid importing torch at import time. +from torch import Tensor + +def forward(w: Tensor, u: Tensor, k: Tensor, v: Tensor, y: Tensor) -> None: + """RWKV WKV forward pass (float32). + + Runs the CUDA kernel and writes the result into ``y`` in-place. + + Args: + w: Decay weights, shape ``[C]``, dtype ``torch.float32``. + u: Input tensor, shape ``[B, T, C]``, dtype ``torch.float32``. + k: Key tensor, shape ``[B, T, C]``, dtype ``torch.float32``. + v: Value tensor, shape ``[B, T, C]``, dtype ``torch.float32``. + y: Output tensor, shape ``[B, T, C]``, dtype ``torch.float32`` (written in-place). + + Notes: + - All tensors must be on the same CUDA device. + - Shapes must agree on ``B``, ``T`` and ``C``. + """ + _validate_device_match((w, u, k, v, y)) + ops.forward(w, u, k, v, y) + + +def forward_bf16(w: Tensor, u: Tensor, k: Tensor, v: Tensor, y: Tensor) -> None: + """RWKV WKV forward pass (bfloat16 inputs/outputs, float32 ``w``). + + Runs the CUDA kernel and writes the result into ``y`` in-place. + + Args: + w: Decay weights, shape ``[C]``, dtype ``torch.float32``. + u: Input tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16``. + k: Key tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16``. + v: Value tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16``. + y: Output tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16`` (written in-place). + + Notes: + - All tensors must be on the same CUDA device. + - Shapes must agree on ``B``, ``T`` and ``C``. + """ + _validate_device_match((w, u, k, v, y)) + ops.forward_bf16(w, u, k, v, y) + + +def forward_with_state(w: Tensor, u: Tensor, k: Tensor, v: Tensor, y: Tensor, s: Tensor) -> None: + """RWKV WKV forward pass with persistent state (float32). + + Runs the CUDA kernel using and updating state ``s`` and writes the result into ``y``. + + Args: + w: Decay weights, shape ``[C]``, dtype ``torch.float32``. + u: Input tensor, shape ``[B, T, C]``, dtype ``torch.float32``. + k: Key tensor, shape ``[B, T, C]``, dtype ``torch.float32``. + v: Value tensor, shape ``[B, T, C]``, dtype ``torch.float32``. + y: Output tensor, shape ``[B, T, C]``, dtype ``torch.float32`` (written in-place). + s: Stateful tensor, shape ``[B, C]``, dtype ``torch.float32`` (updated in-place). + + Notes: + - All tensors must be on the same CUDA device. + - Shapes must agree on ``B`` and ``C``; ``y`` shares ``[B, T, C]`` with inputs. + """ + _validate_device_match((w, u, k, v, y, s)) + ops.forward_with_state(w, u, k, v, y, s) + + +def forward_with_state_bf16(w: Tensor, u: Tensor, k: Tensor, v: Tensor, y: Tensor, s: Tensor) -> None: + """RWKV WKV forward pass with persistent state (bfloat16 inputs/outputs, float32 ``w`` and ``s``). + + Runs the CUDA kernel using and updating state ``s`` and writes the result into ``y``. + + Args: + w: Decay weights, shape ``[C]``, dtype ``torch.float32``. + u: Input tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16``. + k: Key tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16``. + v: Value tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16``. + y: Output tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16`` (written in-place). + s: Stateful tensor, shape ``[B, C]``, dtype ``torch.float32`` (updated in-place). + + Notes: + - All tensors must be on the same CUDA device. + - Shapes must agree on ``B`` and ``C``; ``y`` shares ``[B, T, C]`` with inputs. + """ + _validate_device_match((w, u, k, v, y, s)) + ops.forward_with_state_bf16(w, u, k, v, y, s) + + +def backward( + w: Tensor, + u: Tensor, + k: Tensor, + v: Tensor, + y: Tensor, + gy: Tensor, + gw: Tensor, + gu: Tensor, + gk: Tensor, + gv: Tensor, +) -> None: + """RWKV WKV backward pass (float32). + + Writes gradients into the provided tensors in-place. + + Args: + w: Decay weights, shape ``[C]``, dtype ``torch.float32``. + u, k, v, y: Forward-pass tensors, shape ``[B, T, C]``, dtype ``torch.float32``. + gy: Gradient of ``y``, shape ``[B, T, C]``, dtype ``torch.float32``. + gw: Gradient for ``w``, shape ``[C]``, dtype ``torch.float32`` (written in-place). + gu, gk, gv: Gradients for ``u``, ``k``, ``v`` respectively, shape ``[B, T, C]``, dtype ``torch.float32`` (written in-place). + + Notes: + - All tensors must be on the same CUDA device. + - Shapes must agree on ``B``, ``T`` and ``C``. + """ + _validate_device_match((w, u, k, v, y, gy, gw, gu, gk, gv)) + ops.backward(w, u, k, v, y, gy, gw, gu, gk, gv) + + +def backward_bf16( + w: Tensor, + u: Tensor, + k: Tensor, + v: Tensor, + y: Tensor, + gy: Tensor, + gw: Tensor, + gu: Tensor, + gk: Tensor, + gv: Tensor, +) -> None: + """RWKV WKV backward pass (bfloat16 inputs/outputs/gradients, float32 ``w``). + + Writes gradients into the provided tensors in-place. + + Args: + w: Decay weights, shape ``[C]``, dtype ``torch.float32``. + u, k, v, y: Forward-pass tensors, shape ``[B, T, C]``, dtype ``torch.bfloat16``. + gy: Gradient of ``y``, shape ``[B, T, C]``, dtype ``torch.bfloat16``. + gw: Gradient for ``w``, shape ``[C]``, dtype ``torch.bfloat16`` (written in-place). + gu, gk, gv: Gradients for ``u``, ``k``, ``v`` respectively, shape ``[B, T, C]``, dtype ``torch.bfloat16`` (written in-place). + + Notes: + - All tensors must be on the same CUDA device. + - Shapes must agree on ``B``, ``T`` and ``C``. + """ + _validate_device_match((w, u, k, v, y, gy, gw, gu, gk, gv)) + ops.backward_bf16(w, u, k, v, y, gy, gw, gu, gk, gv) + + +def _validate_device_match(tensors: Tuple[Tensor, ...]) -> None: + """Minimal runtime validation that all tensors live on the same CUDA device.""" + if not tensors: + return + device = tensors[0].device + if not device.type == "cuda": + raise RuntimeError("RWKV CUDA ops require CUDA tensors") + for t in tensors[1:]: + if t.device != device: + raise RuntimeError("All tensors must be on the same CUDA device") + + +__all__ = [ + "forward", + "forward_bf16", + "forward_with_state", + "forward_with_state_bf16", + "backward", + "backward_bf16", +] \ No newline at end of file diff --git a/build/torch28-cxx11-cu129-x86_64-linux/_ops.py b/build/torch28-cxx11-cu129-x86_64-linux/_ops.py new file mode 100644 index 0000000000000000000000000000000000000000..38c698ed4f2208d589ead1ef3852c6019d3eb01c --- /dev/null +++ b/build/torch28-cxx11-cu129-x86_64-linux/_ops.py @@ -0,0 +1,9 @@ +import torch +from . import _rwkv_d5e72cf +ops = torch.ops._rwkv_d5e72cf + +def add_op_namespace_prefix(op_name: str): + """ + Prefix op by namespace. + """ + return f"_rwkv_d5e72cf::{op_name}" \ No newline at end of file diff --git a/build/torch28-cxx11-cu129-x86_64-linux/_rwkv_d5e72cf.abi3.so b/build/torch28-cxx11-cu129-x86_64-linux/_rwkv_d5e72cf.abi3.so new file mode 100644 index 0000000000000000000000000000000000000000..0003abb6973ce68efcf8ec58161fd2e53de4fc58 --- /dev/null +++ b/build/torch28-cxx11-cu129-x86_64-linux/_rwkv_d5e72cf.abi3.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:43c6ab910e3286ed53a60ca6cc4292b15734bb26c706e297d366313d07f67266 +size 2335432 diff --git a/build/torch28-cxx11-cu129-x86_64-linux/metadata.json b/build/torch28-cxx11-cu129-x86_64-linux/metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..76bafa5f33b6818aa6bb4cab04be811b87519b44 --- /dev/null +++ b/build/torch28-cxx11-cu129-x86_64-linux/metadata.json @@ -0,0 +1 @@ +{"python-depends":[]} \ No newline at end of file diff --git a/build/torch28-cxx11-cu129-x86_64-linux/rwkv/__init__.py b/build/torch28-cxx11-cu129-x86_64-linux/rwkv/__init__.py index d4ea22dbc160d2b75b32c3c8c5d882d25c77b494..03dbc1afe1cf156661a2b1b22003cd5f599a0309 100644 --- a/build/torch28-cxx11-cu129-x86_64-linux/rwkv/__init__.py +++ b/build/torch28-cxx11-cu129-x86_64-linux/rwkv/__init__.py @@ -1,170 +1,26 @@ -from ._ops import ops -from typing import Tuple, Any - -# Use a broad Tensor alias to avoid importing torch at import time. -from torch import Tensor - -def forward(w: Tensor, u: Tensor, k: Tensor, v: Tensor, y: Tensor) -> None: - """RWKV WKV forward pass (float32). - - Runs the CUDA kernel and writes the result into ``y`` in-place. - - Args: - w: Decay weights, shape ``[C]``, dtype ``torch.float32``. - u: Input tensor, shape ``[B, T, C]``, dtype ``torch.float32``. - k: Key tensor, shape ``[B, T, C]``, dtype ``torch.float32``. - v: Value tensor, shape ``[B, T, C]``, dtype ``torch.float32``. - y: Output tensor, shape ``[B, T, C]``, dtype ``torch.float32`` (written in-place). - - Notes: - - All tensors must be on the same CUDA device. - - Shapes must agree on ``B``, ``T`` and ``C``. - """ - _validate_device_match((w, u, k, v, y)) - ops.forward(w, u, k, v, y) - - -def forward_bf16(w: Tensor, u: Tensor, k: Tensor, v: Tensor, y: Tensor) -> None: - """RWKV WKV forward pass (bfloat16 inputs/outputs, float32 ``w``). - - Runs the CUDA kernel and writes the result into ``y`` in-place. - - Args: - w: Decay weights, shape ``[C]``, dtype ``torch.float32``. - u: Input tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16``. - k: Key tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16``. - v: Value tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16``. - y: Output tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16`` (written in-place). - - Notes: - - All tensors must be on the same CUDA device. - - Shapes must agree on ``B``, ``T`` and ``C``. - """ - _validate_device_match((w, u, k, v, y)) - ops.forward_bf16(w, u, k, v, y) - - -def forward_with_state(w: Tensor, u: Tensor, k: Tensor, v: Tensor, y: Tensor, s: Tensor) -> None: - """RWKV WKV forward pass with persistent state (float32). - - Runs the CUDA kernel using and updating state ``s`` and writes the result into ``y``. - - Args: - w: Decay weights, shape ``[C]``, dtype ``torch.float32``. - u: Input tensor, shape ``[B, T, C]``, dtype ``torch.float32``. - k: Key tensor, shape ``[B, T, C]``, dtype ``torch.float32``. - v: Value tensor, shape ``[B, T, C]``, dtype ``torch.float32``. - y: Output tensor, shape ``[B, T, C]``, dtype ``torch.float32`` (written in-place). - s: Stateful tensor, shape ``[B, C]``, dtype ``torch.float32`` (updated in-place). - - Notes: - - All tensors must be on the same CUDA device. - - Shapes must agree on ``B`` and ``C``; ``y`` shares ``[B, T, C]`` with inputs. - """ - _validate_device_match((w, u, k, v, y, s)) - ops.forward_with_state(w, u, k, v, y, s) - - -def forward_with_state_bf16(w: Tensor, u: Tensor, k: Tensor, v: Tensor, y: Tensor, s: Tensor) -> None: - """RWKV WKV forward pass with persistent state (bfloat16 inputs/outputs, float32 ``w`` and ``s``). - - Runs the CUDA kernel using and updating state ``s`` and writes the result into ``y``. - - Args: - w: Decay weights, shape ``[C]``, dtype ``torch.float32``. - u: Input tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16``. - k: Key tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16``. - v: Value tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16``. - y: Output tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16`` (written in-place). - s: Stateful tensor, shape ``[B, C]``, dtype ``torch.float32`` (updated in-place). - - Notes: - - All tensors must be on the same CUDA device. - - Shapes must agree on ``B`` and ``C``; ``y`` shares ``[B, T, C]`` with inputs. - """ - _validate_device_match((w, u, k, v, y, s)) - ops.forward_with_state_bf16(w, u, k, v, y, s) - - -def backward( - w: Tensor, - u: Tensor, - k: Tensor, - v: Tensor, - y: Tensor, - gy: Tensor, - gw: Tensor, - gu: Tensor, - gk: Tensor, - gv: Tensor, -) -> None: - """RWKV WKV backward pass (float32). - - Writes gradients into the provided tensors in-place. - - Args: - w: Decay weights, shape ``[C]``, dtype ``torch.float32``. - u, k, v, y: Forward-pass tensors, shape ``[B, T, C]``, dtype ``torch.float32``. - gy: Gradient of ``y``, shape ``[B, T, C]``, dtype ``torch.float32``. - gw: Gradient for ``w``, shape ``[C]``, dtype ``torch.float32`` (written in-place). - gu, gk, gv: Gradients for ``u``, ``k``, ``v`` respectively, shape ``[B, T, C]``, dtype ``torch.float32`` (written in-place). - - Notes: - - All tensors must be on the same CUDA device. - - Shapes must agree on ``B``, ``T`` and ``C``. - """ - _validate_device_match((w, u, k, v, y, gy, gw, gu, gk, gv)) - ops.backward(w, u, k, v, y, gy, gw, gu, gk, gv) - - -def backward_bf16( - w: Tensor, - u: Tensor, - k: Tensor, - v: Tensor, - y: Tensor, - gy: Tensor, - gw: Tensor, - gu: Tensor, - gk: Tensor, - gv: Tensor, -) -> None: - """RWKV WKV backward pass (bfloat16 inputs/outputs/gradients, float32 ``w``). - - Writes gradients into the provided tensors in-place. - - Args: - w: Decay weights, shape ``[C]``, dtype ``torch.float32``. - u, k, v, y: Forward-pass tensors, shape ``[B, T, C]``, dtype ``torch.bfloat16``. - gy: Gradient of ``y``, shape ``[B, T, C]``, dtype ``torch.bfloat16``. - gw: Gradient for ``w``, shape ``[C]``, dtype ``torch.bfloat16`` (written in-place). - gu, gk, gv: Gradients for ``u``, ``k``, ``v`` respectively, shape ``[B, T, C]``, dtype ``torch.bfloat16`` (written in-place). - - Notes: - - All tensors must be on the same CUDA device. - - Shapes must agree on ``B``, ``T`` and ``C``. - """ - _validate_device_match((w, u, k, v, y, gy, gw, gu, gk, gv)) - ops.backward_bf16(w, u, k, v, y, gy, gw, gu, gk, gv) - - -def _validate_device_match(tensors: Tuple[Tensor, ...]) -> None: - """Minimal runtime validation that all tensors live on the same CUDA device.""" - if not tensors: - return - device = tensors[0].device - if not device.type == "cuda": - raise RuntimeError("RWKV CUDA ops require CUDA tensors") - for t in tensors[1:]: - if t.device != device: - raise RuntimeError("All tensors must be on the same CUDA device") - - -__all__ = [ - "forward", - "forward_bf16", - "forward_with_state", - "forward_with_state_bf16", - "backward", - "backward_bf16", -] \ No newline at end of file +import ctypes +import sys + +import importlib +from pathlib import Path +from types import ModuleType + +def _import_from_path(file_path: Path) -> ModuleType: + # We cannot use the module name as-is, after adding it to `sys.modules`, + # it would also be used for other imports. So, we make a module name that + # depends on the path for it to be unique using the hex-encoded hash of + # the path. + path_hash = "{:x}".format(ctypes.c_size_t(hash(file_path.absolute())).value) + module_name = path_hash + spec = importlib.util.spec_from_file_location(module_name, file_path) + if spec is None: + raise ImportError(f"Cannot load spec for {module_name} from {file_path}") + module = importlib.util.module_from_spec(spec) + if module is None: + raise ImportError(f"Cannot load module {module_name} from spec") + sys.modules[module_name] = module + spec.loader.exec_module(module) # type: ignore + return module + + +globals().update(vars(_import_from_path(Path(__file__).parent.parent / "__init__.py"))) diff --git a/build/torch28-cxx11-cu129-x86_64-linux/rwkv/__pycache__/__init__.cpython-313.pyc b/build/torch28-cxx11-cu129-x86_64-linux/rwkv/__pycache__/__init__.cpython-313.pyc deleted file mode 100644 index d05e2cb26dd126e4b099e9199df47b89fb06aef7..0000000000000000000000000000000000000000 Binary files a/build/torch28-cxx11-cu129-x86_64-linux/rwkv/__pycache__/__init__.cpython-313.pyc and /dev/null differ diff --git a/build/torch28-cxx11-cu129-x86_64-linux/rwkv/__pycache__/_ops.cpython-313.pyc b/build/torch28-cxx11-cu129-x86_64-linux/rwkv/__pycache__/_ops.cpython-313.pyc deleted file mode 100644 index 73ca3824b521b3696ac6260932103cd9775f7e99..0000000000000000000000000000000000000000 Binary files a/build/torch28-cxx11-cu129-x86_64-linux/rwkv/__pycache__/_ops.cpython-313.pyc and /dev/null differ diff --git a/build/torch28-cxx11-cu129-x86_64-linux/rwkv/_rwkv_0aa7cb0.abi3.so b/build/torch28-cxx11-cu129-x86_64-linux/rwkv/_rwkv_0aa7cb0.abi3.so deleted file mode 100644 index 69d1dc828a3f21b3870e82b3eedbb36b8a1005b3..0000000000000000000000000000000000000000 --- a/build/torch28-cxx11-cu129-x86_64-linux/rwkv/_rwkv_0aa7cb0.abi3.so +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:1e4b4a6455eb8e2736a3b1eb3c7f07bd8e4e0c90e47f0b959961ae3fd93a07e4 -size 2330344 diff --git a/build/torch29-cxx11-cu126-x86_64-linux/__init__.py b/build/torch29-cxx11-cu126-x86_64-linux/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..d4ea22dbc160d2b75b32c3c8c5d882d25c77b494 --- /dev/null +++ b/build/torch29-cxx11-cu126-x86_64-linux/__init__.py @@ -0,0 +1,170 @@ +from ._ops import ops +from typing import Tuple, Any + +# Use a broad Tensor alias to avoid importing torch at import time. +from torch import Tensor + +def forward(w: Tensor, u: Tensor, k: Tensor, v: Tensor, y: Tensor) -> None: + """RWKV WKV forward pass (float32). + + Runs the CUDA kernel and writes the result into ``y`` in-place. + + Args: + w: Decay weights, shape ``[C]``, dtype ``torch.float32``. + u: Input tensor, shape ``[B, T, C]``, dtype ``torch.float32``. + k: Key tensor, shape ``[B, T, C]``, dtype ``torch.float32``. + v: Value tensor, shape ``[B, T, C]``, dtype ``torch.float32``. + y: Output tensor, shape ``[B, T, C]``, dtype ``torch.float32`` (written in-place). + + Notes: + - All tensors must be on the same CUDA device. + - Shapes must agree on ``B``, ``T`` and ``C``. + """ + _validate_device_match((w, u, k, v, y)) + ops.forward(w, u, k, v, y) + + +def forward_bf16(w: Tensor, u: Tensor, k: Tensor, v: Tensor, y: Tensor) -> None: + """RWKV WKV forward pass (bfloat16 inputs/outputs, float32 ``w``). + + Runs the CUDA kernel and writes the result into ``y`` in-place. + + Args: + w: Decay weights, shape ``[C]``, dtype ``torch.float32``. + u: Input tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16``. + k: Key tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16``. + v: Value tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16``. + y: Output tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16`` (written in-place). + + Notes: + - All tensors must be on the same CUDA device. + - Shapes must agree on ``B``, ``T`` and ``C``. + """ + _validate_device_match((w, u, k, v, y)) + ops.forward_bf16(w, u, k, v, y) + + +def forward_with_state(w: Tensor, u: Tensor, k: Tensor, v: Tensor, y: Tensor, s: Tensor) -> None: + """RWKV WKV forward pass with persistent state (float32). + + Runs the CUDA kernel using and updating state ``s`` and writes the result into ``y``. + + Args: + w: Decay weights, shape ``[C]``, dtype ``torch.float32``. + u: Input tensor, shape ``[B, T, C]``, dtype ``torch.float32``. + k: Key tensor, shape ``[B, T, C]``, dtype ``torch.float32``. + v: Value tensor, shape ``[B, T, C]``, dtype ``torch.float32``. + y: Output tensor, shape ``[B, T, C]``, dtype ``torch.float32`` (written in-place). + s: Stateful tensor, shape ``[B, C]``, dtype ``torch.float32`` (updated in-place). + + Notes: + - All tensors must be on the same CUDA device. + - Shapes must agree on ``B`` and ``C``; ``y`` shares ``[B, T, C]`` with inputs. + """ + _validate_device_match((w, u, k, v, y, s)) + ops.forward_with_state(w, u, k, v, y, s) + + +def forward_with_state_bf16(w: Tensor, u: Tensor, k: Tensor, v: Tensor, y: Tensor, s: Tensor) -> None: + """RWKV WKV forward pass with persistent state (bfloat16 inputs/outputs, float32 ``w`` and ``s``). + + Runs the CUDA kernel using and updating state ``s`` and writes the result into ``y``. + + Args: + w: Decay weights, shape ``[C]``, dtype ``torch.float32``. + u: Input tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16``. + k: Key tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16``. + v: Value tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16``. + y: Output tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16`` (written in-place). + s: Stateful tensor, shape ``[B, C]``, dtype ``torch.float32`` (updated in-place). + + Notes: + - All tensors must be on the same CUDA device. + - Shapes must agree on ``B`` and ``C``; ``y`` shares ``[B, T, C]`` with inputs. + """ + _validate_device_match((w, u, k, v, y, s)) + ops.forward_with_state_bf16(w, u, k, v, y, s) + + +def backward( + w: Tensor, + u: Tensor, + k: Tensor, + v: Tensor, + y: Tensor, + gy: Tensor, + gw: Tensor, + gu: Tensor, + gk: Tensor, + gv: Tensor, +) -> None: + """RWKV WKV backward pass (float32). + + Writes gradients into the provided tensors in-place. + + Args: + w: Decay weights, shape ``[C]``, dtype ``torch.float32``. + u, k, v, y: Forward-pass tensors, shape ``[B, T, C]``, dtype ``torch.float32``. + gy: Gradient of ``y``, shape ``[B, T, C]``, dtype ``torch.float32``. + gw: Gradient for ``w``, shape ``[C]``, dtype ``torch.float32`` (written in-place). + gu, gk, gv: Gradients for ``u``, ``k``, ``v`` respectively, shape ``[B, T, C]``, dtype ``torch.float32`` (written in-place). + + Notes: + - All tensors must be on the same CUDA device. + - Shapes must agree on ``B``, ``T`` and ``C``. + """ + _validate_device_match((w, u, k, v, y, gy, gw, gu, gk, gv)) + ops.backward(w, u, k, v, y, gy, gw, gu, gk, gv) + + +def backward_bf16( + w: Tensor, + u: Tensor, + k: Tensor, + v: Tensor, + y: Tensor, + gy: Tensor, + gw: Tensor, + gu: Tensor, + gk: Tensor, + gv: Tensor, +) -> None: + """RWKV WKV backward pass (bfloat16 inputs/outputs/gradients, float32 ``w``). + + Writes gradients into the provided tensors in-place. + + Args: + w: Decay weights, shape ``[C]``, dtype ``torch.float32``. + u, k, v, y: Forward-pass tensors, shape ``[B, T, C]``, dtype ``torch.bfloat16``. + gy: Gradient of ``y``, shape ``[B, T, C]``, dtype ``torch.bfloat16``. + gw: Gradient for ``w``, shape ``[C]``, dtype ``torch.bfloat16`` (written in-place). + gu, gk, gv: Gradients for ``u``, ``k``, ``v`` respectively, shape ``[B, T, C]``, dtype ``torch.bfloat16`` (written in-place). + + Notes: + - All tensors must be on the same CUDA device. + - Shapes must agree on ``B``, ``T`` and ``C``. + """ + _validate_device_match((w, u, k, v, y, gy, gw, gu, gk, gv)) + ops.backward_bf16(w, u, k, v, y, gy, gw, gu, gk, gv) + + +def _validate_device_match(tensors: Tuple[Tensor, ...]) -> None: + """Minimal runtime validation that all tensors live on the same CUDA device.""" + if not tensors: + return + device = tensors[0].device + if not device.type == "cuda": + raise RuntimeError("RWKV CUDA ops require CUDA tensors") + for t in tensors[1:]: + if t.device != device: + raise RuntimeError("All tensors must be on the same CUDA device") + + +__all__ = [ + "forward", + "forward_bf16", + "forward_with_state", + "forward_with_state_bf16", + "backward", + "backward_bf16", +] \ No newline at end of file diff --git a/build/torch29-cxx11-cu126-x86_64-linux/_ops.py b/build/torch29-cxx11-cu126-x86_64-linux/_ops.py new file mode 100644 index 0000000000000000000000000000000000000000..38c698ed4f2208d589ead1ef3852c6019d3eb01c --- /dev/null +++ b/build/torch29-cxx11-cu126-x86_64-linux/_ops.py @@ -0,0 +1,9 @@ +import torch +from . import _rwkv_d5e72cf +ops = torch.ops._rwkv_d5e72cf + +def add_op_namespace_prefix(op_name: str): + """ + Prefix op by namespace. + """ + return f"_rwkv_d5e72cf::{op_name}" \ No newline at end of file diff --git a/build/torch29-cxx11-cu126-x86_64-linux/_rwkv_d5e72cf.abi3.so b/build/torch29-cxx11-cu126-x86_64-linux/_rwkv_d5e72cf.abi3.so new file mode 100644 index 0000000000000000000000000000000000000000..e2c36a114d6ec611bfd0132faef09b77a54b673e --- /dev/null +++ b/build/torch29-cxx11-cu126-x86_64-linux/_rwkv_d5e72cf.abi3.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eaed9fa222b7cb56faadc9a92d91e0e9acf1a2635e794de99d59e32b68a2d8aa +size 2111480 diff --git a/build/torch29-cxx11-cu126-x86_64-linux/metadata.json b/build/torch29-cxx11-cu126-x86_64-linux/metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..76bafa5f33b6818aa6bb4cab04be811b87519b44 --- /dev/null +++ b/build/torch29-cxx11-cu126-x86_64-linux/metadata.json @@ -0,0 +1 @@ +{"python-depends":[]} \ No newline at end of file diff --git a/build/torch29-cxx11-cu126-x86_64-linux/rwkv/__init__.py b/build/torch29-cxx11-cu126-x86_64-linux/rwkv/__init__.py index d4ea22dbc160d2b75b32c3c8c5d882d25c77b494..03dbc1afe1cf156661a2b1b22003cd5f599a0309 100644 --- a/build/torch29-cxx11-cu126-x86_64-linux/rwkv/__init__.py +++ b/build/torch29-cxx11-cu126-x86_64-linux/rwkv/__init__.py @@ -1,170 +1,26 @@ -from ._ops import ops -from typing import Tuple, Any - -# Use a broad Tensor alias to avoid importing torch at import time. -from torch import Tensor - -def forward(w: Tensor, u: Tensor, k: Tensor, v: Tensor, y: Tensor) -> None: - """RWKV WKV forward pass (float32). - - Runs the CUDA kernel and writes the result into ``y`` in-place. - - Args: - w: Decay weights, shape ``[C]``, dtype ``torch.float32``. - u: Input tensor, shape ``[B, T, C]``, dtype ``torch.float32``. - k: Key tensor, shape ``[B, T, C]``, dtype ``torch.float32``. - v: Value tensor, shape ``[B, T, C]``, dtype ``torch.float32``. - y: Output tensor, shape ``[B, T, C]``, dtype ``torch.float32`` (written in-place). - - Notes: - - All tensors must be on the same CUDA device. - - Shapes must agree on ``B``, ``T`` and ``C``. - """ - _validate_device_match((w, u, k, v, y)) - ops.forward(w, u, k, v, y) - - -def forward_bf16(w: Tensor, u: Tensor, k: Tensor, v: Tensor, y: Tensor) -> None: - """RWKV WKV forward pass (bfloat16 inputs/outputs, float32 ``w``). - - Runs the CUDA kernel and writes the result into ``y`` in-place. - - Args: - w: Decay weights, shape ``[C]``, dtype ``torch.float32``. - u: Input tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16``. - k: Key tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16``. - v: Value tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16``. - y: Output tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16`` (written in-place). - - Notes: - - All tensors must be on the same CUDA device. - - Shapes must agree on ``B``, ``T`` and ``C``. - """ - _validate_device_match((w, u, k, v, y)) - ops.forward_bf16(w, u, k, v, y) - - -def forward_with_state(w: Tensor, u: Tensor, k: Tensor, v: Tensor, y: Tensor, s: Tensor) -> None: - """RWKV WKV forward pass with persistent state (float32). - - Runs the CUDA kernel using and updating state ``s`` and writes the result into ``y``. - - Args: - w: Decay weights, shape ``[C]``, dtype ``torch.float32``. - u: Input tensor, shape ``[B, T, C]``, dtype ``torch.float32``. - k: Key tensor, shape ``[B, T, C]``, dtype ``torch.float32``. - v: Value tensor, shape ``[B, T, C]``, dtype ``torch.float32``. - y: Output tensor, shape ``[B, T, C]``, dtype ``torch.float32`` (written in-place). - s: Stateful tensor, shape ``[B, C]``, dtype ``torch.float32`` (updated in-place). - - Notes: - - All tensors must be on the same CUDA device. - - Shapes must agree on ``B`` and ``C``; ``y`` shares ``[B, T, C]`` with inputs. - """ - _validate_device_match((w, u, k, v, y, s)) - ops.forward_with_state(w, u, k, v, y, s) - - -def forward_with_state_bf16(w: Tensor, u: Tensor, k: Tensor, v: Tensor, y: Tensor, s: Tensor) -> None: - """RWKV WKV forward pass with persistent state (bfloat16 inputs/outputs, float32 ``w`` and ``s``). - - Runs the CUDA kernel using and updating state ``s`` and writes the result into ``y``. - - Args: - w: Decay weights, shape ``[C]``, dtype ``torch.float32``. - u: Input tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16``. - k: Key tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16``. - v: Value tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16``. - y: Output tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16`` (written in-place). - s: Stateful tensor, shape ``[B, C]``, dtype ``torch.float32`` (updated in-place). - - Notes: - - All tensors must be on the same CUDA device. - - Shapes must agree on ``B`` and ``C``; ``y`` shares ``[B, T, C]`` with inputs. - """ - _validate_device_match((w, u, k, v, y, s)) - ops.forward_with_state_bf16(w, u, k, v, y, s) - - -def backward( - w: Tensor, - u: Tensor, - k: Tensor, - v: Tensor, - y: Tensor, - gy: Tensor, - gw: Tensor, - gu: Tensor, - gk: Tensor, - gv: Tensor, -) -> None: - """RWKV WKV backward pass (float32). - - Writes gradients into the provided tensors in-place. - - Args: - w: Decay weights, shape ``[C]``, dtype ``torch.float32``. - u, k, v, y: Forward-pass tensors, shape ``[B, T, C]``, dtype ``torch.float32``. - gy: Gradient of ``y``, shape ``[B, T, C]``, dtype ``torch.float32``. - gw: Gradient for ``w``, shape ``[C]``, dtype ``torch.float32`` (written in-place). - gu, gk, gv: Gradients for ``u``, ``k``, ``v`` respectively, shape ``[B, T, C]``, dtype ``torch.float32`` (written in-place). - - Notes: - - All tensors must be on the same CUDA device. - - Shapes must agree on ``B``, ``T`` and ``C``. - """ - _validate_device_match((w, u, k, v, y, gy, gw, gu, gk, gv)) - ops.backward(w, u, k, v, y, gy, gw, gu, gk, gv) - - -def backward_bf16( - w: Tensor, - u: Tensor, - k: Tensor, - v: Tensor, - y: Tensor, - gy: Tensor, - gw: Tensor, - gu: Tensor, - gk: Tensor, - gv: Tensor, -) -> None: - """RWKV WKV backward pass (bfloat16 inputs/outputs/gradients, float32 ``w``). - - Writes gradients into the provided tensors in-place. - - Args: - w: Decay weights, shape ``[C]``, dtype ``torch.float32``. - u, k, v, y: Forward-pass tensors, shape ``[B, T, C]``, dtype ``torch.bfloat16``. - gy: Gradient of ``y``, shape ``[B, T, C]``, dtype ``torch.bfloat16``. - gw: Gradient for ``w``, shape ``[C]``, dtype ``torch.bfloat16`` (written in-place). - gu, gk, gv: Gradients for ``u``, ``k``, ``v`` respectively, shape ``[B, T, C]``, dtype ``torch.bfloat16`` (written in-place). - - Notes: - - All tensors must be on the same CUDA device. - - Shapes must agree on ``B``, ``T`` and ``C``. - """ - _validate_device_match((w, u, k, v, y, gy, gw, gu, gk, gv)) - ops.backward_bf16(w, u, k, v, y, gy, gw, gu, gk, gv) - - -def _validate_device_match(tensors: Tuple[Tensor, ...]) -> None: - """Minimal runtime validation that all tensors live on the same CUDA device.""" - if not tensors: - return - device = tensors[0].device - if not device.type == "cuda": - raise RuntimeError("RWKV CUDA ops require CUDA tensors") - for t in tensors[1:]: - if t.device != device: - raise RuntimeError("All tensors must be on the same CUDA device") - - -__all__ = [ - "forward", - "forward_bf16", - "forward_with_state", - "forward_with_state_bf16", - "backward", - "backward_bf16", -] \ No newline at end of file +import ctypes +import sys + +import importlib +from pathlib import Path +from types import ModuleType + +def _import_from_path(file_path: Path) -> ModuleType: + # We cannot use the module name as-is, after adding it to `sys.modules`, + # it would also be used for other imports. So, we make a module name that + # depends on the path for it to be unique using the hex-encoded hash of + # the path. + path_hash = "{:x}".format(ctypes.c_size_t(hash(file_path.absolute())).value) + module_name = path_hash + spec = importlib.util.spec_from_file_location(module_name, file_path) + if spec is None: + raise ImportError(f"Cannot load spec for {module_name} from {file_path}") + module = importlib.util.module_from_spec(spec) + if module is None: + raise ImportError(f"Cannot load module {module_name} from spec") + sys.modules[module_name] = module + spec.loader.exec_module(module) # type: ignore + return module + + +globals().update(vars(_import_from_path(Path(__file__).parent.parent / "__init__.py"))) diff --git a/build/torch29-cxx11-cu126-x86_64-linux/rwkv/__pycache__/__init__.cpython-313.pyc b/build/torch29-cxx11-cu126-x86_64-linux/rwkv/__pycache__/__init__.cpython-313.pyc deleted file mode 100644 index d31f7775faa21b8b7522267267b2bda4d4a3f38a..0000000000000000000000000000000000000000 Binary files a/build/torch29-cxx11-cu126-x86_64-linux/rwkv/__pycache__/__init__.cpython-313.pyc and /dev/null differ diff --git a/build/torch29-cxx11-cu126-x86_64-linux/rwkv/__pycache__/_ops.cpython-313.pyc b/build/torch29-cxx11-cu126-x86_64-linux/rwkv/__pycache__/_ops.cpython-313.pyc deleted file mode 100644 index 82e2573fafb432b8719ac124ebf5be0637157b48..0000000000000000000000000000000000000000 Binary files a/build/torch29-cxx11-cu126-x86_64-linux/rwkv/__pycache__/_ops.cpython-313.pyc and /dev/null differ diff --git a/build/torch29-cxx11-cu126-x86_64-linux/rwkv/_rwkv_0aa7cb0.abi3.so b/build/torch29-cxx11-cu126-x86_64-linux/rwkv/_rwkv_0aa7cb0.abi3.so deleted file mode 100644 index 8180ad9313be7ce5d8efb26f231060e7fd8b597a..0000000000000000000000000000000000000000 --- a/build/torch29-cxx11-cu126-x86_64-linux/rwkv/_rwkv_0aa7cb0.abi3.so +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:2b36eaf10c6889d31ab600ff7c57d61afb12bed812c8d4d71f3c06c735c37957 -size 2106408 diff --git a/build/torch29-cxx11-cu128-x86_64-linux/__init__.py b/build/torch29-cxx11-cu128-x86_64-linux/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..d4ea22dbc160d2b75b32c3c8c5d882d25c77b494 --- /dev/null +++ b/build/torch29-cxx11-cu128-x86_64-linux/__init__.py @@ -0,0 +1,170 @@ +from ._ops import ops +from typing import Tuple, Any + +# Use a broad Tensor alias to avoid importing torch at import time. +from torch import Tensor + +def forward(w: Tensor, u: Tensor, k: Tensor, v: Tensor, y: Tensor) -> None: + """RWKV WKV forward pass (float32). + + Runs the CUDA kernel and writes the result into ``y`` in-place. + + Args: + w: Decay weights, shape ``[C]``, dtype ``torch.float32``. + u: Input tensor, shape ``[B, T, C]``, dtype ``torch.float32``. + k: Key tensor, shape ``[B, T, C]``, dtype ``torch.float32``. + v: Value tensor, shape ``[B, T, C]``, dtype ``torch.float32``. + y: Output tensor, shape ``[B, T, C]``, dtype ``torch.float32`` (written in-place). + + Notes: + - All tensors must be on the same CUDA device. + - Shapes must agree on ``B``, ``T`` and ``C``. + """ + _validate_device_match((w, u, k, v, y)) + ops.forward(w, u, k, v, y) + + +def forward_bf16(w: Tensor, u: Tensor, k: Tensor, v: Tensor, y: Tensor) -> None: + """RWKV WKV forward pass (bfloat16 inputs/outputs, float32 ``w``). + + Runs the CUDA kernel and writes the result into ``y`` in-place. + + Args: + w: Decay weights, shape ``[C]``, dtype ``torch.float32``. + u: Input tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16``. + k: Key tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16``. + v: Value tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16``. + y: Output tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16`` (written in-place). + + Notes: + - All tensors must be on the same CUDA device. + - Shapes must agree on ``B``, ``T`` and ``C``. + """ + _validate_device_match((w, u, k, v, y)) + ops.forward_bf16(w, u, k, v, y) + + +def forward_with_state(w: Tensor, u: Tensor, k: Tensor, v: Tensor, y: Tensor, s: Tensor) -> None: + """RWKV WKV forward pass with persistent state (float32). + + Runs the CUDA kernel using and updating state ``s`` and writes the result into ``y``. + + Args: + w: Decay weights, shape ``[C]``, dtype ``torch.float32``. + u: Input tensor, shape ``[B, T, C]``, dtype ``torch.float32``. + k: Key tensor, shape ``[B, T, C]``, dtype ``torch.float32``. + v: Value tensor, shape ``[B, T, C]``, dtype ``torch.float32``. + y: Output tensor, shape ``[B, T, C]``, dtype ``torch.float32`` (written in-place). + s: Stateful tensor, shape ``[B, C]``, dtype ``torch.float32`` (updated in-place). + + Notes: + - All tensors must be on the same CUDA device. + - Shapes must agree on ``B`` and ``C``; ``y`` shares ``[B, T, C]`` with inputs. + """ + _validate_device_match((w, u, k, v, y, s)) + ops.forward_with_state(w, u, k, v, y, s) + + +def forward_with_state_bf16(w: Tensor, u: Tensor, k: Tensor, v: Tensor, y: Tensor, s: Tensor) -> None: + """RWKV WKV forward pass with persistent state (bfloat16 inputs/outputs, float32 ``w`` and ``s``). + + Runs the CUDA kernel using and updating state ``s`` and writes the result into ``y``. + + Args: + w: Decay weights, shape ``[C]``, dtype ``torch.float32``. + u: Input tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16``. + k: Key tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16``. + v: Value tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16``. + y: Output tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16`` (written in-place). + s: Stateful tensor, shape ``[B, C]``, dtype ``torch.float32`` (updated in-place). + + Notes: + - All tensors must be on the same CUDA device. + - Shapes must agree on ``B`` and ``C``; ``y`` shares ``[B, T, C]`` with inputs. + """ + _validate_device_match((w, u, k, v, y, s)) + ops.forward_with_state_bf16(w, u, k, v, y, s) + + +def backward( + w: Tensor, + u: Tensor, + k: Tensor, + v: Tensor, + y: Tensor, + gy: Tensor, + gw: Tensor, + gu: Tensor, + gk: Tensor, + gv: Tensor, +) -> None: + """RWKV WKV backward pass (float32). + + Writes gradients into the provided tensors in-place. + + Args: + w: Decay weights, shape ``[C]``, dtype ``torch.float32``. + u, k, v, y: Forward-pass tensors, shape ``[B, T, C]``, dtype ``torch.float32``. + gy: Gradient of ``y``, shape ``[B, T, C]``, dtype ``torch.float32``. + gw: Gradient for ``w``, shape ``[C]``, dtype ``torch.float32`` (written in-place). + gu, gk, gv: Gradients for ``u``, ``k``, ``v`` respectively, shape ``[B, T, C]``, dtype ``torch.float32`` (written in-place). + + Notes: + - All tensors must be on the same CUDA device. + - Shapes must agree on ``B``, ``T`` and ``C``. + """ + _validate_device_match((w, u, k, v, y, gy, gw, gu, gk, gv)) + ops.backward(w, u, k, v, y, gy, gw, gu, gk, gv) + + +def backward_bf16( + w: Tensor, + u: Tensor, + k: Tensor, + v: Tensor, + y: Tensor, + gy: Tensor, + gw: Tensor, + gu: Tensor, + gk: Tensor, + gv: Tensor, +) -> None: + """RWKV WKV backward pass (bfloat16 inputs/outputs/gradients, float32 ``w``). + + Writes gradients into the provided tensors in-place. + + Args: + w: Decay weights, shape ``[C]``, dtype ``torch.float32``. + u, k, v, y: Forward-pass tensors, shape ``[B, T, C]``, dtype ``torch.bfloat16``. + gy: Gradient of ``y``, shape ``[B, T, C]``, dtype ``torch.bfloat16``. + gw: Gradient for ``w``, shape ``[C]``, dtype ``torch.bfloat16`` (written in-place). + gu, gk, gv: Gradients for ``u``, ``k``, ``v`` respectively, shape ``[B, T, C]``, dtype ``torch.bfloat16`` (written in-place). + + Notes: + - All tensors must be on the same CUDA device. + - Shapes must agree on ``B``, ``T`` and ``C``. + """ + _validate_device_match((w, u, k, v, y, gy, gw, gu, gk, gv)) + ops.backward_bf16(w, u, k, v, y, gy, gw, gu, gk, gv) + + +def _validate_device_match(tensors: Tuple[Tensor, ...]) -> None: + """Minimal runtime validation that all tensors live on the same CUDA device.""" + if not tensors: + return + device = tensors[0].device + if not device.type == "cuda": + raise RuntimeError("RWKV CUDA ops require CUDA tensors") + for t in tensors[1:]: + if t.device != device: + raise RuntimeError("All tensors must be on the same CUDA device") + + +__all__ = [ + "forward", + "forward_bf16", + "forward_with_state", + "forward_with_state_bf16", + "backward", + "backward_bf16", +] \ No newline at end of file diff --git a/build/torch29-cxx11-cu128-x86_64-linux/_ops.py b/build/torch29-cxx11-cu128-x86_64-linux/_ops.py new file mode 100644 index 0000000000000000000000000000000000000000..38c698ed4f2208d589ead1ef3852c6019d3eb01c --- /dev/null +++ b/build/torch29-cxx11-cu128-x86_64-linux/_ops.py @@ -0,0 +1,9 @@ +import torch +from . import _rwkv_d5e72cf +ops = torch.ops._rwkv_d5e72cf + +def add_op_namespace_prefix(op_name: str): + """ + Prefix op by namespace. + """ + return f"_rwkv_d5e72cf::{op_name}" \ No newline at end of file diff --git a/build/torch29-cxx11-cu128-x86_64-linux/_rwkv_d5e72cf.abi3.so b/build/torch29-cxx11-cu128-x86_64-linux/_rwkv_d5e72cf.abi3.so new file mode 100644 index 0000000000000000000000000000000000000000..d4a9b0e6d008d82720b3ff495de23fd50d6de109 --- /dev/null +++ b/build/torch29-cxx11-cu128-x86_64-linux/_rwkv_d5e72cf.abi3.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f4286b12bc927f8f66e912917ed774b706314e0515a624219a87f9db059963f5 +size 2318000 diff --git a/build/torch29-cxx11-cu128-x86_64-linux/metadata.json b/build/torch29-cxx11-cu128-x86_64-linux/metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..76bafa5f33b6818aa6bb4cab04be811b87519b44 --- /dev/null +++ b/build/torch29-cxx11-cu128-x86_64-linux/metadata.json @@ -0,0 +1 @@ +{"python-depends":[]} \ No newline at end of file diff --git a/build/torch29-cxx11-cu128-x86_64-linux/rwkv/__init__.py b/build/torch29-cxx11-cu128-x86_64-linux/rwkv/__init__.py index d4ea22dbc160d2b75b32c3c8c5d882d25c77b494..03dbc1afe1cf156661a2b1b22003cd5f599a0309 100644 --- a/build/torch29-cxx11-cu128-x86_64-linux/rwkv/__init__.py +++ b/build/torch29-cxx11-cu128-x86_64-linux/rwkv/__init__.py @@ -1,170 +1,26 @@ -from ._ops import ops -from typing import Tuple, Any - -# Use a broad Tensor alias to avoid importing torch at import time. -from torch import Tensor - -def forward(w: Tensor, u: Tensor, k: Tensor, v: Tensor, y: Tensor) -> None: - """RWKV WKV forward pass (float32). - - Runs the CUDA kernel and writes the result into ``y`` in-place. - - Args: - w: Decay weights, shape ``[C]``, dtype ``torch.float32``. - u: Input tensor, shape ``[B, T, C]``, dtype ``torch.float32``. - k: Key tensor, shape ``[B, T, C]``, dtype ``torch.float32``. - v: Value tensor, shape ``[B, T, C]``, dtype ``torch.float32``. - y: Output tensor, shape ``[B, T, C]``, dtype ``torch.float32`` (written in-place). - - Notes: - - All tensors must be on the same CUDA device. - - Shapes must agree on ``B``, ``T`` and ``C``. - """ - _validate_device_match((w, u, k, v, y)) - ops.forward(w, u, k, v, y) - - -def forward_bf16(w: Tensor, u: Tensor, k: Tensor, v: Tensor, y: Tensor) -> None: - """RWKV WKV forward pass (bfloat16 inputs/outputs, float32 ``w``). - - Runs the CUDA kernel and writes the result into ``y`` in-place. - - Args: - w: Decay weights, shape ``[C]``, dtype ``torch.float32``. - u: Input tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16``. - k: Key tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16``. - v: Value tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16``. - y: Output tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16`` (written in-place). - - Notes: - - All tensors must be on the same CUDA device. - - Shapes must agree on ``B``, ``T`` and ``C``. - """ - _validate_device_match((w, u, k, v, y)) - ops.forward_bf16(w, u, k, v, y) - - -def forward_with_state(w: Tensor, u: Tensor, k: Tensor, v: Tensor, y: Tensor, s: Tensor) -> None: - """RWKV WKV forward pass with persistent state (float32). - - Runs the CUDA kernel using and updating state ``s`` and writes the result into ``y``. - - Args: - w: Decay weights, shape ``[C]``, dtype ``torch.float32``. - u: Input tensor, shape ``[B, T, C]``, dtype ``torch.float32``. - k: Key tensor, shape ``[B, T, C]``, dtype ``torch.float32``. - v: Value tensor, shape ``[B, T, C]``, dtype ``torch.float32``. - y: Output tensor, shape ``[B, T, C]``, dtype ``torch.float32`` (written in-place). - s: Stateful tensor, shape ``[B, C]``, dtype ``torch.float32`` (updated in-place). - - Notes: - - All tensors must be on the same CUDA device. - - Shapes must agree on ``B`` and ``C``; ``y`` shares ``[B, T, C]`` with inputs. - """ - _validate_device_match((w, u, k, v, y, s)) - ops.forward_with_state(w, u, k, v, y, s) - - -def forward_with_state_bf16(w: Tensor, u: Tensor, k: Tensor, v: Tensor, y: Tensor, s: Tensor) -> None: - """RWKV WKV forward pass with persistent state (bfloat16 inputs/outputs, float32 ``w`` and ``s``). - - Runs the CUDA kernel using and updating state ``s`` and writes the result into ``y``. - - Args: - w: Decay weights, shape ``[C]``, dtype ``torch.float32``. - u: Input tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16``. - k: Key tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16``. - v: Value tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16``. - y: Output tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16`` (written in-place). - s: Stateful tensor, shape ``[B, C]``, dtype ``torch.float32`` (updated in-place). - - Notes: - - All tensors must be on the same CUDA device. - - Shapes must agree on ``B`` and ``C``; ``y`` shares ``[B, T, C]`` with inputs. - """ - _validate_device_match((w, u, k, v, y, s)) - ops.forward_with_state_bf16(w, u, k, v, y, s) - - -def backward( - w: Tensor, - u: Tensor, - k: Tensor, - v: Tensor, - y: Tensor, - gy: Tensor, - gw: Tensor, - gu: Tensor, - gk: Tensor, - gv: Tensor, -) -> None: - """RWKV WKV backward pass (float32). - - Writes gradients into the provided tensors in-place. - - Args: - w: Decay weights, shape ``[C]``, dtype ``torch.float32``. - u, k, v, y: Forward-pass tensors, shape ``[B, T, C]``, dtype ``torch.float32``. - gy: Gradient of ``y``, shape ``[B, T, C]``, dtype ``torch.float32``. - gw: Gradient for ``w``, shape ``[C]``, dtype ``torch.float32`` (written in-place). - gu, gk, gv: Gradients for ``u``, ``k``, ``v`` respectively, shape ``[B, T, C]``, dtype ``torch.float32`` (written in-place). - - Notes: - - All tensors must be on the same CUDA device. - - Shapes must agree on ``B``, ``T`` and ``C``. - """ - _validate_device_match((w, u, k, v, y, gy, gw, gu, gk, gv)) - ops.backward(w, u, k, v, y, gy, gw, gu, gk, gv) - - -def backward_bf16( - w: Tensor, - u: Tensor, - k: Tensor, - v: Tensor, - y: Tensor, - gy: Tensor, - gw: Tensor, - gu: Tensor, - gk: Tensor, - gv: Tensor, -) -> None: - """RWKV WKV backward pass (bfloat16 inputs/outputs/gradients, float32 ``w``). - - Writes gradients into the provided tensors in-place. - - Args: - w: Decay weights, shape ``[C]``, dtype ``torch.float32``. - u, k, v, y: Forward-pass tensors, shape ``[B, T, C]``, dtype ``torch.bfloat16``. - gy: Gradient of ``y``, shape ``[B, T, C]``, dtype ``torch.bfloat16``. - gw: Gradient for ``w``, shape ``[C]``, dtype ``torch.bfloat16`` (written in-place). - gu, gk, gv: Gradients for ``u``, ``k``, ``v`` respectively, shape ``[B, T, C]``, dtype ``torch.bfloat16`` (written in-place). - - Notes: - - All tensors must be on the same CUDA device. - - Shapes must agree on ``B``, ``T`` and ``C``. - """ - _validate_device_match((w, u, k, v, y, gy, gw, gu, gk, gv)) - ops.backward_bf16(w, u, k, v, y, gy, gw, gu, gk, gv) - - -def _validate_device_match(tensors: Tuple[Tensor, ...]) -> None: - """Minimal runtime validation that all tensors live on the same CUDA device.""" - if not tensors: - return - device = tensors[0].device - if not device.type == "cuda": - raise RuntimeError("RWKV CUDA ops require CUDA tensors") - for t in tensors[1:]: - if t.device != device: - raise RuntimeError("All tensors must be on the same CUDA device") - - -__all__ = [ - "forward", - "forward_bf16", - "forward_with_state", - "forward_with_state_bf16", - "backward", - "backward_bf16", -] \ No newline at end of file +import ctypes +import sys + +import importlib +from pathlib import Path +from types import ModuleType + +def _import_from_path(file_path: Path) -> ModuleType: + # We cannot use the module name as-is, after adding it to `sys.modules`, + # it would also be used for other imports. So, we make a module name that + # depends on the path for it to be unique using the hex-encoded hash of + # the path. + path_hash = "{:x}".format(ctypes.c_size_t(hash(file_path.absolute())).value) + module_name = path_hash + spec = importlib.util.spec_from_file_location(module_name, file_path) + if spec is None: + raise ImportError(f"Cannot load spec for {module_name} from {file_path}") + module = importlib.util.module_from_spec(spec) + if module is None: + raise ImportError(f"Cannot load module {module_name} from spec") + sys.modules[module_name] = module + spec.loader.exec_module(module) # type: ignore + return module + + +globals().update(vars(_import_from_path(Path(__file__).parent.parent / "__init__.py"))) diff --git a/build/torch29-cxx11-cu128-x86_64-linux/rwkv/__pycache__/__init__.cpython-313.pyc b/build/torch29-cxx11-cu128-x86_64-linux/rwkv/__pycache__/__init__.cpython-313.pyc deleted file mode 100644 index 6d79c81786b018c3475ab40423809501ffb4fe9d..0000000000000000000000000000000000000000 Binary files a/build/torch29-cxx11-cu128-x86_64-linux/rwkv/__pycache__/__init__.cpython-313.pyc and /dev/null differ diff --git a/build/torch29-cxx11-cu128-x86_64-linux/rwkv/__pycache__/_ops.cpython-313.pyc b/build/torch29-cxx11-cu128-x86_64-linux/rwkv/__pycache__/_ops.cpython-313.pyc deleted file mode 100644 index c1247fe373ff8f8052dbc5124a47cece0f1eef02..0000000000000000000000000000000000000000 Binary files a/build/torch29-cxx11-cu128-x86_64-linux/rwkv/__pycache__/_ops.cpython-313.pyc and /dev/null differ diff --git a/build/torch29-cxx11-cu128-x86_64-linux/rwkv/_ops.py b/build/torch29-cxx11-cu128-x86_64-linux/rwkv/_ops.py deleted file mode 100644 index c54b92a73953ef7168a8a45b1d3caa2e64acc4cb..0000000000000000000000000000000000000000 --- a/build/torch29-cxx11-cu128-x86_64-linux/rwkv/_ops.py +++ /dev/null @@ -1,9 +0,0 @@ -import torch -from . import _rwkv_0aa7cb0 -ops = torch.ops._rwkv_0aa7cb0 - -def add_op_namespace_prefix(op_name: str): - """ - Prefix op by namespace. - """ - return f"_rwkv_0aa7cb0::{op_name}" \ No newline at end of file diff --git a/build/torch29-cxx11-cu128-x86_64-linux/rwkv/_rwkv_0aa7cb0.abi3.so b/build/torch29-cxx11-cu128-x86_64-linux/rwkv/_rwkv_0aa7cb0.abi3.so deleted file mode 100644 index 168e06cd9eb46bbfa026641866b7a39716ddfd07..0000000000000000000000000000000000000000 --- a/build/torch29-cxx11-cu128-x86_64-linux/rwkv/_rwkv_0aa7cb0.abi3.so +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:24d6e882c3c502d3858ca8b52935f47d34cf49709aa8417353fa2d27bebeedab -size 2308824 diff --git a/build/torch29-cxx11-cu130-x86_64-linux/__init__.py b/build/torch29-cxx11-cu130-x86_64-linux/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..d4ea22dbc160d2b75b32c3c8c5d882d25c77b494 --- /dev/null +++ b/build/torch29-cxx11-cu130-x86_64-linux/__init__.py @@ -0,0 +1,170 @@ +from ._ops import ops +from typing import Tuple, Any + +# Use a broad Tensor alias to avoid importing torch at import time. +from torch import Tensor + +def forward(w: Tensor, u: Tensor, k: Tensor, v: Tensor, y: Tensor) -> None: + """RWKV WKV forward pass (float32). + + Runs the CUDA kernel and writes the result into ``y`` in-place. + + Args: + w: Decay weights, shape ``[C]``, dtype ``torch.float32``. + u: Input tensor, shape ``[B, T, C]``, dtype ``torch.float32``. + k: Key tensor, shape ``[B, T, C]``, dtype ``torch.float32``. + v: Value tensor, shape ``[B, T, C]``, dtype ``torch.float32``. + y: Output tensor, shape ``[B, T, C]``, dtype ``torch.float32`` (written in-place). + + Notes: + - All tensors must be on the same CUDA device. + - Shapes must agree on ``B``, ``T`` and ``C``. + """ + _validate_device_match((w, u, k, v, y)) + ops.forward(w, u, k, v, y) + + +def forward_bf16(w: Tensor, u: Tensor, k: Tensor, v: Tensor, y: Tensor) -> None: + """RWKV WKV forward pass (bfloat16 inputs/outputs, float32 ``w``). + + Runs the CUDA kernel and writes the result into ``y`` in-place. + + Args: + w: Decay weights, shape ``[C]``, dtype ``torch.float32``. + u: Input tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16``. + k: Key tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16``. + v: Value tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16``. + y: Output tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16`` (written in-place). + + Notes: + - All tensors must be on the same CUDA device. + - Shapes must agree on ``B``, ``T`` and ``C``. + """ + _validate_device_match((w, u, k, v, y)) + ops.forward_bf16(w, u, k, v, y) + + +def forward_with_state(w: Tensor, u: Tensor, k: Tensor, v: Tensor, y: Tensor, s: Tensor) -> None: + """RWKV WKV forward pass with persistent state (float32). + + Runs the CUDA kernel using and updating state ``s`` and writes the result into ``y``. + + Args: + w: Decay weights, shape ``[C]``, dtype ``torch.float32``. + u: Input tensor, shape ``[B, T, C]``, dtype ``torch.float32``. + k: Key tensor, shape ``[B, T, C]``, dtype ``torch.float32``. + v: Value tensor, shape ``[B, T, C]``, dtype ``torch.float32``. + y: Output tensor, shape ``[B, T, C]``, dtype ``torch.float32`` (written in-place). + s: Stateful tensor, shape ``[B, C]``, dtype ``torch.float32`` (updated in-place). + + Notes: + - All tensors must be on the same CUDA device. + - Shapes must agree on ``B`` and ``C``; ``y`` shares ``[B, T, C]`` with inputs. + """ + _validate_device_match((w, u, k, v, y, s)) + ops.forward_with_state(w, u, k, v, y, s) + + +def forward_with_state_bf16(w: Tensor, u: Tensor, k: Tensor, v: Tensor, y: Tensor, s: Tensor) -> None: + """RWKV WKV forward pass with persistent state (bfloat16 inputs/outputs, float32 ``w`` and ``s``). + + Runs the CUDA kernel using and updating state ``s`` and writes the result into ``y``. + + Args: + w: Decay weights, shape ``[C]``, dtype ``torch.float32``. + u: Input tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16``. + k: Key tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16``. + v: Value tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16``. + y: Output tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16`` (written in-place). + s: Stateful tensor, shape ``[B, C]``, dtype ``torch.float32`` (updated in-place). + + Notes: + - All tensors must be on the same CUDA device. + - Shapes must agree on ``B`` and ``C``; ``y`` shares ``[B, T, C]`` with inputs. + """ + _validate_device_match((w, u, k, v, y, s)) + ops.forward_with_state_bf16(w, u, k, v, y, s) + + +def backward( + w: Tensor, + u: Tensor, + k: Tensor, + v: Tensor, + y: Tensor, + gy: Tensor, + gw: Tensor, + gu: Tensor, + gk: Tensor, + gv: Tensor, +) -> None: + """RWKV WKV backward pass (float32). + + Writes gradients into the provided tensors in-place. + + Args: + w: Decay weights, shape ``[C]``, dtype ``torch.float32``. + u, k, v, y: Forward-pass tensors, shape ``[B, T, C]``, dtype ``torch.float32``. + gy: Gradient of ``y``, shape ``[B, T, C]``, dtype ``torch.float32``. + gw: Gradient for ``w``, shape ``[C]``, dtype ``torch.float32`` (written in-place). + gu, gk, gv: Gradients for ``u``, ``k``, ``v`` respectively, shape ``[B, T, C]``, dtype ``torch.float32`` (written in-place). + + Notes: + - All tensors must be on the same CUDA device. + - Shapes must agree on ``B``, ``T`` and ``C``. + """ + _validate_device_match((w, u, k, v, y, gy, gw, gu, gk, gv)) + ops.backward(w, u, k, v, y, gy, gw, gu, gk, gv) + + +def backward_bf16( + w: Tensor, + u: Tensor, + k: Tensor, + v: Tensor, + y: Tensor, + gy: Tensor, + gw: Tensor, + gu: Tensor, + gk: Tensor, + gv: Tensor, +) -> None: + """RWKV WKV backward pass (bfloat16 inputs/outputs/gradients, float32 ``w``). + + Writes gradients into the provided tensors in-place. + + Args: + w: Decay weights, shape ``[C]``, dtype ``torch.float32``. + u, k, v, y: Forward-pass tensors, shape ``[B, T, C]``, dtype ``torch.bfloat16``. + gy: Gradient of ``y``, shape ``[B, T, C]``, dtype ``torch.bfloat16``. + gw: Gradient for ``w``, shape ``[C]``, dtype ``torch.bfloat16`` (written in-place). + gu, gk, gv: Gradients for ``u``, ``k``, ``v`` respectively, shape ``[B, T, C]``, dtype ``torch.bfloat16`` (written in-place). + + Notes: + - All tensors must be on the same CUDA device. + - Shapes must agree on ``B``, ``T`` and ``C``. + """ + _validate_device_match((w, u, k, v, y, gy, gw, gu, gk, gv)) + ops.backward_bf16(w, u, k, v, y, gy, gw, gu, gk, gv) + + +def _validate_device_match(tensors: Tuple[Tensor, ...]) -> None: + """Minimal runtime validation that all tensors live on the same CUDA device.""" + if not tensors: + return + device = tensors[0].device + if not device.type == "cuda": + raise RuntimeError("RWKV CUDA ops require CUDA tensors") + for t in tensors[1:]: + if t.device != device: + raise RuntimeError("All tensors must be on the same CUDA device") + + +__all__ = [ + "forward", + "forward_bf16", + "forward_with_state", + "forward_with_state_bf16", + "backward", + "backward_bf16", +] \ No newline at end of file diff --git a/build/torch29-cxx11-cu130-x86_64-linux/_ops.py b/build/torch29-cxx11-cu130-x86_64-linux/_ops.py new file mode 100644 index 0000000000000000000000000000000000000000..38c698ed4f2208d589ead1ef3852c6019d3eb01c --- /dev/null +++ b/build/torch29-cxx11-cu130-x86_64-linux/_ops.py @@ -0,0 +1,9 @@ +import torch +from . import _rwkv_d5e72cf +ops = torch.ops._rwkv_d5e72cf + +def add_op_namespace_prefix(op_name: str): + """ + Prefix op by namespace. + """ + return f"_rwkv_d5e72cf::{op_name}" \ No newline at end of file diff --git a/build/torch29-cxx11-cu130-x86_64-linux/_rwkv_d5e72cf.abi3.so b/build/torch29-cxx11-cu130-x86_64-linux/_rwkv_d5e72cf.abi3.so new file mode 100644 index 0000000000000000000000000000000000000000..1dee529e4ddd5fb3a3a584a4d31a2e66d641f909 --- /dev/null +++ b/build/torch29-cxx11-cu130-x86_64-linux/_rwkv_d5e72cf.abi3.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:492b7547a3aaf00db089b5c10a379cae2889f127578a318b6a242aa89f40b46d +size 2343392 diff --git a/build/torch29-cxx11-cu130-x86_64-linux/metadata.json b/build/torch29-cxx11-cu130-x86_64-linux/metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..76bafa5f33b6818aa6bb4cab04be811b87519b44 --- /dev/null +++ b/build/torch29-cxx11-cu130-x86_64-linux/metadata.json @@ -0,0 +1 @@ +{"python-depends":[]} \ No newline at end of file diff --git a/build/torch29-cxx11-cu130-x86_64-linux/rwkv/__init__.py b/build/torch29-cxx11-cu130-x86_64-linux/rwkv/__init__.py index d4ea22dbc160d2b75b32c3c8c5d882d25c77b494..03dbc1afe1cf156661a2b1b22003cd5f599a0309 100644 --- a/build/torch29-cxx11-cu130-x86_64-linux/rwkv/__init__.py +++ b/build/torch29-cxx11-cu130-x86_64-linux/rwkv/__init__.py @@ -1,170 +1,26 @@ -from ._ops import ops -from typing import Tuple, Any - -# Use a broad Tensor alias to avoid importing torch at import time. -from torch import Tensor - -def forward(w: Tensor, u: Tensor, k: Tensor, v: Tensor, y: Tensor) -> None: - """RWKV WKV forward pass (float32). - - Runs the CUDA kernel and writes the result into ``y`` in-place. - - Args: - w: Decay weights, shape ``[C]``, dtype ``torch.float32``. - u: Input tensor, shape ``[B, T, C]``, dtype ``torch.float32``. - k: Key tensor, shape ``[B, T, C]``, dtype ``torch.float32``. - v: Value tensor, shape ``[B, T, C]``, dtype ``torch.float32``. - y: Output tensor, shape ``[B, T, C]``, dtype ``torch.float32`` (written in-place). - - Notes: - - All tensors must be on the same CUDA device. - - Shapes must agree on ``B``, ``T`` and ``C``. - """ - _validate_device_match((w, u, k, v, y)) - ops.forward(w, u, k, v, y) - - -def forward_bf16(w: Tensor, u: Tensor, k: Tensor, v: Tensor, y: Tensor) -> None: - """RWKV WKV forward pass (bfloat16 inputs/outputs, float32 ``w``). - - Runs the CUDA kernel and writes the result into ``y`` in-place. - - Args: - w: Decay weights, shape ``[C]``, dtype ``torch.float32``. - u: Input tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16``. - k: Key tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16``. - v: Value tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16``. - y: Output tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16`` (written in-place). - - Notes: - - All tensors must be on the same CUDA device. - - Shapes must agree on ``B``, ``T`` and ``C``. - """ - _validate_device_match((w, u, k, v, y)) - ops.forward_bf16(w, u, k, v, y) - - -def forward_with_state(w: Tensor, u: Tensor, k: Tensor, v: Tensor, y: Tensor, s: Tensor) -> None: - """RWKV WKV forward pass with persistent state (float32). - - Runs the CUDA kernel using and updating state ``s`` and writes the result into ``y``. - - Args: - w: Decay weights, shape ``[C]``, dtype ``torch.float32``. - u: Input tensor, shape ``[B, T, C]``, dtype ``torch.float32``. - k: Key tensor, shape ``[B, T, C]``, dtype ``torch.float32``. - v: Value tensor, shape ``[B, T, C]``, dtype ``torch.float32``. - y: Output tensor, shape ``[B, T, C]``, dtype ``torch.float32`` (written in-place). - s: Stateful tensor, shape ``[B, C]``, dtype ``torch.float32`` (updated in-place). - - Notes: - - All tensors must be on the same CUDA device. - - Shapes must agree on ``B`` and ``C``; ``y`` shares ``[B, T, C]`` with inputs. - """ - _validate_device_match((w, u, k, v, y, s)) - ops.forward_with_state(w, u, k, v, y, s) - - -def forward_with_state_bf16(w: Tensor, u: Tensor, k: Tensor, v: Tensor, y: Tensor, s: Tensor) -> None: - """RWKV WKV forward pass with persistent state (bfloat16 inputs/outputs, float32 ``w`` and ``s``). - - Runs the CUDA kernel using and updating state ``s`` and writes the result into ``y``. - - Args: - w: Decay weights, shape ``[C]``, dtype ``torch.float32``. - u: Input tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16``. - k: Key tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16``. - v: Value tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16``. - y: Output tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16`` (written in-place). - s: Stateful tensor, shape ``[B, C]``, dtype ``torch.float32`` (updated in-place). - - Notes: - - All tensors must be on the same CUDA device. - - Shapes must agree on ``B`` and ``C``; ``y`` shares ``[B, T, C]`` with inputs. - """ - _validate_device_match((w, u, k, v, y, s)) - ops.forward_with_state_bf16(w, u, k, v, y, s) - - -def backward( - w: Tensor, - u: Tensor, - k: Tensor, - v: Tensor, - y: Tensor, - gy: Tensor, - gw: Tensor, - gu: Tensor, - gk: Tensor, - gv: Tensor, -) -> None: - """RWKV WKV backward pass (float32). - - Writes gradients into the provided tensors in-place. - - Args: - w: Decay weights, shape ``[C]``, dtype ``torch.float32``. - u, k, v, y: Forward-pass tensors, shape ``[B, T, C]``, dtype ``torch.float32``. - gy: Gradient of ``y``, shape ``[B, T, C]``, dtype ``torch.float32``. - gw: Gradient for ``w``, shape ``[C]``, dtype ``torch.float32`` (written in-place). - gu, gk, gv: Gradients for ``u``, ``k``, ``v`` respectively, shape ``[B, T, C]``, dtype ``torch.float32`` (written in-place). - - Notes: - - All tensors must be on the same CUDA device. - - Shapes must agree on ``B``, ``T`` and ``C``. - """ - _validate_device_match((w, u, k, v, y, gy, gw, gu, gk, gv)) - ops.backward(w, u, k, v, y, gy, gw, gu, gk, gv) - - -def backward_bf16( - w: Tensor, - u: Tensor, - k: Tensor, - v: Tensor, - y: Tensor, - gy: Tensor, - gw: Tensor, - gu: Tensor, - gk: Tensor, - gv: Tensor, -) -> None: - """RWKV WKV backward pass (bfloat16 inputs/outputs/gradients, float32 ``w``). - - Writes gradients into the provided tensors in-place. - - Args: - w: Decay weights, shape ``[C]``, dtype ``torch.float32``. - u, k, v, y: Forward-pass tensors, shape ``[B, T, C]``, dtype ``torch.bfloat16``. - gy: Gradient of ``y``, shape ``[B, T, C]``, dtype ``torch.bfloat16``. - gw: Gradient for ``w``, shape ``[C]``, dtype ``torch.bfloat16`` (written in-place). - gu, gk, gv: Gradients for ``u``, ``k``, ``v`` respectively, shape ``[B, T, C]``, dtype ``torch.bfloat16`` (written in-place). - - Notes: - - All tensors must be on the same CUDA device. - - Shapes must agree on ``B``, ``T`` and ``C``. - """ - _validate_device_match((w, u, k, v, y, gy, gw, gu, gk, gv)) - ops.backward_bf16(w, u, k, v, y, gy, gw, gu, gk, gv) - - -def _validate_device_match(tensors: Tuple[Tensor, ...]) -> None: - """Minimal runtime validation that all tensors live on the same CUDA device.""" - if not tensors: - return - device = tensors[0].device - if not device.type == "cuda": - raise RuntimeError("RWKV CUDA ops require CUDA tensors") - for t in tensors[1:]: - if t.device != device: - raise RuntimeError("All tensors must be on the same CUDA device") - - -__all__ = [ - "forward", - "forward_bf16", - "forward_with_state", - "forward_with_state_bf16", - "backward", - "backward_bf16", -] \ No newline at end of file +import ctypes +import sys + +import importlib +from pathlib import Path +from types import ModuleType + +def _import_from_path(file_path: Path) -> ModuleType: + # We cannot use the module name as-is, after adding it to `sys.modules`, + # it would also be used for other imports. So, we make a module name that + # depends on the path for it to be unique using the hex-encoded hash of + # the path. + path_hash = "{:x}".format(ctypes.c_size_t(hash(file_path.absolute())).value) + module_name = path_hash + spec = importlib.util.spec_from_file_location(module_name, file_path) + if spec is None: + raise ImportError(f"Cannot load spec for {module_name} from {file_path}") + module = importlib.util.module_from_spec(spec) + if module is None: + raise ImportError(f"Cannot load module {module_name} from spec") + sys.modules[module_name] = module + spec.loader.exec_module(module) # type: ignore + return module + + +globals().update(vars(_import_from_path(Path(__file__).parent.parent / "__init__.py"))) diff --git a/build/torch29-cxx11-cu130-x86_64-linux/rwkv/__pycache__/__init__.cpython-313.pyc b/build/torch29-cxx11-cu130-x86_64-linux/rwkv/__pycache__/__init__.cpython-313.pyc deleted file mode 100644 index 8d62f7f619c9d8a7904d02b73a75ae484f041cad..0000000000000000000000000000000000000000 Binary files a/build/torch29-cxx11-cu130-x86_64-linux/rwkv/__pycache__/__init__.cpython-313.pyc and /dev/null differ diff --git a/build/torch29-cxx11-cu130-x86_64-linux/rwkv/__pycache__/_ops.cpython-313.pyc b/build/torch29-cxx11-cu130-x86_64-linux/rwkv/__pycache__/_ops.cpython-313.pyc deleted file mode 100644 index aa3af97f6b755ba8e007f89b5c2c7c7d3e4c3a9d..0000000000000000000000000000000000000000 Binary files a/build/torch29-cxx11-cu130-x86_64-linux/rwkv/__pycache__/_ops.cpython-313.pyc and /dev/null differ diff --git a/build/torch29-cxx11-cu130-x86_64-linux/rwkv/_ops.py b/build/torch29-cxx11-cu130-x86_64-linux/rwkv/_ops.py deleted file mode 100644 index c54b92a73953ef7168a8a45b1d3caa2e64acc4cb..0000000000000000000000000000000000000000 --- a/build/torch29-cxx11-cu130-x86_64-linux/rwkv/_ops.py +++ /dev/null @@ -1,9 +0,0 @@ -import torch -from . import _rwkv_0aa7cb0 -ops = torch.ops._rwkv_0aa7cb0 - -def add_op_namespace_prefix(op_name: str): - """ - Prefix op by namespace. - """ - return f"_rwkv_0aa7cb0::{op_name}" \ No newline at end of file diff --git a/build/torch29-cxx11-cu130-x86_64-linux/rwkv/_rwkv_0aa7cb0.abi3.so b/build/torch29-cxx11-cu130-x86_64-linux/rwkv/_rwkv_0aa7cb0.abi3.so deleted file mode 100644 index 42b4080b996835265476077f89c7311f178448cc..0000000000000000000000000000000000000000 --- a/build/torch29-cxx11-cu130-x86_64-linux/rwkv/_rwkv_0aa7cb0.abi3.so +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:78b560c6b54a4308ccf53c089923e692f7cac5f623e85f12fc4a0e303677b10b -size 2334712