Kernels:

kernels-community
/

tinygrad-rms

Trusted publisher

Kernel card Files Files and versions

xet

Community

kernels-bot commited on 27 days ago

Commit

8a449d4

verified ·

1 Parent(s): 468341e

Uploaded using `kernel-builder`.

Browse files

Files changed (34) hide show

.gitattributes +9 -0
build/torch210-cxx11-cu126-x86_64-linux/_ops.py +3 -3
build/torch210-cxx11-cu126-x86_64-linux/{_tinygrad_rms_cuda_86f75d9.abi3.so → _tinygrad_rms_cuda_1917536.abi3.so} +1 -1
build/torch210-cxx11-cu126-x86_64-linux/metadata.json +1 -1
build/torch210-cxx11-cu128-x86_64-linux/_ops.py +3 -3
build/torch210-cxx11-cu128-x86_64-linux/{_tinygrad_rms_cuda_86f75d9.abi3.so → _tinygrad_rms_cuda_1917536.abi3.so} +1 -1
build/torch210-cxx11-cu128-x86_64-linux/metadata.json +1 -1
build/torch210-cxx11-cu130-x86_64-linux/_ops.py +3 -3
build/torch210-cxx11-cu130-x86_64-linux/{_tinygrad_rms_cuda_86f75d9.abi3.so → _tinygrad_rms_cuda_1917536.abi3.so} +1 -1
build/torch210-cxx11-cu130-x86_64-linux/metadata.json +1 -1
build/torch211-cxx11-cu126-x86_64-linux/_ops.py +3 -3
build/torch211-cxx11-cu126-x86_64-linux/{_tinygrad_rms_cuda_86f75d9.abi3.so → _tinygrad_rms_cuda_1917536.abi3.so} +1 -1
build/torch211-cxx11-cu126-x86_64-linux/metadata.json +1 -1
build/torch211-cxx11-cu128-x86_64-linux/_ops.py +3 -3
build/torch211-cxx11-cu128-x86_64-linux/{_tinygrad_rms_cuda_86f75d9.abi3.so → _tinygrad_rms_cuda_1917536.abi3.so} +1 -1
build/torch211-cxx11-cu128-x86_64-linux/metadata.json +1 -1
build/torch211-cxx11-cu130-x86_64-linux/_ops.py +3 -3
build/torch211-cxx11-cu130-x86_64-linux/{_tinygrad_rms_cuda_86f75d9.abi3.so → _tinygrad_rms_cuda_1917536.abi3.so} +1 -1
build/torch211-cxx11-cu130-x86_64-linux/metadata.json +1 -1
build/torch212-cxx11-cu126-x86_64-linux/__init__.py +63 -0
build/torch212-cxx11-cu126-x86_64-linux/_ops.py +9 -0
build/torch212-cxx11-cu126-x86_64-linux/_tinygrad_rms_cuda_1917536.abi3.so +3 -0
build/torch212-cxx11-cu126-x86_64-linux/metadata.json +20 -0
build/torch212-cxx11-cu126-x86_64-linux/tinygrad_rms/__init__.py +26 -0
build/torch212-cxx11-cu130-x86_64-linux/__init__.py +63 -0
build/torch212-cxx11-cu130-x86_64-linux/_ops.py +9 -0
build/torch212-cxx11-cu130-x86_64-linux/_tinygrad_rms_cuda_1917536.abi3.so +3 -0
build/torch212-cxx11-cu130-x86_64-linux/metadata.json +21 -0
build/torch212-cxx11-cu130-x86_64-linux/tinygrad_rms/__init__.py +26 -0
build/torch212-cxx11-cu132-x86_64-linux/__init__.py +63 -0
build/torch212-cxx11-cu132-x86_64-linux/_ops.py +9 -0
build/torch212-cxx11-cu132-x86_64-linux/_tinygrad_rms_cuda_1917536.abi3.so +3 -0
build/torch212-cxx11-cu132-x86_64-linux/metadata.json +21 -0
build/torch212-cxx11-cu132-x86_64-linux/tinygrad_rms/__init__.py +26 -0

.gitattributes CHANGED Viewed

@@ -144,3 +144,12 @@ build/torch210-cxx11-cu130-x86_64-linux/_tinygrad_rms_cuda_86f75d9.abi3.so filte
 build/torch211-cxx11-cu126-x86_64-linux/_tinygrad_rms_cuda_86f75d9.abi3.so filter=lfs diff=lfs merge=lfs -text
 build/torch211-cxx11-cu128-x86_64-linux/_tinygrad_rms_cuda_86f75d9.abi3.so filter=lfs diff=lfs merge=lfs -text
 build/torch211-cxx11-cu130-x86_64-linux/_tinygrad_rms_cuda_86f75d9.abi3.so filter=lfs diff=lfs merge=lfs -text

 build/torch211-cxx11-cu126-x86_64-linux/_tinygrad_rms_cuda_86f75d9.abi3.so filter=lfs diff=lfs merge=lfs -text
 build/torch211-cxx11-cu128-x86_64-linux/_tinygrad_rms_cuda_86f75d9.abi3.so filter=lfs diff=lfs merge=lfs -text
 build/torch211-cxx11-cu130-x86_64-linux/_tinygrad_rms_cuda_86f75d9.abi3.so filter=lfs diff=lfs merge=lfs -text
+build/torch210-cxx11-cu126-x86_64-linux/_tinygrad_rms_cuda_1917536.abi3.so filter=lfs diff=lfs merge=lfs -text
+build/torch210-cxx11-cu128-x86_64-linux/_tinygrad_rms_cuda_1917536.abi3.so filter=lfs diff=lfs merge=lfs -text
+build/torch210-cxx11-cu130-x86_64-linux/_tinygrad_rms_cuda_1917536.abi3.so filter=lfs diff=lfs merge=lfs -text
+build/torch211-cxx11-cu126-x86_64-linux/_tinygrad_rms_cuda_1917536.abi3.so filter=lfs diff=lfs merge=lfs -text
+build/torch211-cxx11-cu128-x86_64-linux/_tinygrad_rms_cuda_1917536.abi3.so filter=lfs diff=lfs merge=lfs -text
+build/torch211-cxx11-cu130-x86_64-linux/_tinygrad_rms_cuda_1917536.abi3.so filter=lfs diff=lfs merge=lfs -text
+build/torch212-cxx11-cu126-x86_64-linux/_tinygrad_rms_cuda_1917536.abi3.so filter=lfs diff=lfs merge=lfs -text
+build/torch212-cxx11-cu130-x86_64-linux/_tinygrad_rms_cuda_1917536.abi3.so filter=lfs diff=lfs merge=lfs -text
+build/torch212-cxx11-cu132-x86_64-linux/_tinygrad_rms_cuda_1917536.abi3.so filter=lfs diff=lfs merge=lfs -text

build/torch210-cxx11-cu126-x86_64-linux/_ops.py CHANGED Viewed

@@ -1,9 +1,9 @@
 import torch
-from . import _tinygrad_rms_cuda_86f75d9
-ops = torch.ops._tinygrad_rms_cuda_86f75d9
 def add_op_namespace_prefix(op_name: str):
     """
     Prefix op by namespace.
     """
-    return f"_tinygrad_rms_cuda_86f75d9::{op_name}"

 import torch
+from . import _tinygrad_rms_cuda_1917536
+ops = torch.ops._tinygrad_rms_cuda_1917536
 def add_op_namespace_prefix(op_name: str):
     """
     Prefix op by namespace.
     """
+    return f"_tinygrad_rms_cuda_1917536::{op_name}"

build/torch210-cxx11-cu126-x86_64-linux/{_tinygrad_rms_cuda_86f75d9.abi3.so → _tinygrad_rms_cuda_1917536.abi3.so} RENAMED Viewed

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9edb15d0fa7507a7f988060c8fe849a11fb4a892f4e36220a4c45dfe10799a28
 size 2123424

 version https://git-lfs.github.com/spec/v1
+oid sha256:3080b93ffdc50607ab3f3539799840bc756275be5e94a69d3ffaa950d516d797
 size 2123424

build/torch210-cxx11-cu126-x86_64-linux/metadata.json CHANGED Viewed

@@ -1,6 +1,6 @@
 {
   "name": "tinygrad-rms",
-  "id": "_tinygrad_rms_cuda_86f75d9",
   "version": 1,
   "license": "MIT",
   "python-depends": [],

 {
   "name": "tinygrad-rms",
+  "id": "_tinygrad_rms_cuda_1917536",
   "version": 1,
   "license": "MIT",
   "python-depends": [],

build/torch210-cxx11-cu128-x86_64-linux/_ops.py CHANGED Viewed

@@ -1,9 +1,9 @@
 import torch
-from . import _tinygrad_rms_cuda_86f75d9
-ops = torch.ops._tinygrad_rms_cuda_86f75d9
 def add_op_namespace_prefix(op_name: str):
     """
     Prefix op by namespace.
     """
-    return f"_tinygrad_rms_cuda_86f75d9::{op_name}"

 import torch
+from . import _tinygrad_rms_cuda_1917536
+ops = torch.ops._tinygrad_rms_cuda_1917536
 def add_op_namespace_prefix(op_name: str):
     """
     Prefix op by namespace.
     """
+    return f"_tinygrad_rms_cuda_1917536::{op_name}"

build/torch210-cxx11-cu128-x86_64-linux/{_tinygrad_rms_cuda_86f75d9.abi3.so → _tinygrad_rms_cuda_1917536.abi3.so} RENAMED Viewed

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:cca0d5853e8d274e8bdcd84f48e8bb1fa67e2132026a23d2bcbf3145bc10f500
 size 2244040

 version https://git-lfs.github.com/spec/v1
+oid sha256:87e8724ff26b4efacab3f751eaee25ec70857d3dfca4e2f61022fc5cde8faa9d
 size 2244040

build/torch210-cxx11-cu128-x86_64-linux/metadata.json CHANGED Viewed

@@ -1,6 +1,6 @@
 {
   "name": "tinygrad-rms",
-  "id": "_tinygrad_rms_cuda_86f75d9",
   "version": 1,
   "license": "MIT",
   "python-depends": [],

 {
   "name": "tinygrad-rms",
+  "id": "_tinygrad_rms_cuda_1917536",
   "version": 1,
   "license": "MIT",
   "python-depends": [],

build/torch210-cxx11-cu130-x86_64-linux/_ops.py CHANGED Viewed

@@ -1,9 +1,9 @@
 import torch
-from . import _tinygrad_rms_cuda_86f75d9
-ops = torch.ops._tinygrad_rms_cuda_86f75d9
 def add_op_namespace_prefix(op_name: str):
     """
     Prefix op by namespace.
     """
-    return f"_tinygrad_rms_cuda_86f75d9::{op_name}"

 import torch
+from . import _tinygrad_rms_cuda_1917536
+ops = torch.ops._tinygrad_rms_cuda_1917536
 def add_op_namespace_prefix(op_name: str):
     """
     Prefix op by namespace.
     """
+    return f"_tinygrad_rms_cuda_1917536::{op_name}"

build/torch210-cxx11-cu130-x86_64-linux/{_tinygrad_rms_cuda_86f75d9.abi3.so → _tinygrad_rms_cuda_1917536.abi3.so} RENAMED Viewed

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1c5b9cbd98034f9303dad29addca97fd8dde5bcee74e9bcc5f5cfb87cb5590e9
 size 2238016

 version https://git-lfs.github.com/spec/v1
+oid sha256:1abc4fbd62072c1fb6219ef99f4c5e058eda6cf45862e4b02388e29fe1bc94db
 size 2238016

build/torch210-cxx11-cu130-x86_64-linux/metadata.json CHANGED Viewed

@@ -1,6 +1,6 @@
 {
   "name": "tinygrad-rms",
-  "id": "_tinygrad_rms_cuda_86f75d9",
   "version": 1,
   "license": "MIT",
   "python-depends": [],

 {
   "name": "tinygrad-rms",
+  "id": "_tinygrad_rms_cuda_1917536",
   "version": 1,
   "license": "MIT",
   "python-depends": [],

build/torch211-cxx11-cu126-x86_64-linux/_ops.py CHANGED Viewed

@@ -1,9 +1,9 @@
 import torch
-from . import _tinygrad_rms_cuda_86f75d9
-ops = torch.ops._tinygrad_rms_cuda_86f75d9
 def add_op_namespace_prefix(op_name: str):
     """
     Prefix op by namespace.
     """
-    return f"_tinygrad_rms_cuda_86f75d9::{op_name}"

 import torch
+from . import _tinygrad_rms_cuda_1917536
+ops = torch.ops._tinygrad_rms_cuda_1917536
 def add_op_namespace_prefix(op_name: str):
     """
     Prefix op by namespace.
     """
+    return f"_tinygrad_rms_cuda_1917536::{op_name}"

build/torch211-cxx11-cu126-x86_64-linux/{_tinygrad_rms_cuda_86f75d9.abi3.so → _tinygrad_rms_cuda_1917536.abi3.so} RENAMED Viewed

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:721ee26a9f60bad3068a37ae257576fee4584495727c7992fe29c3e520abd739
 size 2112256

 version https://git-lfs.github.com/spec/v1
+oid sha256:161dd9c9edc0e157296d3d69eb92b105d14174f5bc2af7e9d194a0e922c614b2
 size 2112256

build/torch211-cxx11-cu126-x86_64-linux/metadata.json CHANGED Viewed

@@ -1,6 +1,6 @@
 {
   "name": "tinygrad-rms",
-  "id": "_tinygrad_rms_cuda_86f75d9",
   "version": 1,
   "license": "MIT",
   "python-depends": [],

 {
   "name": "tinygrad-rms",
+  "id": "_tinygrad_rms_cuda_1917536",
   "version": 1,
   "license": "MIT",
   "python-depends": [],

build/torch211-cxx11-cu128-x86_64-linux/_ops.py CHANGED Viewed

@@ -1,9 +1,9 @@
 import torch
-from . import _tinygrad_rms_cuda_86f75d9
-ops = torch.ops._tinygrad_rms_cuda_86f75d9
 def add_op_namespace_prefix(op_name: str):
     """
     Prefix op by namespace.
     """
-    return f"_tinygrad_rms_cuda_86f75d9::{op_name}"

 import torch
+from . import _tinygrad_rms_cuda_1917536
+ops = torch.ops._tinygrad_rms_cuda_1917536
 def add_op_namespace_prefix(op_name: str):
     """
     Prefix op by namespace.
     """
+    return f"_tinygrad_rms_cuda_1917536::{op_name}"

build/torch211-cxx11-cu128-x86_64-linux/{_tinygrad_rms_cuda_86f75d9.abi3.so → _tinygrad_rms_cuda_1917536.abi3.so} RENAMED Viewed

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:50c37e69a4dea91ea16dbd0230d180b9a0434bff894bfdd614f9c0f2358efd92
 size 2232880

 version https://git-lfs.github.com/spec/v1
+oid sha256:7f78b8b95c674d06e24367eceb342c36f7f4fc8f6a8746473421780ff04f4a5e
 size 2232880

build/torch211-cxx11-cu128-x86_64-linux/metadata.json CHANGED Viewed

@@ -1,6 +1,6 @@
 {
   "name": "tinygrad-rms",
-  "id": "_tinygrad_rms_cuda_86f75d9",
   "version": 1,
   "license": "MIT",
   "python-depends": [],

 {
   "name": "tinygrad-rms",
+  "id": "_tinygrad_rms_cuda_1917536",
   "version": 1,
   "license": "MIT",
   "python-depends": [],

build/torch211-cxx11-cu130-x86_64-linux/_ops.py CHANGED Viewed

@@ -1,9 +1,9 @@
 import torch
-from . import _tinygrad_rms_cuda_86f75d9
-ops = torch.ops._tinygrad_rms_cuda_86f75d9
 def add_op_namespace_prefix(op_name: str):
     """
     Prefix op by namespace.
     """
-    return f"_tinygrad_rms_cuda_86f75d9::{op_name}"

 import torch
+from . import _tinygrad_rms_cuda_1917536
+ops = torch.ops._tinygrad_rms_cuda_1917536
 def add_op_namespace_prefix(op_name: str):
     """
     Prefix op by namespace.
     """
+    return f"_tinygrad_rms_cuda_1917536::{op_name}"

build/torch211-cxx11-cu130-x86_64-linux/{_tinygrad_rms_cuda_86f75d9.abi3.so → _tinygrad_rms_cuda_1917536.abi3.so} RENAMED Viewed

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:926f0afea6a970cbaa46e7bc417908bf419e030976a03f3a4462f7d51d2a868c
 size 2230952

 version https://git-lfs.github.com/spec/v1
+oid sha256:b2b41b8dbb85a370e627d021a8fc198e8b0d357a5e2a054929b54088f6f913f4
 size 2230952

build/torch211-cxx11-cu130-x86_64-linux/metadata.json CHANGED Viewed

@@ -1,6 +1,6 @@
 {
   "name": "tinygrad-rms",
-  "id": "_tinygrad_rms_cuda_86f75d9",
   "version": 1,
   "license": "MIT",
   "python-depends": [],

 {
   "name": "tinygrad-rms",
+  "id": "_tinygrad_rms_cuda_1917536",
   "version": 1,
   "license": "MIT",
   "python-depends": [],

build/torch212-cxx11-cu126-x86_64-linux/__init__.py ADDED Viewed

	@@ -0,0 +1,63 @@

+from typing import Optional, Tuple
+import torch
+from ._ops import ops
+def tinygrad_rms_norm(
+    x: torch.Tensor,
+    epsilon: float = 1e-6,
+    out: Optional[torch.Tensor] = None,
+) -> Tuple[torch.Tensor, torch.Tensor]:
+    """
+    Compute RMSNorm using tinygrad-style CUDA kernels.
+    RMSNorm(x) = x * (1 / sqrt(mean(x^2) + epsilon))
+    This implementation uses a two-kernel approach:
+    1. Compute 1/sqrt(mean(x^2) + epsilon) for each row
+    2. Multiply input by the computed factor
+    Args:
+        x: Input tensor of shape (..., hidden_size)
+        epsilon: Small constant for numerical stability
+        out: Optional pre-allocated output tensor
+    Returns:
+        Tuple of (output tensor, rms_inv tensor)
+    """
+    if out is None:
+        out = torch.empty_like(x)
+    hidden_size = x.size(-1)
+    num_rows = x.numel() // hidden_size
+    rms_inv = torch.empty(num_rows, dtype=x.dtype, device=x.device)
+    ops.tinygrad_rms_norm(out, rms_inv, x, epsilon)
+    return out, rms_inv
+def tinygrad_rms_norm_simple(
+    x: torch.Tensor,
+    epsilon: float = 1e-6,
+    out: Optional[torch.Tensor] = None,
+) -> torch.Tensor:
+    """
+    Compute RMSNorm using tinygrad-style CUDA kernels.
+    This is a simpler interface that only returns the normalized output.
+    Args:
+        x: Input tensor of shape (..., hidden_size)
+        epsilon: Small constant for numerical stability
+        out: Optional pre-allocated output tensor
+    Returns:
+        Normalized output tensor
+    """
+    if out is None:
+        out = torch.empty_like(x)
+    ops.tinygrad_rms_norm_inplace(out, x, epsilon)
+    return out

build/torch212-cxx11-cu126-x86_64-linux/_ops.py ADDED Viewed

	@@ -0,0 +1,9 @@

+import torch
+from . import _tinygrad_rms_cuda_1917536
+ops = torch.ops._tinygrad_rms_cuda_1917536
+def add_op_namespace_prefix(op_name: str):
+    """
+    Prefix op by namespace.
+    """
+    return f"_tinygrad_rms_cuda_1917536::{op_name}"

build/torch212-cxx11-cu126-x86_64-linux/_tinygrad_rms_cuda_1917536.abi3.so ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:40b198b0696a1dc1998c570fe04473b8bab587d60c822c82c0741d7356c27fe6
+size 2117808

build/torch212-cxx11-cu126-x86_64-linux/metadata.json ADDED Viewed

	@@ -0,0 +1,20 @@

+{
+  "name": "tinygrad-rms",
+  "id": "_tinygrad_rms_cuda_1917536",
+  "version": 1,
+  "license": "MIT",
+  "python-depends": [],
+  "backend": {
+    "type": "cuda",
+    "archs": [
+      "7.0",
+      "7.2",
+      "7.5",
+      "8.0",
+      "8.6",
+      "8.7",
+      "8.9",
+      "9.0+PTX"
+    ]
+  }
+}

build/torch212-cxx11-cu126-x86_64-linux/tinygrad_rms/__init__.py ADDED Viewed

	@@ -0,0 +1,26 @@

+import ctypes
+import importlib.util
+import sys
+from pathlib import Path
+from types import ModuleType
+def _import_from_path(file_path: Path) -> ModuleType:
+    # We cannot use the module name as-is, after adding it to `sys.modules`,
+    # it would also be used for other imports. So, we make a module name that
+    # depends on the path for it to be unique using the hex-encoded hash of
+    # the path.
+    path_hash = "{:x}".format(ctypes.c_size_t(hash(file_path.absolute())).value)
+    module_name = path_hash
+    spec = importlib.util.spec_from_file_location(module_name, file_path)
+    if spec is None:
+        raise ImportError(f"Cannot load spec for {module_name} from {file_path}")
+    module = importlib.util.module_from_spec(spec)
+    if module is None:
+        raise ImportError(f"Cannot load module {module_name} from spec")
+    sys.modules[module_name] = module
+    spec.loader.exec_module(module)  # type: ignore
+    return module
+globals().update(vars(_import_from_path(Path(__file__).parent.parent / "__init__.py")))

build/torch212-cxx11-cu130-x86_64-linux/__init__.py ADDED Viewed

	@@ -0,0 +1,63 @@

+from typing import Optional, Tuple
+import torch
+from ._ops import ops
+def tinygrad_rms_norm(
+    x: torch.Tensor,
+    epsilon: float = 1e-6,
+    out: Optional[torch.Tensor] = None,
+) -> Tuple[torch.Tensor, torch.Tensor]:
+    """
+    Compute RMSNorm using tinygrad-style CUDA kernels.
+    RMSNorm(x) = x * (1 / sqrt(mean(x^2) + epsilon))
+    This implementation uses a two-kernel approach:
+    1. Compute 1/sqrt(mean(x^2) + epsilon) for each row
+    2. Multiply input by the computed factor
+    Args:
+        x: Input tensor of shape (..., hidden_size)
+        epsilon: Small constant for numerical stability
+        out: Optional pre-allocated output tensor
+    Returns:
+        Tuple of (output tensor, rms_inv tensor)
+    """
+    if out is None:
+        out = torch.empty_like(x)
+    hidden_size = x.size(-1)
+    num_rows = x.numel() // hidden_size
+    rms_inv = torch.empty(num_rows, dtype=x.dtype, device=x.device)
+    ops.tinygrad_rms_norm(out, rms_inv, x, epsilon)
+    return out, rms_inv
+def tinygrad_rms_norm_simple(
+    x: torch.Tensor,
+    epsilon: float = 1e-6,
+    out: Optional[torch.Tensor] = None,
+) -> torch.Tensor:
+    """
+    Compute RMSNorm using tinygrad-style CUDA kernels.
+    This is a simpler interface that only returns the normalized output.
+    Args:
+        x: Input tensor of shape (..., hidden_size)
+        epsilon: Small constant for numerical stability
+        out: Optional pre-allocated output tensor
+    Returns:
+        Normalized output tensor
+    """
+    if out is None:
+        out = torch.empty_like(x)
+    ops.tinygrad_rms_norm_inplace(out, x, epsilon)
+    return out

build/torch212-cxx11-cu130-x86_64-linux/_ops.py ADDED Viewed

	@@ -0,0 +1,9 @@

+import torch
+from . import _tinygrad_rms_cuda_1917536
+ops = torch.ops._tinygrad_rms_cuda_1917536
+def add_op_namespace_prefix(op_name: str):
+    """
+    Prefix op by namespace.
+    """
+    return f"_tinygrad_rms_cuda_1917536::{op_name}"

build/torch212-cxx11-cu130-x86_64-linux/_tinygrad_rms_cuda_1917536.abi3.so ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:c545bbef2b47b7fddc5416c86bbe60edaccd7fd2d34b9f80c009f8e311ec9886
+size 2232352

build/torch212-cxx11-cu130-x86_64-linux/metadata.json ADDED Viewed

	@@ -0,0 +1,21 @@

+{
+  "name": "tinygrad-rms",
+  "id": "_tinygrad_rms_cuda_1917536",
+  "version": 1,
+  "license": "MIT",
+  "python-depends": [],
+  "backend": {
+    "type": "cuda",
+    "archs": [
+      "10.0",
+      "11.0",
+      "12.0+PTX",
+      "7.5",
+      "8.0",
+      "8.6",
+      "8.7",
+      "8.9",
+      "9.0"
+    ]
+  }
+}

build/torch212-cxx11-cu130-x86_64-linux/tinygrad_rms/__init__.py ADDED Viewed

	@@ -0,0 +1,26 @@

+import ctypes
+import importlib.util
+import sys
+from pathlib import Path
+from types import ModuleType
+def _import_from_path(file_path: Path) -> ModuleType:
+    # We cannot use the module name as-is, after adding it to `sys.modules`,
+    # it would also be used for other imports. So, we make a module name that
+    # depends on the path for it to be unique using the hex-encoded hash of
+    # the path.
+    path_hash = "{:x}".format(ctypes.c_size_t(hash(file_path.absolute())).value)
+    module_name = path_hash
+    spec = importlib.util.spec_from_file_location(module_name, file_path)
+    if spec is None:
+        raise ImportError(f"Cannot load spec for {module_name} from {file_path}")
+    module = importlib.util.module_from_spec(spec)
+    if module is None:
+        raise ImportError(f"Cannot load module {module_name} from spec")
+    sys.modules[module_name] = module
+    spec.loader.exec_module(module)  # type: ignore
+    return module
+globals().update(vars(_import_from_path(Path(__file__).parent.parent / "__init__.py")))

build/torch212-cxx11-cu132-x86_64-linux/__init__.py ADDED Viewed

	@@ -0,0 +1,63 @@

+from typing import Optional, Tuple
+import torch
+from ._ops import ops
+def tinygrad_rms_norm(
+    x: torch.Tensor,
+    epsilon: float = 1e-6,
+    out: Optional[torch.Tensor] = None,
+) -> Tuple[torch.Tensor, torch.Tensor]:
+    """
+    Compute RMSNorm using tinygrad-style CUDA kernels.
+    RMSNorm(x) = x * (1 / sqrt(mean(x^2) + epsilon))
+    This implementation uses a two-kernel approach:
+    1. Compute 1/sqrt(mean(x^2) + epsilon) for each row
+    2. Multiply input by the computed factor
+    Args:
+        x: Input tensor of shape (..., hidden_size)
+        epsilon: Small constant for numerical stability
+        out: Optional pre-allocated output tensor
+    Returns:
+        Tuple of (output tensor, rms_inv tensor)
+    """
+    if out is None:
+        out = torch.empty_like(x)
+    hidden_size = x.size(-1)
+    num_rows = x.numel() // hidden_size
+    rms_inv = torch.empty(num_rows, dtype=x.dtype, device=x.device)
+    ops.tinygrad_rms_norm(out, rms_inv, x, epsilon)
+    return out, rms_inv
+def tinygrad_rms_norm_simple(
+    x: torch.Tensor,
+    epsilon: float = 1e-6,
+    out: Optional[torch.Tensor] = None,
+) -> torch.Tensor:
+    """
+    Compute RMSNorm using tinygrad-style CUDA kernels.
+    This is a simpler interface that only returns the normalized output.
+    Args:
+        x: Input tensor of shape (..., hidden_size)
+        epsilon: Small constant for numerical stability
+        out: Optional pre-allocated output tensor
+    Returns:
+        Normalized output tensor
+    """
+    if out is None:
+        out = torch.empty_like(x)
+    ops.tinygrad_rms_norm_inplace(out, x, epsilon)
+    return out

build/torch212-cxx11-cu132-x86_64-linux/_ops.py ADDED Viewed

	@@ -0,0 +1,9 @@

+import torch
+from . import _tinygrad_rms_cuda_1917536
+ops = torch.ops._tinygrad_rms_cuda_1917536
+def add_op_namespace_prefix(op_name: str):
+    """
+    Prefix op by namespace.
+    """
+    return f"_tinygrad_rms_cuda_1917536::{op_name}"

build/torch212-cxx11-cu132-x86_64-linux/_tinygrad_rms_cuda_1917536.abi3.so ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:84572d82898e0bbff7e6796614efc63d73f332732a6de471037466b9a2c990dc
+size 2291952

build/torch212-cxx11-cu132-x86_64-linux/metadata.json ADDED Viewed

	@@ -0,0 +1,21 @@

+{
+  "name": "tinygrad-rms",
+  "id": "_tinygrad_rms_cuda_1917536",
+  "version": 1,
+  "license": "MIT",
+  "python-depends": [],
+  "backend": {
+    "type": "cuda",
+    "archs": [
+      "10.0",
+      "11.0",
+      "12.0+PTX",
+      "7.5",
+      "8.0",
+      "8.6",
+      "8.7",
+      "8.9",
+      "9.0"
+    ]
+  }
+}

build/torch212-cxx11-cu132-x86_64-linux/tinygrad_rms/__init__.py ADDED Viewed

	@@ -0,0 +1,26 @@

+import ctypes
+import importlib.util
+import sys
+from pathlib import Path
+from types import ModuleType
+def _import_from_path(file_path: Path) -> ModuleType:
+    # We cannot use the module name as-is, after adding it to `sys.modules`,
+    # it would also be used for other imports. So, we make a module name that
+    # depends on the path for it to be unique using the hex-encoded hash of
+    # the path.
+    path_hash = "{:x}".format(ctypes.c_size_t(hash(file_path.absolute())).value)
+    module_name = path_hash
+    spec = importlib.util.spec_from_file_location(module_name, file_path)
+    if spec is None:
+        raise ImportError(f"Cannot load spec for {module_name} from {file_path}")
+    module = importlib.util.module_from_spec(spec)
+    if module is None:
+        raise ImportError(f"Cannot load module {module_name} from spec")
+    sys.modules[module_name] = module
+    spec.loader.exec_module(module)  # type: ignore
+    return module
+globals().update(vars(_import_from_path(Path(__file__).parent.parent / "__init__.py")))