Buckets:

hf-doc-build
/

doc-dev

Files

xet

hf-doc-build/doc-dev / kernels /pr_520 /en /layers.html

HuggingFaceDocBuilder

7 days ago

download

raw

65.6 kB

	<meta charset="utf-8" /><meta name="hf:doc:metadata" content="{"title":"Layers","local":"layers","sections":[{"title":"Making a layer extensible with kernels from the hub","local":"making-a-layer-extensible-with-kernels-from-the-hub","sections":[{"title":"Using a decorator","local":"using-a-decorator","sections":[],"depth":3},{"title":"External layers","local":"external-layers","sections":[],"depth":3},{"title":"Using a function as a layer","local":"using-a-function-as-a-layer","sections":[],"depth":3}],"depth":2},{"title":"Kernelizing a model","local":"kernelizing-a-model","sections":[{"title":"Kernel device","local":"kernel-device","sections":[],"depth":3},{"title":"Fallback forward","local":"fallback-forward","sections":[],"depth":3},{"title":"Inspecting which kernels are used","local":"inspecting-which-kernels-are-used","sections":[],"depth":3}],"depth":2},{"title":"Registering a hub kernel for a layer","local":"registering-a-hub-kernel-for-a-layer","sections":[{"title":"Registering kernels for specific modes","local":"registering-kernels-for-specific-modes","sections":[],"depth":3},{"title":"Registering kernels for specific CUDA capabilities","local":"registering-kernels-for-specific-cuda-capabilities","sections":[],"depth":3},{"title":"Registering kernels for specific ROCm capabilities","local":"registering-kernels-for-specific-rocm-capabilities","sections":[],"depth":3},{"title":"Loading from a local repository for testing","local":"loading-from-a-local-repository-for-testing","sections":[],"depth":3}],"depth":2}],"depth":1}">
	<link href="/docs/kernels/pr_520/en/_app/immutable/assets/0.e3b0c442.css" rel="modulepreload">
	<link rel="modulepreload" href="/docs/kernels/pr_520/en/_app/immutable/entry/start.7813b24f.js">
	<link rel="modulepreload" href="/docs/kernels/pr_520/en/_app/immutable/chunks/scheduler.f3b1e791.js">
	<link rel="modulepreload" href="/docs/kernels/pr_520/en/_app/immutable/chunks/singletons.d3bd4a42.js">
	<link rel="modulepreload" href="/docs/kernels/pr_520/en/_app/immutable/chunks/paths.6059ca6d.js">
	<link rel="modulepreload" href="/docs/kernels/pr_520/en/_app/immutable/entry/app.d97547bc.js">
	<link rel="modulepreload" href="/docs/kernels/pr_520/en/_app/immutable/chunks/preload-helper.4c8a338e.js">
	<link rel="modulepreload" href="/docs/kernels/pr_520/en/_app/immutable/chunks/index.023a9934.js">
	<link rel="modulepreload" href="/docs/kernels/pr_520/en/_app/immutable/nodes/0.884fe107.js">
	<link rel="modulepreload" href="/docs/kernels/pr_520/en/_app/immutable/chunks/each.e59479a4.js">
	<link rel="modulepreload" href="/docs/kernels/pr_520/en/_app/immutable/nodes/27.8f268624.js">
	<link rel="modulepreload" href="/docs/kernels/pr_520/en/_app/immutable/chunks/CopyLLMTxtMenu.d8c1f5b0.js">
	<link rel="modulepreload" href="/docs/kernels/pr_520/en/_app/immutable/chunks/MermaidChart.svelte_svelte_type_style_lang.11da6958.js">
	<link rel="modulepreload" href="/docs/kernels/pr_520/en/_app/immutable/chunks/CodeBlock.436ee8e3.js"><!-- HEAD_svelte-u9bgzb_START --><meta name="hf:doc:metadata" content="{"title":"Layers","local":"layers","sections":[{"title":"Making a layer extensible with kernels from the hub","local":"making-a-layer-extensible-with-kernels-from-the-hub","sections":[{"title":"Using a decorator","local":"using-a-decorator","sections":[],"depth":3},{"title":"External layers","local":"external-layers","sections":[],"depth":3},{"title":"Using a function as a layer","local":"using-a-function-as-a-layer","sections":[],"depth":3}],"depth":2},{"title":"Kernelizing a model","local":"kernelizing-a-model","sections":[{"title":"Kernel device","local":"kernel-device","sections":[],"depth":3},{"title":"Fallback forward","local":"fallback-forward","sections":[],"depth":3},{"title":"Inspecting which kernels are used","local":"inspecting-which-kernels-are-used","sections":[],"depth":3}],"depth":2},{"title":"Registering a hub kernel for a layer","local":"registering-a-hub-kernel-for-a-layer","sections":[{"title":"Registering kernels for specific modes","local":"registering-kernels-for-specific-modes","sections":[],"depth":3},{"title":"Registering kernels for specific CUDA capabilities","local":"registering-kernels-for-specific-cuda-capabilities","sections":[],"depth":3},{"title":"Registering kernels for specific ROCm capabilities","local":"registering-kernels-for-specific-rocm-capabilities","sections":[],"depth":3},{"title":"Loading from a local repository for testing","local":"loading-from-a-local-repository-for-testing","sections":[],"depth":3}],"depth":2}],"depth":1}"><!-- HEAD_svelte-u9bgzb_END --> <p></p> <div class="items-center shrink-0 min-w-[100px] max-sm:min-w-[50px] justify-end ml-auto flex" style="float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"><div class="inline-flex rounded-md max-sm:rounded-sm"><button class="inline-flex items-center gap-1 h-7 max-sm:h-7 px-2 max-sm:px-1.5 text-sm font-medium text-gray-800 border border-r-0 rounded-l-md max-sm:rounded-l-sm border-gray-200 bg-white hover:shadow-inner dark:border-gray-850 dark:bg-gray-950 dark:text-gray-200 dark:hover:bg-gray-800" aria-live="polite"><span class="inline-flex items-center justify-center rounded-md p-0.5 max-sm:p-0 hover:text-gray-800 dark:hover:text-gray-200"><svg class="sm:size-3.5 size-3" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg></span> <span>Copy page</span></button> <button class="inline-flex items-center justify-center w-6 max-sm:w-5 h-7 max-sm:h-7 disabled:pointer-events-none text-sm text-gray-500 hover:text-gray-700 dark:hover:text-white rounded-r-md max-sm:rounded-r-sm border border-l transition border-gray-200 bg-white hover:shadow-inner dark:border-gray-850 dark:bg-gray-950 dark:text-gray-200 dark:hover:bg-gray-800" aria-haspopup="menu" aria-expanded="false" aria-label="Open copy menu"><svg class="transition-transform text-gray-400 overflow-visible sm:size-3.5 size-3 rotate-0" width="1em" height="1em" viewBox="0 0 12 7" fill="none" xmlns="http://www.w3.org/2000/svg"><path d="M1 1L6 6L11 1" stroke="currentColor"></path></svg></button></div> </div> <h1 class="relative group"><a id="layers" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#layers"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Layers</span></h1> <p data-svelte-h="svelte-178x673">A kernel can provide layers in addition to kernel functions. A layer from
	the Hub can replace the <code>forward</code> method of an existing layer for a certain
	device type. This makes it possible to provide more performant kernels for
	existing layers.</p> <p data-svelte-h="svelte-35znf1">See <a href="kernel-requirements">Kernel requirements</a> for more information on the
	requirements of Hub layers.</p> <h2 class="relative group"><a id="making-a-layer-extensible-with-kernels-from-the-hub" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#making-a-layer-extensible-with-kernels-from-the-hub"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Making a layer extensible with kernels from the hub</span></h2> <h3 class="relative group"><a id="using-a-decorator" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#using-a-decorator"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Using a decorator</span></h3> <p data-svelte-h="svelte-1jd41lf">A layer can be made extensible with the <code>use_kernel_forward_from_hub</code>
	decorator. For example:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-meta">@use_kernel_forward_from_hub(<span class="hljs-params"><span class="hljs-string">"SiluAndMul"</span></span>)</span>
	<span class="hljs-keyword">class</span> <span class="hljs-title class_">SiluAndMul</span>(nn.Module):
	<span class="hljs-keyword">def</span> <span class="hljs-title function_">forward</span>(<span class="hljs-params">self, <span class="hljs-built_in">input</span>: torch.Tensor</span>) -> torch.Tensor:
	d = <span class="hljs-built_in">input</span>.shape[-<span class="hljs-number">1</span>] // <span class="hljs-number">2</span>
	<span class="hljs-keyword">return</span> F.silu(<span class="hljs-built_in">input</span>[..., :d]) * <span class="hljs-built_in">input</span>[..., d:]<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-tily1x">The decorator does not change the behavior of the class — it annotates
	the class with the given name (here <code>SiluAndMul</code>). The <code>kernelize</code> function
	described below uses this name to look up kernels for the layer.</p> <h3 class="relative group"><a id="external-layers" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#external-layers"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>External layers</span></h3> <p data-svelte-h="svelte-1hj9ahw">An existing layer that does not (yet) have the <code>use_kernel_forward_from_hub</code>
	decorator can be made extensible using the <code>replace_kernel_forward_from_hub</code>
	function:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-keyword">from</span> somelibrary <span class="hljs-keyword">import</span> SiluAndMul

	replace_kernel_forward_from_hub(SiluAndMul, <span class="hljs-string">"SiluAndMul"</span>)<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-13coke1"><strong>Warning:</strong> we strongly recommend using layers with a decorator, since
	it signifies that the maintainer intends to keep the <code>forward</code> signature
	compatible with layers from the hub.</p> <h3 class="relative group"><a id="using-a-function-as-a-layer" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#using-a-function-as-a-layer"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Using a function as a layer</span></h3> <p data-svelte-h="svelte-kwbso1">Sometimes it can be useful to make a function extensible, for example
	because the function cannot be replaced by a layer. In such cases, you
	can annotate the function with the <code>use_kernel_func_from_hub</code> decorator:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-meta">@use_kernel_func_from_hub(<span class="hljs-params"><span class="hljs-string">"silu_and_mul"</span></span>)</span>
	<span class="hljs-keyword">def</span> <span class="hljs-title function_">silu_and_mul</span>(<span class="hljs-params">x: torch.Tensor</span>) -> torch.Tensor:
	d = x.shape[-<span class="hljs-number">1</span>] // <span class="hljs-number">2</span>
	<span class="hljs-keyword">return</span> F.silu(x[..., :d]) * x[..., d:]<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-1w74xdp">This will replace the function by an instantiated <code>torch.nn.Module</code>
	(singleton) that calls the function itself in its forward method.</p> <p data-svelte-h="svelte-t3eq4x"><strong>Note:</strong> for kernelization to see the function, it must be a member of
	another <code>torch.nn.Module</code> that is part of the model. For example:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-keyword">class</span> <span class="hljs-title class_">FeedForward</span>(nn.Module):
	<span class="hljs-keyword">def</span> <span class="hljs-title function_">__init__</span>(<span class="hljs-params">self, in_features: <span class="hljs-built_in">int</span>, out_features: <span class="hljs-built_in">int</span></span>):
	self.linear = nn.Linear(in_features, out_features)
	<span class="hljs-comment"># Note: silu_and_mul is a Torch module.</span>
	self.silu_and_mul = silu_and_mul

	<span class="hljs-keyword">def</span> <span class="hljs-title function_">forward</span>(<span class="hljs-params">self, x: torch.Tensor</span>) -> torch.Tensor:
	<span class="hljs-keyword">return</span> self.silu_and_mul(self.linear(x))<!-- HTML_TAG_END --></pre></div> <h2 class="relative group"><a id="kernelizing-a-model" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#kernelizing-a-model"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Kernelizing a model</span></h2> <p data-svelte-h="svelte-stkx9y">A model will not use Hub kernels by default, even if it contains extensible
	layers. To enable the use of Hub kernels in the model, it needs to be
	‘kernelized’ using the <code>kernelize</code> function. This function traverses the
	model graph and replaces the <code>forward</code> methods of extensible layers for which
	Hub kernels are registered. <code>kernelize</code> can be used as follows:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->model = MyModel(...)
	model = kernelize(model, mode=Mode.INFERENCE)<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-1mdicvi">The <code>kernelize</code> function modifies the model in-place, the model itself is
	returned as a convenience. The <code>mode</code> specifies that the model will be used
	in inference. Similarly, you can ask <code>kernelize</code> to prepare the model for
	training:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->model = MyModel(...)
	model = kernelize(model, mode=Mode.TRAINING)<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-xhc5xj">A model that is kernelized for training can also be used for inference, but
	not the other way around. If you want to change the mode of the kernelized
	model, you can just run <code>kernelize</code> on the model again with the new mode.</p> <p data-svelte-h="svelte-195gdzl">If you want to compile a model with <code>torch.compile</code>, this should be indicated
	in the mode as well. You can do this by combining <code>Mode.INFERENCE</code> or
	<code>Mode.TRAINING</code> with <code>Mode.TORCH_COMPILE</code> using the set union (<code>\|</code>) operator:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->model = MyModel(...)

	<span class="hljs-comment"># Inference</span>
	model = kernelize(model, mode=Mode.INFERENCE \| Mode.TORCH_COMPILE)

	<span class="hljs-comment"># Training</span>
	model = kernelize(model, mode=Mode.TRAINING \| Mode.TORCH_COMPILE)<!-- HTML_TAG_END --></pre></div> <h3 class="relative group"><a id="kernel-device" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#kernel-device"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Kernel device</span></h3> <p data-svelte-h="svelte-808tau">Kernels can be registered per device type. For instance, separate <code>cuda</code> and
	<code>metal</code> kernels could be registered for the name <code>SiluAndMul</code>. By default,
	<code>kernelize</code> will try to infer the device type from the model’s parameters.
	You can pass the device type to <code>kernelize</code> if the device type cannot be
	inferred (e.g. because the model has no parameters):</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->model = MyModel(...)
	model = kernelize(model, device=<span class="hljs-string">"cuda"</span>, mode=Mode.INFERENCE)<!-- HTML_TAG_END --></pre></div> <h3 class="relative group"><a id="fallback-forward" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#fallback-forward"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Fallback forward</span></h3> <p data-svelte-h="svelte-yu926d">If the <code>TRAINING</code> and/or <code>TORCH_COMPILE</code> modes are used, but a registered
	kernel does not support backward passes or <code>torch.compile</code> respectively,
	<code>kernelize</code> will fall back to the original, non-kernelized, layer. You
	can let <code>kernelize</code> raise an exception instead by using <code>use_fallback=False</code>:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->model = MyModel(...)
	model = kernelize(model, mode=Mode.INFERENCE \| Mode.TORCH_COMPILE, use_fallback=<span class="hljs-literal">False</span>)<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-1lkusth">This can be useful if you want to guarantee that Hub kernels are used.</p> <h3 class="relative group"><a id="inspecting-which-kernels-are-used" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#inspecting-which-kernels-are-used"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Inspecting which kernels are used</span></h3> <p data-svelte-h="svelte-161p6z8">The kernels that are used are logged at the <code>INFO</code> level by <code>kernelize</code>.
	See the <a href="https://docs.python.org/3/library/logging.html" rel="nofollow">Python logging</a>
	documentation for information on how to configure logging.</p> <h2 class="relative group"><a id="registering-a-hub-kernel-for-a-layer" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#registering-a-hub-kernel-for-a-layer"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Registering a hub kernel for a layer</span></h2> <p data-svelte-h="svelte-1r5il31"><code>kernelize</code> relies on kernel mappings to find Hub kernels for layers.
	Kernel mappings map a kernel name such as <code>SiluAndMul</code> to a kernel on
	the Hub. For example:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->kernel_layer_mapping = {
	<span class="hljs-string">"SiluAndMul"</span>: {
	<span class="hljs-string">"cuda"</span>: LayerRepository(
	repo_id=<span class="hljs-string">"kernels-community/activation"</span>,
	layer_name=<span class="hljs-string">"SiluAndMul"</span>,
	version=<span class="hljs-number">1</span>,
	),
	<span class="hljs-string">"rocm"</span>: LayerRepository(
	repo_id=<span class="hljs-string">"kernels-community/activation"</span>,
	layer_name=<span class="hljs-string">"SiluAndMul"</span>,
	version=<span class="hljs-number">1</span>,
	)
	}
	}<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-135k7m5">This uses version <code>1</code> of the <code>SiluAndMul</code> kernel layer from
	<code>kernels-community/activation</code> for the <code>cuda</code> and <code>rocm</code> backends. Kernel
	layers are versioned using a major version number. Using <code>version=1</code>
	will get the latest kernel build from the <code>v1</code> branch. Kernel layers
	within a version branch must never break the API or remove builds for
	older PyTorch versions. This ensures that your code will continue to
	work.</p> <p data-svelte-h="svelte-62g4c">Some kernels have not yet been updated to use versioning yet. In these cases,
	you can use <code>LayerRepository</code> without the <code>version</code> argument.</p> <p data-svelte-h="svelte-1ax1kej">You can register a mapping, like the one above, using <code>register_kernel_mapping</code>:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->register_kernel_mapping(kernel_layer_mapping)<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-p5ro9n">This will register the kernel mapping in the current context, which is
	normally global. It is recommended to scope the mapping to where it is
	used with the <code>use_kernel_mapping</code> context manager:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-keyword">with</span> use_kernel_mapping(kernel_layer_mapping):
	<span class="hljs-comment"># Use the layer for which the mapping is applied.</span>
	model = kernelize(model, mode=Mode.TRAINING \| Mode.TORCH_COMPILE)<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-z3xuuf">This ensures that the mapping is not active anymore outside the
	<code>with</code>-scope.</p> <p data-svelte-h="svelte-1uvzpxy">If the layer is stateless (it does not use member variables in its forward <em>or</em> it was
	originally a function that was converted into a kernel layer with
	<code>use_kernel_func_from_hub</code>), it can also be mapped to a kernel function:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->kernel_layer_mapping = {
	<span class="hljs-string">"SiluAndMul"</span>: {
	<span class="hljs-string">"cuda"</span>: FuncRepository(
	repo_id=<span class="hljs-string">"kernels-community/activation"</span>,
	func_name=<span class="hljs-string">"silu_and_mul"</span>,
	),
	}
	}<!-- HTML_TAG_END --></pre></div> <h3 class="relative group"><a id="registering-kernels-for-specific-modes" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#registering-kernels-for-specific-modes"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Registering kernels for specific modes</span></h3> <p data-svelte-h="svelte-1ed3b6y">You might want to register two different kernels for a particular layer,
	where one kernel is optimized for a specific mode. You can do so by
	registering layer repositories for specific modes. For example:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->kernel_layer_mapping = {
	<span class="hljs-string">"SiluAndMul"</span>: {
	<span class="hljs-string">"cuda"</span>: {
	Mode.INFERENCE: LayerRepository(
	repo_id=<span class="hljs-string">"kernels-community/activation-inference-optimized"</span>,
	layer_name=<span class="hljs-string">"SiluAndMul"</span>,
	version=<span class="hljs-number">1</span>,
	),
	Mode.TRAINING \| Mode.TORCH_COMPILE: LayerRepository(
	repo_id=<span class="hljs-string">"kernels-community/activation-training-optimized"</span>,
	layer_name=<span class="hljs-string">"SiluAndMul"</span>,
	version=<span class="hljs-number">1</span>,
	),
	}
	}
	}<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-sek2d8">The <code>kernelize</code> function will attempt to use the following registered
	kernels for a given mode:</p> <ul data-svelte-h="svelte-ewhtf4"><li><code>INFERENCE</code>: <code>INFERENCE</code> → <code>INFERENCE \| TORCH_COMPILE</code> → <code>TRAINING</code> →
	<code>TRAINING \| TORCH_COMPILE</code> → <code>FALLBACK</code></li> <li><code>INFERENCE \| TORCH_COMPILE</code>: <code>INFERENCE \| TORCH_COMPILE</code> →
	<code>TRAINING \| TORCH_COMPILE</code> → <code>FALLBACK</code></li> <li><code>TRAINING</code>: <code>TRAINING</code> → <code>TRAINING \| TORCH_COMPILE</code> → <code>FALLBACK</code></li> <li><code>TRAINING \| TORCH_COMPILE</code>: <code>TRAINING \| TORCH_COMPILE</code> → <code>FALLBACK</code></li></ul> <p data-svelte-h="svelte-m4furw"><code>Mode.FALLBACK</code> is a special mode that is used when no other mode matches. It
	is also used when a kernel is registered without a mode, as described in the
	previous section.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->kernel_layer_mapping = {
	<span class="hljs-string">"SiluAndMul"</span>: {
	<span class="hljs-string">"cuda"</span>: {
	Mode.FALLBACK: LayerRepository(
	repo_id=<span class="hljs-string">"kernels-community/activation"</span>,
	layer_name=<span class="hljs-string">"SiluAndMul"</span>,
	version=<span class="hljs-number">1</span>,
	),
	Mode.INFERENCE: LayerRepository(
	repo_id=<span class="hljs-string">"kernels-community/activation-inference-optimized"</span>,
	layer_name=<span class="hljs-string">"SiluAndMul"</span>,
	version=<span class="hljs-number">1</span>,
	),
	Mode.TRAINING: LayerRepository(
	repo_id=<span class="hljs-string">"kernels-community/activation-training-optimized"</span>,
	layer_name=<span class="hljs-string">"SiluAndMul"</span>,
	version=<span class="hljs-number">1</span>,
	),
	}
	}
	}<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-1oi4pcx">In this case, both <code>Mode.INFERENCE \| Mode.TORCH_COMPILE</code> and
	<code>Mode.TRAINING \| Mode.TORCH_COMPILE</code> will use the <code>Mode.FALLBACK</code> kernel,
	since the other kernels do not support <code>torch.compile</code>.</p> <h3 class="relative group"><a id="registering-kernels-for-specific-cuda-capabilities" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#registering-kernels-for-specific-cuda-capabilities"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Registering kernels for specific CUDA capabilities</span></h3> <p data-svelte-h="svelte-1wl92tu">Some kernels only work with newer CUDA architectures. For instance, some
	kernels require capability 9.0 for the TMA unit on Hopper GPUs. <code>kernels</code>
	supports registering layers for a range of CUDA capabilities. To do so,
	you need to register the layer for a <code>Device</code> with type <code>cuda</code> and
	set the supported range of CUDA capabilities with using <code>CUDAProperties</code>:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->kernel_layer_mapping = {
	<span class="hljs-string">"SiluAndMul"</span>: {
	Device(
	<span class="hljs-built_in">type</span>=<span class="hljs-string">"cuda"</span>,
	properties=CUDAProperties(
	min_capability=<span class="hljs-number">75</span>, max_capability=<span class="hljs-number">89</span>
	),
	): LayerRepository(
	repo_id=<span class="hljs-string">"kernels-community/activation"</span>,
	layer_name=<span class="hljs-string">"SiluAndMul"</span>,
	version=<span class="hljs-number">1</span>,
	),
	Device(
	<span class="hljs-built_in">type</span>=<span class="hljs-string">"cuda"</span>,
	properties=CUDAProperties(
	min_capability=<span class="hljs-number">90</span>, max_capability=sys.maxsize
	),
	): LayerRepository(
	repo_id=<span class="hljs-string">"kernels-community/activation-hopper"</span>,
	layer_name=<span class="hljs-string">"SiluAndMul"</span>,
	version=<span class="hljs-number">1</span>,
	),
	}
	}<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-1k7nbkd">Capabilities behave as follows:</p> <ul data-svelte-h="svelte-xyngc6"><li><p>The minimum and maximum capabilities are inclusive.</p></li> <li><p>When a new kernel is registered with the same min/max capabilities as
	an existing kernel, the new kernel will replace the old kernel.</p></li> <li><p>When there are multiple kernels that support a capability, the kernel
	with the smaller capability interval will be used. E.g. given:</p> <ul><li><code>KernelA</code> with <code>min_capability=80</code> and <code>max_capability=89</code>;</li> <li><code>KernelB</code> with <code>min_capability=75</code> and <code>max_capability=89</code>;</li> <li><code>kernelize</code> runs on a system with capability 8.6.</li></ul> <p>Then <code>KernelA</code> will be used because the interval 80..89 is smaller
	than 75..89. The motivation is that kernels with smaller ranges
	tend to be more optimized for a specific set of GPUs. <strong>This behavior
	might still change in the future.</strong></p></li></ul> <h3 class="relative group"><a id="registering-kernels-for-specific-rocm-capabilities" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#registering-kernels-for-specific-rocm-capabilities"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Registering kernels for specific ROCm capabilities</span></h3> <p data-svelte-h="svelte-1q9vv7t">Registering kernels for the ROCm architecture follows the exact same
	pattern as CUDA kernels, using <code>min_capability</code> and <code>max_capability</code> to restrict
	a kernel to a range of ROCm capabilities.</p> <h3 class="relative group"><a id="loading-from-a-local-repository-for-testing" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#loading-from-a-local-repository-for-testing"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Loading from a local repository for testing</span></h3> <p data-svelte-h="svelte-qa4iob">The <code>LocalLayerRepository</code> class is provided to load a repository from
	a local directory. For example:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-keyword">with</span> use_kernel_mapping(
	{
	<span class="hljs-string">"SiluAndMul"</span>: {
	<span class="hljs-string">"cuda"</span>: LocalLayerRepository(
	repo_path=<span class="hljs-string">"/home/daniel/kernels/activation"</span>,
	package_name=<span class="hljs-string">"activation"</span>,
	layer_name=<span class="hljs-string">"SiluAndMul"</span>,
	)
	}
	},
	inherit_mapping=<span class="hljs-literal">False</span>,
	):
	kernelize(linear, mode=Mode.INFERENCE)<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-f12z19">Similarly, the <code>LocalFuncRepository</code> class can be used to load a kernel
	function from a local directory:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-keyword">with</span> use_kernel_mapping(
	{
	<span class="hljs-string">"silu_and_mul"</span>: {
	<span class="hljs-string">"cuda"</span>: LocalFuncRepository(
	repo_path=<span class="hljs-string">"/home/daniel/kernels/activation"</span>,
	package_name=<span class="hljs-string">"activation"</span>,
	func_name=<span class="hljs-string">"silu_and_mul"</span>,
	)
	}
	},
	inherit_mapping=<span class="hljs-literal">False</span>,
	):
	kernelize(model, mode=Mode.INFERENCE)<!-- HTML_TAG_END --></pre></div> <a class="!text-gray-400 !no-underline text-sm flex items-center not-prose mt-4" href="https://github.com/huggingface/kernels/blob/main/docs/source/layers.md" target="_blank"><svg class="mr-1" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M31,16l-7,7l-1.41-1.41L28.17,16l-5.58-5.59L24,9l7,7z"></path><path d="M1,16l7-7l1.41,1.41L3.83,16l5.58,5.59L8,23l-7-7z"></path><path d="M12.419,25.484L17.639,6.552l1.932,0.518L14.351,26.002z"></path></svg> <span data-svelte-h="svelte-zjs2n5"><span class="underline">Update</span> on GitHub</span></a> <p></p>

	<script>
	{
	__sveltekit_lhya45 = {
	assets: "/docs/kernels/pr_520/en",
	base: "/docs/kernels/pr_520/en",
	env: {}
	};

	const element = document.currentScript.parentElement;

	const data = [null,null];

	Promise.all([
	import("/docs/kernels/pr_520/en/_app/immutable/entry/start.7813b24f.js"),
	import("/docs/kernels/pr_520/en/_app/immutable/entry/app.d97547bc.js")
	]).then(([kit, app]) => {
	kit.start(app, element, {
	node_ids: [0, 27],
	data,
	form: null,
	error: null
	});
	});
	}
	</script>

Xet Storage Details

Size:: 65.6 kB
Xet hash:: 6e596b75c3e94a4f6ceca764e69fcc3022c1ee5cf61879f4d231ea3f2c2f4882

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.