Buckets:
| <meta charset="utf-8" /><meta name="hf:doc:metadata" content="{"title":"Layers","local":"layers","sections":[{"title":"Making a layer extensible with kernels from the hub","local":"making-a-layer-extensible-with-kernels-from-the-hub","sections":[{"title":"Using a decorator","local":"using-a-decorator","sections":[],"depth":3},{"title":"External layers","local":"external-layers","sections":[],"depth":3}],"depth":2},{"title":"Kernelizing a model","local":"kernelizing-a-model","sections":[{"title":"Kernel device","local":"kernel-device","sections":[],"depth":3},{"title":"Fallback forward","local":"fallback-forward","sections":[],"depth":3},{"title":"Inspecting which kernels are used","local":"inspecting-which-kernels-are-used","sections":[],"depth":3}],"depth":2},{"title":"Registering a hub kernel for a layer","local":"registering-a-hub-kernel-for-a-layer","sections":[{"title":"Using version bounds","local":"using-version-bounds","sections":[],"depth":3},{"title":"Registering kernels for specific modes","local":"registering-kernels-for-specific-modes","sections":[],"depth":3},{"title":"Registering kernels for specific CUDA capabilities","local":"registering-kernels-for-specific-cuda-capabilities","sections":[],"depth":3},{"title":"Registering kernels for specific ROCm capabilities","local":"registering-kernels-for-specific-rocm-capabilities","sections":[],"depth":3},{"title":"Loading from a local repository for testing","local":"loading-from-a-local-repository-for-testing","sections":[],"depth":3}],"depth":2}],"depth":1}"> | |
| <link href="/docs/kernels/pr_150/en/_app/immutable/assets/0.e3b0c442.css" rel="modulepreload"> | |
| <link rel="modulepreload" href="/docs/kernels/pr_150/en/_app/immutable/entry/start.15f67972.js"> | |
| <link rel="modulepreload" href="/docs/kernels/pr_150/en/_app/immutable/chunks/scheduler.7da89386.js"> | |
| <link rel="modulepreload" href="/docs/kernels/pr_150/en/_app/immutable/chunks/singletons.e3529dd1.js"> | |
| <link rel="modulepreload" href="/docs/kernels/pr_150/en/_app/immutable/chunks/paths.dcb90d49.js"> | |
| <link rel="modulepreload" href="/docs/kernels/pr_150/en/_app/immutable/entry/app.13bdcfa3.js"> | |
| <link rel="modulepreload" href="/docs/kernels/pr_150/en/_app/immutable/chunks/index.0b7befd3.js"> | |
| <link rel="modulepreload" href="/docs/kernels/pr_150/en/_app/immutable/nodes/0.fadb2962.js"> | |
| <link rel="modulepreload" href="/docs/kernels/pr_150/en/_app/immutable/chunks/each.e59479a4.js"> | |
| <link rel="modulepreload" href="/docs/kernels/pr_150/en/_app/immutable/nodes/11.53874149.js"> | |
| <link rel="modulepreload" href="/docs/kernels/pr_150/en/_app/immutable/chunks/CodeBlock.c5b6371f.js"> | |
| <link rel="modulepreload" href="/docs/kernels/pr_150/en/_app/immutable/chunks/getInferenceSnippets.dcce2733.js"><!-- HEAD_svelte-u9bgzb_START --><meta name="hf:doc:metadata" content="{"title":"Layers","local":"layers","sections":[{"title":"Making a layer extensible with kernels from the hub","local":"making-a-layer-extensible-with-kernels-from-the-hub","sections":[{"title":"Using a decorator","local":"using-a-decorator","sections":[],"depth":3},{"title":"External layers","local":"external-layers","sections":[],"depth":3}],"depth":2},{"title":"Kernelizing a model","local":"kernelizing-a-model","sections":[{"title":"Kernel device","local":"kernel-device","sections":[],"depth":3},{"title":"Fallback forward","local":"fallback-forward","sections":[],"depth":3},{"title":"Inspecting which kernels are used","local":"inspecting-which-kernels-are-used","sections":[],"depth":3}],"depth":2},{"title":"Registering a hub kernel for a layer","local":"registering-a-hub-kernel-for-a-layer","sections":[{"title":"Using version bounds","local":"using-version-bounds","sections":[],"depth":3},{"title":"Registering kernels for specific modes","local":"registering-kernels-for-specific-modes","sections":[],"depth":3},{"title":"Registering kernels for specific CUDA capabilities","local":"registering-kernels-for-specific-cuda-capabilities","sections":[],"depth":3},{"title":"Registering kernels for specific ROCm capabilities","local":"registering-kernels-for-specific-rocm-capabilities","sections":[],"depth":3},{"title":"Loading from a local repository for testing","local":"loading-from-a-local-repository-for-testing","sections":[],"depth":3}],"depth":2}],"depth":1}"><!-- HEAD_svelte-u9bgzb_END --> <p></p> <h1 class="relative group"><a id="layers" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#layers"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Layers</span></h1> <p data-svelte-h="svelte-178x673">A kernel can provide layers in addition to kernel functions. A layer from | |
| the Hub can replace the <code>forward</code> method of an existing layer for a certain | |
| device type. This makes it possible to provide more performant kernels for | |
| existing layers.</p> <p data-svelte-h="svelte-j3oddw">See <a href="kernel-requirements.md">Kernel requirements</a> for more information on the | |
| requirements of Hub layers.</p> <h2 class="relative group"><a id="making-a-layer-extensible-with-kernels-from-the-hub" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#making-a-layer-extensible-with-kernels-from-the-hub"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Making a layer extensible with kernels from the hub</span></h2> <h3 class="relative group"><a id="using-a-decorator" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#using-a-decorator"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Using a decorator</span></h3> <p data-svelte-h="svelte-1jd41lf">A layer can be made extensible with the <code>use_kernel_forward_from_hub</code> | |
| decorator. For example:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-meta">@use_kernel_forward_from_hub(<span class="hljs-params"><span class="hljs-string">"SiluAndMul"</span></span>)</span> | |
| <span class="hljs-keyword">class</span> <span class="hljs-title class_">SiluAndMul</span>(nn.Module): | |
| <span class="hljs-keyword">def</span> <span class="hljs-title function_">forward</span>(<span class="hljs-params">self, <span class="hljs-built_in">input</span>: torch.Tensor</span>) -> torch.Tensor: | |
| d = <span class="hljs-built_in">input</span>.shape[-<span class="hljs-number">1</span>] // <span class="hljs-number">2</span> | |
| <span class="hljs-keyword">return</span> F.silu(<span class="hljs-built_in">input</span>[..., :d]) * <span class="hljs-built_in">input</span>[..., d:]<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-tily1x">The decorator does not change the behavior of the class — it annotates | |
| the class with the given name (here <code>SiluAndMul</code>). The <code>kernelize</code> function | |
| described below uses this name to look up kernels for the layer.</p> <h3 class="relative group"><a id="external-layers" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#external-layers"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>External layers</span></h3> <p data-svelte-h="svelte-1hj9ahw">An existing layer that does not (yet) have the <code>use_kernel_forward_from_hub</code> | |
| decorator can be made extensible using the <code>replace_kernel_forward_from_hub</code> | |
| function:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-keyword">from</span> somelibrary <span class="hljs-keyword">import</span> SiluAndMul | |
| replace_kernel_forward_from_hub(SiluAndMul, <span class="hljs-string">"SiluAndMul"</span>)<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-13coke1"><strong>Warning:</strong> we strongly recommend using layers with a decorator, since | |
| it signifies that the maintainer intends to keep the <code>forward</code> signature | |
| compatible with layers from the hub.</p> <h2 class="relative group"><a id="kernelizing-a-model" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#kernelizing-a-model"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Kernelizing a model</span></h2> <p data-svelte-h="svelte-stkx9y">A model will not use Hub kernels by default, even if it contains extensible | |
| layers. To enable the use of Hub kernels in the model, it needs to be | |
| ‘kernelized’ using the <code>kernelize</code> function. This function traverses the | |
| model graph and replaces the <code>forward</code> methods of extensible layers for which | |
| Hub kernels are registered. <code>kernelize</code> can be used as follows:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->model = MyModel(...) | |
| model = kernelize(model, mode=Mode.INFERENCE)<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-1mdicvi">The <code>kernelize</code> function modifies the model in-place, the model itself is | |
| returned as a convenience. The <code>mode</code> specifies that the model will be used | |
| in inference. Similarly, you can ask <code>kernelize</code> to prepare the model for | |
| training:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->model = MyModel(...) | |
| model = kernelize(model, mode=Mode.TRAINING)<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-xhc5xj">A model that is kernelized for training can also be used for inference, but | |
| not the other way around. If you want to change the mode of the kernelized | |
| model, you can just run <code>kernelize</code> on the model again with the new mode.</p> <p data-svelte-h="svelte-195gdzl">If you want to compile a model with <code>torch.compile</code>, this should be indicated | |
| in the mode as well. You can do this by combining <code>Mode.INFERENCE</code> or | |
| <code>Mode.TRAINING</code> with <code>Mode.TORCH_COMPILE</code> using the set union (<code>|</code>) operator:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->model = MyModel(...) | |
| <span class="hljs-comment"># Inference</span> | |
| model = kernelize(model, mode=Mode.INFERENCE | Mode.TORCH_COMPILE) | |
| <span class="hljs-comment"># Training</span> | |
| model = kernelize(model, mode=Mode.TRAINING | Mode.TORCH_COMPILE)<!-- HTML_TAG_END --></pre></div> <h3 class="relative group"><a id="kernel-device" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#kernel-device"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Kernel device</span></h3> <p data-svelte-h="svelte-808tau">Kernels can be registered per device type. For instance, separate <code>cuda</code> and | |
| <code>metal</code> kernels could be registered for the name <code>SiluAndMul</code>. By default, | |
| <code>kernelize</code> will try to infer the device type from the model’s parameters. | |
| You can pass the device type to <code>kernelize</code> if the device type cannot be | |
| inferred (e.g. because the model has no parameters):</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->model = MyModel(...) | |
| model = kernelize(model, device=<span class="hljs-string">"cuda"</span>, mode=Mode.INFERENCE)<!-- HTML_TAG_END --></pre></div> <h3 class="relative group"><a id="fallback-forward" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#fallback-forward"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Fallback forward</span></h3> <p data-svelte-h="svelte-yu926d">If the <code>TRAINING</code> and/or <code>TORCH_COMPILE</code> modes are used, but a registered | |
| kernel does not support backward passes or <code>torch.compile</code> respectively, | |
| <code>kernelize</code> will fall back to the original, non-kernelized, layer. You | |
| can let <code>kernelize</code> raise an exception instead by using <code>use_fallback=False</code>:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->model = MyModel(...) | |
| model = kernelize(model, mode=Mode.INFERENCE | Mode.TORCH_COMPILE, use_fallback=<span class="hljs-literal">False</span>)<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-1lkusth">This can be useful if you want to guarantee that Hub kernels are used.</p> <h3 class="relative group"><a id="inspecting-which-kernels-are-used" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#inspecting-which-kernels-are-used"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Inspecting which kernels are used</span></h3> <p data-svelte-h="svelte-161p6z8">The kernels that are used are logged at the <code>INFO</code> level by <code>kernelize</code>. | |
| See the <a href="https://docs.python.org/3/library/logging.html" rel="nofollow">Python logging</a> | |
| documentation for information on how to configure logging.</p> <h2 class="relative group"><a id="registering-a-hub-kernel-for-a-layer" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#registering-a-hub-kernel-for-a-layer"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Registering a hub kernel for a layer</span></h2> <p data-svelte-h="svelte-1r5il31"><code>kernelize</code> relies on kernel mappings to find Hub kernels for layers. | |
| Kernel mappings map a kernel name such as <code>SiluAndMul</code> to a kernel on | |
| the Hub. For example:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->kernel_layer_mapping = { | |
| <span class="hljs-string">"SiluAndMul"</span>: { | |
| <span class="hljs-string">"cuda"</span>: LayerRepository( | |
| repo_id=<span class="hljs-string">"kernels-community/activation"</span>, | |
| layer_name=<span class="hljs-string">"SiluAndMul"</span>, | |
| ), | |
| <span class="hljs-string">"rocm"</span>: LayerRepository( | |
| repo_id=<span class="hljs-string">"kernels-community/activation"</span>, | |
| layer_name=<span class="hljs-string">"SiluAndMul"</span>, | |
| ) | |
| } | |
| }<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-1mzc83p">You can register such a mapping using <code>register_kernel_mapping</code>:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->register_kernel_mapping(kernel_layer_mapping)<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-p5ro9n">This will register the kernel mapping in the current context, which is | |
| normally global. It is recommended to scope the mapping to where it is | |
| used with the <code>use_kernel_mapping</code> context manager:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-keyword">with</span> use_kernel_mapping(kernel_layer_mapping): | |
| <span class="hljs-comment"># Use the layer for which the mapping is applied.</span> | |
| model = kernelize(model, mode=Mode.TRAINING | Mode.TORCH_COMPILE)<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-z3xuuf">This ensures that the mapping is not active anymore outside the | |
| <code>with</code>-scope.</p> <h3 class="relative group"><a id="using-version-bounds" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#using-version-bounds"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Using version bounds</span></h3> <p data-svelte-h="svelte-130qoue">Kernels are versioned using tags of the form <code>v<major>.<minor>.<patch></code>. | |
| You can specify which version of the kernel to download using Python version | |
| specifiers:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->kernel_layer_mapping = { | |
| <span class="hljs-string">"SiluAndMul"</span>: { | |
| <span class="hljs-string">"cuda"</span>: LayerRepository( | |
| repo_id=<span class="hljs-string">"kernels-community/activation"</span>, | |
| layer_name=<span class="hljs-string">"SiluAndMul"</span>, | |
| version=<span class="hljs-string">">=0.0.4,<0.1.0"</span>, | |
| ), | |
| <span class="hljs-string">"rocm"</span>: LayerRepository( | |
| repo_id=<span class="hljs-string">"kernels-community/activation"</span>, | |
| layer_name=<span class="hljs-string">"SiluAndMul"</span>, | |
| version=<span class="hljs-string">">=0.0.4,<0.1.0"</span>, | |
| ) | |
| } | |
| }<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-8ksrq2">This will get the layer from latest kernel tagged <code>v0.0.z</code> where <code>z</code> is at | |
| least 4. It is strongly recommended to specify a version bound, since a | |
| kernel author might push incompatible changes to the <code>main</code> branch.</p> <h3 class="relative group"><a id="registering-kernels-for-specific-modes" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#registering-kernels-for-specific-modes"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Registering kernels for specific modes</span></h3> <p data-svelte-h="svelte-1ed3b6y">You might want to register two different kernels for a particular layer, | |
| where one kernel is optimized for a specific mode. You can do so by | |
| registering layer repositories for specific modes. For example:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->kernel_layer_mapping = { | |
| <span class="hljs-string">"SiluAndMul"</span>: { | |
| <span class="hljs-string">"cuda"</span>: { | |
| Mode.INFERENCE: LayerRepository( | |
| repo_id=<span class="hljs-string">"kernels-community/activation-inference-optimized"</span>, | |
| layer_name=<span class="hljs-string">"SiluAndMul"</span>, | |
| ), | |
| Mode.TRAINING | Mode.TORCH_COMPILE: LayerRepository( | |
| repo_id=<span class="hljs-string">"kernels-community/activation-training-optimized"</span>, | |
| layer_name=<span class="hljs-string">"SiluAndMul"</span>, | |
| ), | |
| } | |
| } | |
| }<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-sek2d8">The <code>kernelize</code> function will attempt to use the following registered | |
| kernels for a given mode:</p> <ul data-svelte-h="svelte-ewhtf4"><li><code>INFERENCE</code>: <code>INFERENCE</code> → <code>INFERENCE | TORCH_COMPILE</code> → <code>TRAINING</code> → | |
| <code>TRAINING | TORCH_COMPILE</code> → <code>FALLBACK</code></li> <li><code>INFERENCE | TORCH_COMPILE</code>: <code>INFERENCE | TORCH_COMPILE</code> → | |
| <code>TRAINING | TORCH_COMPILE</code> → <code>FALLBACK</code></li> <li><code>TRAINING</code>: <code>TRAINING</code> → <code>TRAINING | TORCH_COMPILE</code> → <code>FALLBACK</code></li> <li><code>TRAINING | TORCH_COMPILE</code>: <code>TRAINING | TORCH_COMPILE</code> → <code>FALLBACK</code></li></ul> <p data-svelte-h="svelte-m4furw"><code>Mode.FALLBACK</code> is a special mode that is used when no other mode matches. It | |
| is also used when a kernel is registered without a mode, as described in the | |
| previous section.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->kernel_layer_mapping = { | |
| <span class="hljs-string">"SiluAndMul"</span>: { | |
| <span class="hljs-string">"cuda"</span>: { | |
| Mode.FALLBACK: LayerRepository( | |
| repo_id=<span class="hljs-string">"kernels-community/activation"</span>, | |
| layer_name=<span class="hljs-string">"SiluAndMul"</span>, | |
| ), | |
| Mode.INFERENCE: LayerRepository( | |
| repo_id=<span class="hljs-string">"kernels-community/activation-inference-optimized"</span>, | |
| layer_name=<span class="hljs-string">"SiluAndMul"</span>, | |
| ), | |
| Mode.TRAINING: LayerRepository( | |
| repo_id=<span class="hljs-string">"kernels-community/activation-training-optimized"</span>, | |
| layer_name=<span class="hljs-string">"SiluAndMul"</span>, | |
| ), | |
| } | |
| } | |
| }<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-1oi4pcx">In this case, both <code>Mode.INFERENCE | Mode.TORCH_COMPILE</code> and | |
| <code>Mode.TRAINING | Mode.TORCH_COMPILE</code> will use the <code>Mode.FALLBACK</code> kernel, | |
| since the other kernels do not support <code>torch.compile</code>.</p> <h3 class="relative group"><a id="registering-kernels-for-specific-cuda-capabilities" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#registering-kernels-for-specific-cuda-capabilities"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Registering kernels for specific CUDA capabilities</span></h3> <p data-svelte-h="svelte-1wl92tu">Some kernels only work with newer CUDA architectures. For instance, some | |
| kernels require capability 9.0 for the TMA unit on Hopper GPUs. <code>kernels</code> | |
| supports registering layers for a range of CUDA capabilities. To do so, | |
| you need to register the layer for a <code>Device</code> with type <code>cuda</code> and | |
| set the supported range of CUDA capabilities with using <code>CUDAProperties</code>:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->kernel_layer_mapping = { | |
| <span class="hljs-string">"SiluAndMul"</span>: { | |
| Device( | |
| <span class="hljs-built_in">type</span>=<span class="hljs-string">"cuda"</span>, | |
| properties=CUDAProperties( | |
| min_capability=<span class="hljs-number">75</span>, max_capability=<span class="hljs-number">89</span> | |
| ), | |
| ): LayerRepository( | |
| repo_id=<span class="hljs-string">"kernels-community/activation"</span>, | |
| layer_name=<span class="hljs-string">"SiluAndMul"</span>, | |
| ), | |
| Device( | |
| <span class="hljs-built_in">type</span>=<span class="hljs-string">"cuda"</span>, | |
| properties=CUDAProperties( | |
| min_capability=<span class="hljs-number">90</span>, max_capability=sys.maxsize | |
| ), | |
| ): LayerRepository( | |
| repo_id=<span class="hljs-string">"kernels-community/activation-hopper"</span>, | |
| layer_name=<span class="hljs-string">"SiluAndMul"</span>, | |
| ), | |
| } | |
| }<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-1k7nbkd">Capabilities behave as follows:</p> <ul data-svelte-h="svelte-xyngc6"><li><p>The minimum and maximum capabilities are inclusive.</p></li> <li><p>When a new kernel is registered with the same min/max capabilities as | |
| an existing kernel, the new kernel will replace the old kernel.</p></li> <li><p>When there are multiple kernels that support a capability, the kernel | |
| with the smaller capability interval will be used. E.g. given:</p> <ul><li><code>KernelA</code> with <code>min_capability=80</code> and <code>max_capability=89</code>;</li> <li><code>KernelB</code> with <code>min_capability=75</code> and <code>max_capability=89</code>;</li> <li><code>kernelize</code> runs on a system with capability 8.6.</li></ul> <p>Then <code>KernelA</code> will be used because the interval 80..89 is smaller | |
| than 75..89. The motivation is that kernels with smaller ranges | |
| tend to be more optimized for a specific set of GPUs. <strong>This behavior | |
| might still change in the future.</strong></p></li></ul> <h3 class="relative group"><a id="registering-kernels-for-specific-rocm-capabilities" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#registering-kernels-for-specific-rocm-capabilities"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Registering kernels for specific ROCm capabilities</span></h3> <p data-svelte-h="svelte-1q9vv7t">Registering kernels for the ROCm architecture follows the exact same | |
| pattern as CUDA kernels, using <code>min_capability</code> and <code>max_capability</code> to restrict | |
| a kernel to a range of ROCm capabilities.</p> <h3 class="relative group"><a id="loading-from-a-local-repository-for-testing" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#loading-from-a-local-repository-for-testing"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Loading from a local repository for testing</span></h3> <p data-svelte-h="svelte-qa4iob">The <code>LocalLayerRepository</code> class is provided to load a repository from | |
| a local directory. For example:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-keyword">with</span> use_kernel_mapping( | |
| { | |
| <span class="hljs-string">"SiluAndMul"</span>: { | |
| <span class="hljs-string">"cuda"</span>: LocalLayerRepository( | |
| repo_path=<span class="hljs-string">"/home/daniel/kernels/activation"</span>, | |
| package_name=<span class="hljs-string">"activation"</span>, | |
| layer_name=<span class="hljs-string">"SiluAndMul"</span>, | |
| ) | |
| } | |
| }, | |
| inherit_mapping=<span class="hljs-literal">False</span>, | |
| ): | |
| kernelize(linear, mode=Mode.INFERENCE)<!-- HTML_TAG_END --></pre></div> <a class="!text-gray-400 !no-underline text-sm flex items-center not-prose mt-4" href="https://github.com/huggingface/kernels/blob/main/docs/source/layers.md" target="_blank"><span data-svelte-h="svelte-1kd6by1"><</span> <span data-svelte-h="svelte-x0xyl0">></span> <span data-svelte-h="svelte-1dajgef"><span class="underline ml-1.5">Update</span> on GitHub</span></a> <p></p> | |
| <script> | |
| { | |
| __sveltekit_cbfb4u = { | |
| assets: "/docs/kernels/pr_150/en", | |
| base: "/docs/kernels/pr_150/en", | |
| env: {} | |
| }; | |
| const element = document.currentScript.parentElement; | |
| const data = [null,null]; | |
| Promise.all([ | |
| import("/docs/kernels/pr_150/en/_app/immutable/entry/start.15f67972.js"), | |
| import("/docs/kernels/pr_150/en/_app/immutable/entry/app.13bdcfa3.js") | |
| ]).then(([kit, app]) => { | |
| kit.start(app, element, { | |
| node_ids: [0, 11], | |
| data, | |
| form: null, | |
| error: null | |
| }); | |
| }); | |
| } | |
| </script> | |
Xet Storage Details
- Size:
- 56.1 kB
- Xet hash:
- b8c404f2ed04600f773a8605de85fb126e026864ff567f8b06d7e4c292b38d83
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.