Hugging Face's logo

Buckets:
hf-doc-build
/
doc-dev

Files
xet
hf-doc-build/doc-dev / kernels /pr_898 /en /use_layers.html
HuggingFaceDocBuilder's picture
HuggingFaceDocBuilder
download
raw
70.2 kB
<meta charset="utf-8" /><meta name="hf:doc:metadata" content="{&quot;title&quot;:&quot;Kernelize your model with ease&quot;,&quot;local&quot;:&quot;kernelize-your-model-with-ease&quot;,&quot;sections&quot;:[{&quot;title&quot;:&quot;Installs and Imports&quot;,&quot;local&quot;:&quot;installs-and-imports&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Download a kernel&quot;,&quot;local&quot;:&quot;download-a-kernel&quot;,&quot;sections&quot;:[{&quot;title&quot;:&quot;Inspect the kernel module&quot;,&quot;local&quot;:&quot;inspect-the-kernel-module&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3}],&quot;depth&quot;:2},{&quot;title&quot;:&quot;How to use a kernel function?&quot;,&quot;local&quot;:&quot;how-to-use-a-kernel-function&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Why do we need Kernel Layers?&quot;,&quot;local&quot;:&quot;why-do-we-need-kernel-layers&quot;,&quot;sections&quot;:[{&quot;title&quot;:&quot;Build a triton kernel manually and patch it&quot;,&quot;local&quot;:&quot;build-a-triton-kernel-manually-and-patch-it&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3},{&quot;title&quot;:&quot;Build a model with kernel layer&quot;,&quot;local&quot;:&quot;build-a-model-with-kernel-layer&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3},{&quot;title&quot;:&quot;Replace a layer definition&quot;,&quot;local&quot;:&quot;replace-a-layer-definition&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3},{&quot;title&quot;:&quot;Using kernel layers with functions&quot;,&quot;local&quot;:&quot;using-kernel-layers-with-functions&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3},{&quot;title&quot;:&quot;Kernel Mapping&quot;,&quot;local&quot;:&quot;kernel-mapping&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3}],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Are kernels useful?&quot;,&quot;local&quot;:&quot;are-kernels-useful&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Conclusion&quot;,&quot;local&quot;:&quot;conclusion&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2}],&quot;depth&quot;:1}"/>
<link href="/docs/kernels/pr_898/en/_app/immutable/entry/start.BxCXruW1.js" rel="modulepreload">
<link href="/docs/kernels/pr_898/en/_app/immutable/chunks/A2qD3dbS.js" rel="modulepreload">
<link href="/docs/kernels/pr_898/en/_app/immutable/chunks/CH-Vfalm.js" rel="modulepreload">
<link href="/docs/kernels/pr_898/en/_app/immutable/entry/app.CzfFppSd.js" rel="modulepreload">
<link href="/docs/kernels/pr_898/en/_app/immutable/chunks/BDXICjAe.js" rel="modulepreload">
<link href="/docs/kernels/pr_898/en/_app/immutable/chunks/BQK_Ww-U.js" rel="modulepreload">
<link href="/docs/kernels/pr_898/en/_app/immutable/chunks/BHxxGx0x.js" rel="modulepreload">
<link href="/docs/kernels/pr_898/en/_app/immutable/chunks/DsnmJJEf.js" rel="modulepreload">
<link href="/docs/kernels/pr_898/en/_app/immutable/nodes/0.BbP0ADgf.js" rel="modulepreload">
<link href="/docs/kernels/pr_898/en/_app/immutable/chunks/DcJV8mP8.js" rel="modulepreload">
<link href="/docs/kernels/pr_898/en/_app/immutable/nodes/5.DXWJlZ1f.js" rel="modulepreload">
<link href="/docs/kernels/pr_898/en/_app/immutable/chunks/BXJlsa6-.js" rel="modulepreload">
<!--kuidxr--><meta name="hf:doc:metadata" content="{&quot;title&quot;:&quot;Kernelize your model with ease&quot;,&quot;local&quot;:&quot;kernelize-your-model-with-ease&quot;,&quot;sections&quot;:[{&quot;title&quot;:&quot;Installs and Imports&quot;,&quot;local&quot;:&quot;installs-and-imports&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Download a kernel&quot;,&quot;local&quot;:&quot;download-a-kernel&quot;,&quot;sections&quot;:[{&quot;title&quot;:&quot;Inspect the kernel module&quot;,&quot;local&quot;:&quot;inspect-the-kernel-module&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3}],&quot;depth&quot;:2},{&quot;title&quot;:&quot;How to use a kernel function?&quot;,&quot;local&quot;:&quot;how-to-use-a-kernel-function&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Why do we need Kernel Layers?&quot;,&quot;local&quot;:&quot;why-do-we-need-kernel-layers&quot;,&quot;sections&quot;:[{&quot;title&quot;:&quot;Build a triton kernel manually and patch it&quot;,&quot;local&quot;:&quot;build-a-triton-kernel-manually-and-patch-it&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3},{&quot;title&quot;:&quot;Build a model with kernel layer&quot;,&quot;local&quot;:&quot;build-a-model-with-kernel-layer&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3},{&quot;title&quot;:&quot;Replace a layer definition&quot;,&quot;local&quot;:&quot;replace-a-layer-definition&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3},{&quot;title&quot;:&quot;Using kernel layers with functions&quot;,&quot;local&quot;:&quot;using-kernel-layers-with-functions&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3},{&quot;title&quot;:&quot;Kernel Mapping&quot;,&quot;local&quot;:&quot;kernel-mapping&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3}],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Are kernels useful?&quot;,&quot;local&quot;:&quot;are-kernels-useful&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Conclusion&quot;,&quot;local&quot;:&quot;conclusion&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2}],&quot;depth&quot;:1}"/><!---->
<link href="/docs/kernels/pr_898/en/_app/immutable/assets/0.tn0RQdqM.css" rel="modulepreload"> <!--[--><!--[0--><!--[--><!--[0--><!--[--><p></p> <div class="items-center shrink-0 min-w-[100px] max-sm:min-w-[50px] justify-end ml-auto flex" style="float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"><div class="inline-flex rounded-md max-sm:rounded-sm"><button class="inline-flex items-center gap-1 h-7 max-sm:h-7 px-2 max-sm:px-1.5 text-sm font-medium text-gray-800 border border-r-0 rounded-l-md max-sm:rounded-l-sm border-gray-200 bg-white hover:shadow-inner dark:border-gray-850 dark:bg-gray-950 dark:text-gray-200 dark:hover:bg-gray-800" aria-live="polite"><span class="inline-flex items-center justify-center rounded-md p-0.5 max-sm:p-0 hover:text-gray-800 dark:hover:text-gray-200"><svg class="sm:size-3.5 size-3" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----></span> <span>Copy page</span></button> <button class="inline-flex items-center justify-center w-6 max-sm:w-5 h-7 max-sm:h-7 disabled:pointer-events-none text-sm text-gray-500 hover:text-gray-700 dark:hover:text-white rounded-r-md max-sm:rounded-r-sm border border-l transition border-gray-200 bg-white hover:shadow-inner dark:border-gray-850 dark:bg-gray-950 dark:text-gray-200 dark:hover:bg-gray-800" aria-haspopup="menu" aria-expanded="false" aria-label="Open copy menu"><svg class="transition-transform text-gray-400 overflow-visible sm:size-3.5 size-3 rotate-0" width="1em" height="1em" viewBox="0 0 12 7" fill="none" xmlns="http://www.w3.org/2000/svg"><path d="M1 1L6 6L11 1" stroke="currentColor"></path></svg><!----></button></div> <!--[-1--><!--]--></div><!----> <div class="flex space-x-1 " style="float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"><!----> <!--[1--><div class="relative colab-dropdown "><button class=" " type="button"><!--[0--><!--[-1--><img alt="Open In Colab" class="!m-0" src="https://colab.research.google.com/assets/colab-badge.svg"/><!--]--><!----><!--]--></button> <!--[-1--><!--]--></div><!--]--> <!--[1--><div class="relative colab-dropdown "><button class=" " type="button"><!--[0--><!--[-1--><img alt="Open In Studio Lab" class="!m-0" src="https://studiolab.sagemaker.aws/studiolab.svg"/><!--]--><!----><!--]--></button> <!--[-1--><!--]--></div><!--]--></div><!----> <!--[0--><h1 class="relative group"><a id="kernelize-your-model-with-ease" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#kernelize-your-model-with-ease"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>Kernelize your model with ease</span></h1><!--]--><!----> <table><thead><tr><th align="left">Authors</th></tr></thead><tbody><tr><td align="left"><a href="https://huggingface.co/ariG23498" rel="nofollow">Aritra Roy Gosthipaty</a></td></tr><tr><td align="left"><a href="https://huggingface.co/sayakpaul" rel="nofollow">Sayak Paul</a></td></tr><tr><td align="left"><a href="https://huggingface.co/danieldk" rel="nofollow">Daniël de Kok</a></td></tr></tbody></table> <p>A compute kernel is a function than can be run on an accelerator (GPUs or TPUs). A kernel can be launched on many compute units (Streaming Multiprocessors in case of NVIDIA GPUs) in parallel which is how it achieves arithmetic density.</p> <p>There are two main branches while working with kernels:</p> <ol><li>Building kernels</li> <li>Using built kernels</li></ol> <p>While building kernels is an art form, in this tutorial we will showcase how we can leverage prebuilt kernels in our models. We use <a href="https://huggingface.co/docs/kernels/index" rel="nofollow"><code>kernels</code> library</a>, inspect kernels (from Hugging Face Hub), and use them inside a PyTorch model.</p> <blockquote><p>🙏 This is an interactive notebook, so feel free to make changes and experiment quickly.</p></blockquote> <!--[1--><h2 class="relative group"><a id="installs-and-imports" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#installs-and-imports"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>Installs and Imports</span></h2><!--]--><!----> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!---->!pip install kernels -qq<!----></pre></div><!----> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-keyword">import</span> sys
<span class="hljs-keyword">import</span> inspect
<span class="hljs-keyword">import</span> logging
logging.basicConfig(level=logging.INFO, force=<span class="hljs-literal">True</span>) <span class="hljs-comment"># important to see the kernels in action</span>
<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> torch <span class="hljs-keyword">import</span> nn
<span class="hljs-keyword">from</span> torch.nn <span class="hljs-keyword">import</span> functional <span class="hljs-keyword">as</span> F
<span class="hljs-keyword">from</span> kernels <span class="hljs-keyword">import</span> get_kernel
<span class="hljs-keyword">from</span> kernels <span class="hljs-keyword">import</span> CUDAProperties
<span class="hljs-keyword">from</span> kernels <span class="hljs-keyword">import</span> LayerRepository, Device
<span class="hljs-keyword">from</span> kernels <span class="hljs-keyword">import</span> kernelize, Mode
<span class="hljs-keyword">from</span> kernels <span class="hljs-keyword">import</span> use_kernel_mapping
<span class="hljs-keyword">from</span> kernels <span class="hljs-keyword">import</span> use_kernelized_func
<span class="hljs-keyword">from</span> kernels <span class="hljs-keyword">import</span> use_kernel_forward_from_hub
<span class="hljs-keyword">from</span> kernels <span class="hljs-keyword">import</span> replace_kernel_forward_from_hub
<span class="hljs-keyword">import</span> triton
<span class="hljs-keyword">import</span> triton.language <span class="hljs-keyword">as</span> tl<!----></pre></div><!----> <!--[1--><h2 class="relative group"><a id="download-a-kernel" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#download-a-kernel"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>Download a kernel</span></h2><!--]--><!----> <p>If you are looking for a model or a dataset, you go to <a href="https://huggingface.co/" rel="nofollow">Hugging Face Hub</a> and select from the wide variety of options. What do you do when you are in search for a compute kernel?</p> <p>With the introduction of <a href="https://huggingface.co/" rel="nofollow"><code>kernels</code></a> you can go to the Hub, click on the <a href="https://huggingface.co/kernels" rel="nofollow">kernels tab</a>, and choose from the wide variety of options 🤗. Compute kernels are now a seperate repository type on the Hub.</p> <p>This means, we can:</p> <ol><li>Go the Hub</li> <li>Select the kernel repository</li> <li>Download the kernel and use it</li></ol> <p>Let’s get started!</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!---->activation_kernel_id = <span class="hljs-string">&quot;kernels-community/activation&quot;</span>
activation_kernel_module = get_kernel(activation_kernel_id, version=<span class="hljs-number">1</span>)<!----></pre></div><!----> <!--[2--><h3 class="relative group"><a id="inspect-the-kernel-module" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#inspect-the-kernel-module"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>Inspect the kernel module</span></h3><!--]--><!----> <p>A kernel module usually holds kernel functions and kernel layers. Functions (as one would know) are input and output mapping. The idea behind a kernel function is that they are optimized for the hardware that you are using. A kernel layer on the other hand can be thought of as an optimized PyTorch module.</p> <p>Let’s first inspect what the <code>kernels-community/activation</code> module holds for us.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!---->functions_available = inspect.getmembers(activation_kernel_module, inspect.isfunction)
layers_available = inspect.getmembers(activation_kernel_module.layers, inspect.isclass)
<span class="hljs-built_in">print</span>(<span class="hljs-string">&quot;Functions:&quot;</span>)
<span class="hljs-keyword">for</span> function <span class="hljs-keyword">in</span> functions_available:
<span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot; <span class="hljs-subst">{function[<span class="hljs-number">0</span>]}</span>&quot;</span>)
<span class="hljs-built_in">print</span>(<span class="hljs-string">&quot;\nLayers:&quot;</span>)
<span class="hljs-keyword">for</span> layer <span class="hljs-keyword">in</span> layers_available:
<span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot; <span class="hljs-subst">{layer[<span class="hljs-number">0</span>]}</span>&quot;</span>)<!----></pre></div><!----> <!--[1--><h2 class="relative group"><a id="how-to-use-a-kernel-function" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#how-to-use-a-kernel-function"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>How to use a kernel function?</span></h2><!--]--><!----> <p>While this cookbook (or tutorial, what ever you want to call it) is intended on showing how one would kernelize their model using a kernel layer, it is quite important to understand how the kernel functions work.</p> <p>The idea is to create the input, get the kernel function ready to be used, and then create an empty tensor that will hold the output.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-built_in">input</span> = torch.randn(<span class="hljs-number">100</span>, <span class="hljs-number">100</span>, device=<span class="hljs-string">&quot;cuda&quot;</span>, dtype=torch.float16)
kernel_output = torch.empty_like(<span class="hljs-built_in">input</span>)
torch_output = F.gelu(<span class="hljs-built_in">input</span>)
activation_kernel_module.gelu(kernel_output, <span class="hljs-built_in">input</span>)
<span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;Torch matches Kernel Functions: <span class="hljs-subst">{torch.allclose(torch_output, kernel_output)}</span>&quot;</span>)<!----></pre></div><!----> <!--[1--><h2 class="relative group"><a id="why-do-we-need-kernel-layers" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#why-do-we-need-kernel-layers"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>Why do we need Kernel Layers?</span></h2><!--]--><!----> <p>With kernel functions one would need to:</p> <ol><li>highlight the layer</li> <li>replace the layer’s <code>forward</code> with the kernel function</li></ol> <p>We wanted to make this more elegant. With a kernel layer, you can just follow:</p> <ol><li>annotate</li> <li>register</li> <li><code>kernelize()</code></li></ol> <p>to achieve the <code>forward</code> swap withouth having to hardcode anything. This enhances readability of code, make the intent more visible and also introduces composability.</p> <p>To see this in action, let’s first build a simple model and iterate on it.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-keyword">class</span> <span class="hljs-title class_">MLP</span>(nn.Module):
<span class="hljs-keyword">def</span> <span class="hljs-title function_">__init__</span>(<span class="hljs-params">self, d_in=<span class="hljs-number">1024</span>, d_hidden=<span class="hljs-number">4096</span>, d_out=<span class="hljs-number">1024</span></span>):
<span class="hljs-built_in">super</span>().__init__()
<span class="hljs-variable language_">self</span>.fc1 = nn.Linear(d_in, d_hidden)
<span class="hljs-variable language_">self</span>.act = nn.GELU()
<span class="hljs-variable language_">self</span>.fc2 = nn.Linear(d_hidden, d_out)
<span class="hljs-keyword">def</span> <span class="hljs-title function_">forward</span>(<span class="hljs-params">self, x</span>):
<span class="hljs-keyword">return</span> <span class="hljs-variable language_">self</span>.fc2(<span class="hljs-variable language_">self</span>.act(<span class="hljs-variable language_">self</span>.fc1(x)))
model = MLP().to(<span class="hljs-string">&quot;cuda&quot;</span>).<span class="hljs-built_in">eval</span>()
x = torch.randn(<span class="hljs-number">64</span>, <span class="hljs-number">1024</span>, device=<span class="hljs-string">&quot;cuda&quot;</span>)
<span class="hljs-keyword">with</span> torch.no_grad():
ref_out = model(x)
<span class="hljs-built_in">print</span>(ref_out.shape)<!----></pre></div><!----> <!--[2--><h3 class="relative group"><a id="build-a-triton-kernel-manually-and-patch-it" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#build-a-triton-kernel-manually-and-patch-it"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>Build a triton kernel manually and patch it</span></h3><!--]--><!----> <p>Here we build a very simple <code>gelu</code> kernel using <a href="https://triton-lang.org/main/index.html" rel="nofollow"><code>triton</code></a>. Readers don’t need to follow this code with great detail, as this is just for demonstration. It is not tuned for the GPUs being used.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-meta">@triton.jit</span>
<span class="hljs-keyword">def</span> <span class="hljs-title function_">gelu_kernel</span>(<span class="hljs-params">x_ptr, out_ptr, n_elements, BLOCK_SIZE: tl.constexpr</span>):
pid = tl.program_id(axis=<span class="hljs-number">0</span>)
offsets = pid * BLOCK_SIZE + tl.arange(<span class="hljs-number">0</span>, BLOCK_SIZE)
mask = offsets &lt; n_elements
x = tl.load(x_ptr + offsets, mask=mask)
xf = x.to(tl.float32)
y = <span class="hljs-number">0.5</span> * xf * (<span class="hljs-number">1.0</span> + tl.math.erf(xf * <span class="hljs-number">0.7071067811865476</span>))
tl.store(out_ptr + offsets, y.to(x.dtype), mask=mask)
<span class="hljs-keyword">def</span> <span class="hljs-title function_">triton_gelu</span>(<span class="hljs-params">x: torch.Tensor</span>) -&gt; torch.Tensor:
x = x.contiguous()
out = torch.empty_like(x)
n = x.numel()
BLOCK_SIZE = <span class="hljs-number">1024</span>
grid = (triton.cdiv(n, BLOCK_SIZE),)
gelu_kernel[grid](x, out, n, BLOCK_SIZE=BLOCK_SIZE)
<span class="hljs-keyword">return</span> out
<span class="hljs-comment"># sanity check against PyTorch</span>
t = torch.randn(<span class="hljs-number">10_000</span>, device=<span class="hljs-string">&quot;cuda&quot;</span>)
<span class="hljs-built_in">print</span>(<span class="hljs-string">&quot;Triton matches F.gelu:&quot;</span>, torch.allclose(triton_gelu(t), F.gelu(t), atol=<span class="hljs-number">1e-5</span>))<!----></pre></div><!----> <p>To use this triton kernel inside our model, we would have to patch the <code>forward</code> of the <code>self.act</code> layer with the triton kernel. This is what we do next.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!---->temp_model = MLP().to(<span class="hljs-string">&quot;cuda&quot;</span>).<span class="hljs-built_in">eval</span>()
temp_model.load_state_dict(model.state_dict())
temp_model.act.forward = triton_gelu <span class="hljs-comment"># patch activation forward with the kernel</span>
<span class="hljs-keyword">with</span> torch.no_grad():
triton_out = temp_model(x)
<span class="hljs-built_in">print</span>(<span class="hljs-string">&quot;Model output matches original:&quot;</span>, torch.allclose(triton_out, ref_out, atol=<span class="hljs-number">1e-5</span>))<!----></pre></div><!----> <p>Now let’s see why this is a really clumsy way of doing things:</p> <ol><li>the lack of composability, everything here is hard-coded and has no leeway to compose models to our liking</li> <li>manually monkey pathcing always leads to convoluted code bases</li> <li>no tooling present around patching methods with kernels</li></ol> <p>To mitigate these issues and more, we introduce kernel layers.</p> <!--[2--><h3 class="relative group"><a id="build-a-model-with-kernel-layer" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#build-a-model-with-kernel-layer"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>Build a model with kernel layer</span></h3><!--]--><!----> <p>Here we take you through the process of using kernel layers inside a model. This is a really helpful way to use kernels if you had the freedom to build the model from scratch.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-comment"># 1. Annotate the layer</span>
<span class="hljs-meta">@use_kernel_forward_from_hub(<span class="hljs-params"><span class="hljs-string">&quot;Gelu&quot;</span></span>)</span>
<span class="hljs-keyword">class</span> <span class="hljs-title class_">Gelu</span>(nn.Module):
<span class="hljs-keyword">def</span> <span class="hljs-title function_">forward</span>(<span class="hljs-params">self, <span class="hljs-built_in">input</span>: torch.Tensor</span>) -&gt; torch.Tensor:
<span class="hljs-keyword">return</span> F.gelu(<span class="hljs-built_in">input</span>)
<span class="hljs-comment"># 2. Build the registry mapping</span>
kernel_layer_mapping = {
<span class="hljs-string">&quot;Gelu&quot;</span>: {
<span class="hljs-string">&quot;cuda&quot;</span>: LayerRepository(
repo_id=<span class="hljs-string">&quot;kernels-community/activation&quot;</span>,
layer_name=<span class="hljs-string">&quot;Gelu&quot;</span>,
version=<span class="hljs-number">1</span>,
),
}
}
<span class="hljs-comment"># 3. Build the model (from source)</span>
<span class="hljs-keyword">class</span> <span class="hljs-title class_">MLP</span>(nn.Module):
<span class="hljs-keyword">def</span> <span class="hljs-title function_">__init__</span>(<span class="hljs-params">self, d_in=<span class="hljs-number">1024</span>, d_hidden=<span class="hljs-number">4096</span>, d_out=<span class="hljs-number">1024</span></span>):
<span class="hljs-built_in">super</span>().__init__()
<span class="hljs-variable language_">self</span>.fc1 = nn.Linear(d_in, d_hidden)
<span class="hljs-variable language_">self</span>.act = Gelu()
<span class="hljs-variable language_">self</span>.fc2 = nn.Linear(d_hidden, d_out)
<span class="hljs-keyword">def</span> <span class="hljs-title function_">forward</span>(<span class="hljs-params">self, x</span>):
<span class="hljs-keyword">return</span> <span class="hljs-variable language_">self</span>.fc2(<span class="hljs-variable language_">self</span>.act(<span class="hljs-variable language_">self</span>.fc1(x)))
<span class="hljs-comment"># 4. Kernelize the model</span>
<span class="hljs-keyword">with</span> use_kernel_mapping(kernel_layer_mapping):
kernelized_model = kernelize(MLP().to(<span class="hljs-string">&quot;cuda&quot;</span>), device=<span class="hljs-string">&quot;cuda&quot;</span>, mode=Mode.INFERENCE)
kernelized_model.load_state_dict(model.state_dict()) <span class="hljs-comment"># load the previous weights for equivalence testing</span>
<span class="hljs-keyword">with</span> torch.no_grad():
hub_out = kernelized_model(x)
<span class="hljs-built_in">print</span>(<span class="hljs-string">&quot;Hub kernel output matches original:&quot;</span>, torch.allclose(hub_out, ref_out, atol=<span class="hljs-number">1e-5</span>))<!----></pre></div><!----> <p>We went through a bunch of steps all of which achieved the patching (as shown in the triton example above) in a more elegant way.</p> <ol><li>We annotate the layer that we want to replace with a kernel layer form the Hub.</li> <li>We then create a mapping for the layer. Here we map the layer to the kernel id and the actual layer we want to use.</li> <li>After that we build the model using our layer.</li> <li>Finally we <code>kernelize()</code> the model: This takes care of traversing through the model and replacing the layer’s <code>forward</code> with our kernel layer from the Hub.</li></ol> <p>If you remember the annotate, map, and kernelize formula you are good to go!</p> <p>This is quite intuitive in our opinon (but we are biased).</p> <!--[2--><h3 class="relative group"><a id="replace-a-layer-definition" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#replace-a-layer-definition"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>Replace a layer definition</span></h3><!--]--><!----> <p>There is another way to kernelize a model using a kernel layer. If you know the definition of the layer you can use our handy <code>replace_kernel_forwared_from_hub</code> API. This annotates the layer definition and <code>kernelize</code> will later replace it with the kernel layer.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-comment"># the model in question</span>
<span class="hljs-built_in">print</span>(model)<!----></pre></div><!----> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-comment"># replace nn.GELU with the kernel Gelu layer</span>
replace_kernel_forward_from_hub(nn.GELU, <span class="hljs-string">&quot;Gelu&quot;</span>)
<span class="hljs-comment"># register the mapping</span>
kernel_layer_mapping = {
<span class="hljs-string">&quot;Gelu&quot;</span>: {
<span class="hljs-string">&quot;cuda&quot;</span>: LayerRepository(
repo_id=<span class="hljs-string">&quot;kernels-community/activation&quot;</span>,
layer_name=<span class="hljs-string">&quot;Gelu&quot;</span>,
version=<span class="hljs-number">1</span>,
),
}
}
<span class="hljs-comment"># kernelize the model</span>
<span class="hljs-keyword">with</span> use_kernel_mapping(kernel_layer_mapping):
kernelized_model = kernelize(model.to(<span class="hljs-string">&quot;cuda&quot;</span>), device=<span class="hljs-string">&quot;cuda&quot;</span>, mode=Mode.INFERENCE)
kernelized_model.load_state_dict(model.state_dict()) <span class="hljs-comment"># load the previous weights for equivalence testing</span>
<span class="hljs-keyword">with</span> torch.no_grad():
hub_out = kernelized_model(x)
<span class="hljs-built_in">print</span>(<span class="hljs-string">&quot;Hub kernel output matches original:&quot;</span>, torch.allclose(hub_out, ref_out, atol=<span class="hljs-number">1e-5</span>))<!----></pre></div><!----> <p>Here we replace the <code>nn.GELU</code> torch module with the <code>Gelu</code> kernel layer we have. To do that we use the <code>replace_kernel_forward_from_hub</code> method. It is also worth noting that the method is equivalent to annotation, modifying the class.</p> <p>We strongly recommended the model developers to add <code>use_kernel_forward_from_hub</code> annotations and that <code>replace_kernel_forward_from_hub</code> is really a last resort. It relies on the upstream layer not changing in an incompatible way, whereas using the decorator implies a contract.</p> <!--[2--><h3 class="relative group"><a id="using-kernel-layers-with-functions" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#using-kernel-layers-with-functions"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>Using kernel layers with functions</span></h3><!--]--><!----> <p>There are times that you have a function that behaves exactly like the layer’s <code>forward</code>. We provide a handy decorator <code>use_kernel_forward_from_hub</code> which can be used to decorate the function in concern.</p> <p>Something to note here is that decoration of functions are really for special cases. It comes in handy when you functions to kernelize, but they cannot replace the function by a layer for e.g. compatibility reasons, API breakage, etc.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-meta">@use_kernel_forward_from_hub(<span class="hljs-params"><span class="hljs-string">&quot;Gelu&quot;</span></span>)</span>
<span class="hljs-keyword">def</span> <span class="hljs-title function_">gelu</span>(<span class="hljs-params"><span class="hljs-built_in">input</span></span>):
<span class="hljs-keyword">return</span> F.gelu(<span class="hljs-built_in">input</span>)
<span class="hljs-meta">@use_kernelized_func(<span class="hljs-params">gelu</span>)</span>
<span class="hljs-keyword">class</span> <span class="hljs-title class_">MLP</span>(nn.Module):
<span class="hljs-keyword">def</span> <span class="hljs-title function_">__init__</span>(<span class="hljs-params">self, d_in=<span class="hljs-number">1024</span>, d_hidden=<span class="hljs-number">4096</span>, d_out=<span class="hljs-number">1024</span></span>):
<span class="hljs-built_in">super</span>().__init__()
<span class="hljs-variable language_">self</span>.fc1 = nn.Linear(d_in, d_hidden)
<span class="hljs-variable language_">self</span>.fc2 = nn.Linear(d_hidden, d_out)
<span class="hljs-keyword">def</span> <span class="hljs-title function_">forward</span>(<span class="hljs-params">self, x</span>):
<span class="hljs-keyword">return</span> <span class="hljs-variable language_">self</span>.fc2(gelu(<span class="hljs-variable language_">self</span>.fc1(x))) <span class="hljs-comment"># notice the use of the function</span>
kernel_layer_mapping = {
<span class="hljs-string">&quot;Gelu&quot;</span>: {
<span class="hljs-string">&quot;cuda&quot;</span>: LayerRepository(
repo_id=<span class="hljs-string">&quot;kernels-community/activation&quot;</span>,
layer_name=<span class="hljs-string">&quot;Gelu&quot;</span>,
version=<span class="hljs-number">1</span>,
),
}
}
<span class="hljs-keyword">with</span> use_kernel_mapping(kernel_layer_mapping):
kernelized_model = kernelize(MLP().to(<span class="hljs-string">&quot;cuda&quot;</span>), device=<span class="hljs-string">&quot;cuda&quot;</span>, mode=Mode.INFERENCE)
kernelized_model.load_state_dict(model.state_dict()) <span class="hljs-comment"># load the previous weights for equivalence testing</span>
<span class="hljs-keyword">with</span> torch.no_grad():
hub_out = kernelized_model(x)
<span class="hljs-built_in">print</span>(<span class="hljs-string">&quot;Hub kernel output matches original:&quot;</span>, torch.allclose(hub_out, ref_out, atol=<span class="hljs-number">1e-5</span>))<!----></pre></div><!----> <!--[2--><h3 class="relative group"><a id="kernel-mapping" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#kernel-mapping"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>Kernel Mapping</span></h3><!--]--><!----> <p>A kernel builder wants maximum usage of their kernels. They take time developing the kernel for one hardware, and then devotes time for other hardwares too. Who are we to stop them?</p> <p>A user can map different kernels with different device specifications all at the same time. This means if you are using a model on different hardware, all the kernels that are written for the hardware follow you without having to write a single line of extra code!</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!---->kernel_layer_mapping = {
<span class="hljs-string">&quot;Gelu&quot;</span>: {
<span class="hljs-string">&quot;cuda&quot;</span>: LayerRepository(
repo_id=<span class="hljs-string">&quot;kernels-community/activation&quot;</span>,
layer_name=<span class="hljs-string">&quot;Gelu&quot;</span>,
version=<span class="hljs-number">1</span>,
),
<span class="hljs-string">&quot;rocm&quot;</span>: LayerRepository(
repo_id=<span class="hljs-string">&quot;kernels-community/activation&quot;</span>,
layer_name=<span class="hljs-string">&quot;Gelu&quot;</span>,
version=<span class="hljs-number">1</span>,
),
}
}<!----></pre></div><!----> <p>There are kernels for specific to inference, training, and some are also torch compilable. These are called modes. Each mode can have their own kernels. Mode is handled inside <code>kernelize</code> but we can also specify it with our kernels mapping.</p> <p>This becomes more explicit and you do not need to kernelize models differently based on the mode (as you would above). The mapping takes care of which kernel to draw from the Hub and replace with.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!---->kernel_layer_mapping = {
<span class="hljs-string">&quot;Gelu&quot;</span>: {
<span class="hljs-string">&quot;cuda&quot;</span>: {
Mode.INFERENCE: LayerRepository(
repo_id=<span class="hljs-string">&quot;kernels-community/activation-inference-optimized&quot;</span>,
layer_name=<span class="hljs-string">&quot;Gelu&quot;</span>,
version=<span class="hljs-number">1</span>,
),
Mode.TRAINING
| Mode.TORCH_COMPILE: LayerRepository(
repo_id=<span class="hljs-string">&quot;kernels-community/activation-training-optimized&quot;</span>,
layer_name=<span class="hljs-string">&quot;Gelu&quot;</span>,
version=<span class="hljs-number">1</span>,
),
}
}
}<!----></pre></div><!----> <p>Another really loved feature is the CUDA Capabilites check. One can specify the CUDA compatibility and map them to the kernels that would work! These kinds of tooling makes like so much easier for the builder and the user.</p> <p>Everything is written out for you, one would just need to check the mapping and they are good to go.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!---->kernel_layer_mapping = {
<span class="hljs-string">&quot;Gelu&quot;</span>: {
Device(
<span class="hljs-built_in">type</span>=<span class="hljs-string">&quot;cuda&quot;</span>,
properties=CUDAProperties(min_capability=<span class="hljs-number">75</span>, max_capability=<span class="hljs-number">89</span>),
): LayerRepository(
repo_id=<span class="hljs-string">&quot;kernels-community/activation&quot;</span>,
layer_name=<span class="hljs-string">&quot;Gelu&quot;</span>,
version=<span class="hljs-number">1</span>,
),
Device(
<span class="hljs-built_in">type</span>=<span class="hljs-string">&quot;cuda&quot;</span>,
properties=CUDAProperties(min_capability=<span class="hljs-number">90</span>, max_capability=sys.maxsize),
): LayerRepository(
repo_id=<span class="hljs-string">&quot;kernels-community/activation-hopper&quot;</span>,
layer_name=<span class="hljs-string">&quot;Gelu&quot;</span>,
version=<span class="hljs-number">1</span>,
),
}
}<!----></pre></div><!----> <p>A side note, which we thought was really interesting. If you provide an empty mapping, it de-kernelizes your model!</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-keyword">with</span> use_kernel_mapping({}):
undo_kernelized_model = kernelize(kernelized_model, device=<span class="hljs-string">&quot;cuda&quot;</span>, mode=Mode.INFERENCE)
<span class="hljs-keyword">with</span> torch.no_grad():
undo_hub_out = undo_kernelized_model(x)<!----></pre></div><!----> <!--[1--><h2 class="relative group"><a id="are-kernels-useful" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#are-kernels-useful"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>Are kernels useful?</span></h2><!--]--><!----> <p>We know this should be at the very top of the cookbook. But knowing that you have read it through, we assume that you already know what kernels get you. In this section we just showcase a simple case where we generate some tokens with a model with and without kernels.</p> <p>Here we showcase the time required to generate 256 tokens for <code>gpt-oss-2b</code>. We benchmark the model with and without using kernels layers. We also showcase how more batch size affects the benchmark.</p> <table><thead><tr><th align="center"><img src="https://huggingface.co/datasets/ariG23498/colab-images/resolve/main/gpt-kernels-plot.png" alt="gpt-oss"/></th></tr></thead><tbody><tr><td align="center">Benchmark <code>gpt-oss-20b</code> with and without kernels</td></tr></tbody></table> <p>You can reproduce the results:</p> <ol><li><a href="https://gist.github.com/ariG23498/b2f6c78ee40d5a02119566d58d00fc56" rel="nofollow">Benchmarking script</a></li> <li>Run it with Hugging Face Jobs: <code>uvx hf jobs uv run --flavor h200 benchmark_gpt.py -d</code></li> <li>See the <a href="https://huggingface.co/jobs/ariG23498/6ac7c9f5fee2c9007016b027" rel="nofollow">Jobs live</a></li></ol> <!--[1--><h2 class="relative group"><a id="conclusion" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#conclusion"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>Conclusion</span></h2><!--]--><!----> <p>We hope this interactive cookbook gets you up to speed with how one would use kernels which are built and uploaded to the Hub. We hope to seeing more of you using kernels for your models and also your libraries.</p> <p>Kernels are really powerful and we want you to use them leaving all the tooling issues for us to tackle! Keep the hardware accelerators warm 🔥.</p> <p>Special thanks to <a href="https://huggingface.co/NoeFlandre" rel="nofollow">Noe</a> for reporting a bug (that is now fixed 😅)</p> <a class="!text-gray-400 !no-underline text-sm flex items-center not-prose mt-4" href="https://github.com/huggingface/kernels/blob/main/docs/source/use_layers.md" target="_blank"><svg class="mr-1" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M31,16l-7,7l-1.41-1.41L28.17,16l-5.58-5.59L24,9l7,7z"></path><path d="M1,16l7-7l1.41,1.41L3.83,16l5.58,5.59L8,23l-7-7z"></path><path d="M12.419,25.484L17.639,6.552l1.932,0.518L14.351,26.002z"></path></svg><!----> <span><span class="underline">Update</span> on GitHub</span></a><!----> <p></p><!--]--><!----><!--]--><!--]--><!--]--> <!--[-1--><!--]--><!--]-->
<script>
{
__sveltekit_izvw3c = {
base: "/docs/kernels/pr_898/en",
assets: "/docs/kernels/pr_898/en"
};
const element = document.currentScript.parentElement;
Promise.all([
import("/docs/kernels/pr_898/en/_app/immutable/entry/start.BxCXruW1.js"),
import("/docs/kernels/pr_898/en/_app/immutable/entry/app.CzfFppSd.js")
]).then(([kit, app]) => {
kit.start(app, element, {
node_ids: [0, 5],
data: [null,null],
form: null,
error: null
});
});
}
</script>

Xet Storage Details

Size:
70.2 kB
·
Xet hash:
4f6ba5ab36bf2d0ffac968b5c4de824e0289fd0be684a3ee41dbcb42a40ac6b3

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.