Buckets:

hf-doc-build
/

doc-dev

Files

xet

hf-doc-build/doc-dev / transformers /pr_43838 /en /kernel_doc /loading_kernels.html

HuggingFaceDocBuilder

10 days ago

download

raw

53.8 kB

	<meta charset="utf-8" /><meta name="hf:doc:metadata" content="{"title":"Loading kernels","local":"loading-kernels","sections":[{"title":"Attention kernels","local":"attention-kernels","sections":[],"depth":2},{"title":"Mode-awareness","local":"mode-awareness","sections":[],"depth":2},{"title":"KernelConfig","local":"kernelconfig","sections":[],"depth":2},{"title":"Module fusion","local":"module-fusion","sections":[],"depth":2},{"title":"Local kernels","local":"local-kernels","sections":[],"depth":2},{"title":"Disabling kernels","local":"disabling-kernels","sections":[],"depth":2},{"title":"Troubleshooting","local":"troubleshooting","sections":[{"title":"Installation issues","local":"installation-issues","sections":[],"depth":3},{"title":"Kernel loading failures","local":"kernel-loading-failures","sections":[],"depth":3},{"title":"Device compatibility","local":"device-compatibility","sections":[],"depth":3}],"depth":2},{"title":"Resources","local":"resources","sections":[],"depth":2}],"depth":1}">
	<link href="/docs/transformers/pr_43838/en/_app/immutable/assets/0.e3b0c442.css" rel="modulepreload">
	<link rel="modulepreload" href="/docs/transformers/pr_43838/en/_app/immutable/entry/start.1bcc8411.js">
	<link rel="modulepreload" href="/docs/transformers/pr_43838/en/_app/immutable/chunks/scheduler.31fdf58d.js">
	<link rel="modulepreload" href="/docs/transformers/pr_43838/en/_app/immutable/chunks/singletons.49522dd1.js">
	<link rel="modulepreload" href="/docs/transformers/pr_43838/en/_app/immutable/chunks/index.252883d5.js">
	<link rel="modulepreload" href="/docs/transformers/pr_43838/en/_app/immutable/chunks/paths.91497448.js">
	<link rel="modulepreload" href="/docs/transformers/pr_43838/en/_app/immutable/entry/app.9d434a07.js">
	<link rel="modulepreload" href="/docs/transformers/pr_43838/en/_app/immutable/chunks/preload-helper.f02a9c49.js">
	<link rel="modulepreload" href="/docs/transformers/pr_43838/en/_app/immutable/chunks/index.2f76fdf0.js">
	<link rel="modulepreload" href="/docs/transformers/pr_43838/en/_app/immutable/nodes/0.b44e4c87.js">
	<link rel="modulepreload" href="/docs/transformers/pr_43838/en/_app/immutable/chunks/each.e59479a4.js">
	<link rel="modulepreload" href="/docs/transformers/pr_43838/en/_app/immutable/nodes/75.b178052a.js">
	<link rel="modulepreload" href="/docs/transformers/pr_43838/en/_app/immutable/chunks/CopyLLMTxtMenu.f658c69c.js">
	<link rel="modulepreload" href="/docs/transformers/pr_43838/en/_app/immutable/chunks/MermaidChart.svelte_svelte_type_style_lang.0d9e7a9c.js">
	<link rel="modulepreload" href="/docs/transformers/pr_43838/en/_app/immutable/chunks/IconCopy.ac192424.js">
	<link rel="modulepreload" href="/docs/transformers/pr_43838/en/_app/immutable/chunks/CodeBlock.e52df5d6.js"><!-- HEAD_svelte-u9bgzb_START --><meta name="hf:doc:metadata" content="{"title":"Loading kernels","local":"loading-kernels","sections":[{"title":"Attention kernels","local":"attention-kernels","sections":[],"depth":2},{"title":"Mode-awareness","local":"mode-awareness","sections":[],"depth":2},{"title":"KernelConfig","local":"kernelconfig","sections":[],"depth":2},{"title":"Module fusion","local":"module-fusion","sections":[],"depth":2},{"title":"Local kernels","local":"local-kernels","sections":[],"depth":2},{"title":"Disabling kernels","local":"disabling-kernels","sections":[],"depth":2},{"title":"Troubleshooting","local":"troubleshooting","sections":[{"title":"Installation issues","local":"installation-issues","sections":[],"depth":3},{"title":"Kernel loading failures","local":"kernel-loading-failures","sections":[],"depth":3},{"title":"Device compatibility","local":"device-compatibility","sections":[],"depth":3}],"depth":2},{"title":"Resources","local":"resources","sections":[],"depth":2}],"depth":1}"><!-- HEAD_svelte-u9bgzb_END --> <p></p> <div class="items-center shrink-0 min-w-[100px] max-sm:min-w-[50px] justify-end ml-auto flex" style="float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"><div class="inline-flex rounded-md max-sm:rounded-sm"><button class="inline-flex items-center gap-1 h-7 max-sm:h-7 px-2 max-sm:px-1.5 text-sm font-medium text-gray-800 border border-r-0 rounded-l-md max-sm:rounded-l-sm border-gray-200 bg-white hover:shadow-inner dark:border-gray-850 dark:bg-gray-950 dark:text-gray-200 dark:hover:bg-gray-800" aria-live="polite"><span class="inline-flex items-center justify-center rounded-md p-0.5 max-sm:p-0 hover:text-gray-800 dark:hover:text-gray-200"><svg class="sm:size-3.5 size-3" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg></span> <span>Copy page</span></button> <button class="inline-flex items-center justify-center w-6 max-sm:w-5 h-7 max-sm:h-7 disabled:pointer-events-none text-sm text-gray-500 hover:text-gray-700 dark:hover:text-white rounded-r-md max-sm:rounded-r-sm border border-l transition border-gray-200 bg-white hover:shadow-inner dark:border-gray-850 dark:bg-gray-950 dark:text-gray-200 dark:hover:bg-gray-800" aria-haspopup="menu" aria-expanded="false" aria-label="Open copy menu"><svg class="transition-transform text-gray-400 overflow-visible sm:size-3.5 size-3 rotate-0" width="1em" height="1em" viewBox="0 0 12 7" fill="none" xmlns="http://www.w3.org/2000/svg"><path d="M1 1L6 6L11 1" stroke="currentColor"></path></svg></button></div> </div> <h1 class="relative group"><a id="loading-kernels" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#loading-kernels"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Loading kernels</span></h1> <p data-svelte-h="svelte-o3idaw">A kernel works as a drop-in replacement for standard PyTorch operations. It swaps the <code>forward</code> method with the optimized kernel implementation without breaking model code.</p> <p data-svelte-h="svelte-1h8hct6">This guide shows how to load kernels to accelerate inference.</p> <p data-svelte-h="svelte-nux6z0">Install the kernels package. We recommend the latest version which provides the best performance and bug fixes.</p> <blockquote class="note" data-svelte-h="svelte-1vjkc7l"><p>kernels >=0.11.0 is the minimum required version for working with Transformers.</p></blockquote> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-bash "><!-- HTML_TAG_START -->pip install -U kernels<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-345u6k">Set <code>use_kernels=True</code> in <a href="/docs/transformers/pr_43838/en/main_classes/model#transformers.PreTrainedModel.from_pretrained">from_pretrained()</a> to load the most performant kernels available on the Hub for your device. This replaces supported PyTorch operations with the kernel implementation.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-py "><!-- HTML_TAG_START --><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM

	model = AutoModelForCausalLM.from_pretrained(
	<span class="hljs-string">"Qwen/Qwen3-0.6B"</span>,
	use_kernels=<span class="hljs-literal">True</span>,
	device_map=<span class="hljs-string">"cuda"</span>
	)<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-3xwiaz">The default kernels differ by device type. The table below lists the Hub repository that supplies each operation’s default kernel. When no default kernel is registered, the operation falls back to standard PyTorch.</p> <table data-svelte-h="svelte-h73h7v"><thead><tr><th>Operation</th> <th>NVIDIA (CUDA)</th> <th>AMD (ROCm)</th> <th>Intel (XPU)</th></tr></thead> <tbody><tr><td>RMSNorm</td> <td><code>kernels-community/liger-kernels</code></td> <td><code>kernels-community/liger-kernels</code></td> <td><code>kernels-community/rmsnorm</code></td></tr> <tr><td>MoE MLP</td> <td><code>kernels-community/megablocks</code></td> <td><code>kernels-community/megablocks</code></td> <td><code>kernels-community/megablocks</code></td></tr> <tr><td>MLP (SwiGLU, GeGLU)</td> <td><code>kernels-community/liger-kernels</code></td> <td>—</td> <td>—</td></tr> <tr><td>Linear</td> <td><code>kernels-community/liger-kernels</code></td> <td>—</td> <td>—</td></tr> <tr><td>Activations (GELU variants, SiLU)</td> <td><code>kernels-community/activation</code></td> <td>—</td> <td>—</td></tr> <tr><td>Rotary embeddings</td> <td><code>kernels-community/rotary</code></td> <td><code>kernels-community/aiter-rope</code></td> <td><code>kernels-community/rotary</code></td></tr> <tr><td>Causal LM loss</td> <td><code>kernels-community/liger-kernels</code></td> <td>—</td> <td>—</td></tr> <tr><td>Deformable attention</td> <td><code>kernels-community/deformable-detr</code></td> <td>—</td> <td>—</td></tr></tbody></table> <blockquote class="note" data-svelte-h="svelte-aoqj5t"><p>AMD GPUs report their device type as <code>cuda</code> in PyTorch. Transformers detects ROCm at runtime and routes supported operations to the AMD kernels above, including <a href="https://github.com/ROCm/aiter" rel="nofollow">AITER</a> builds such as <code>kernels-community/aiter-rope</code>. You don’t need to set the device type yourself.</p></blockquote> <p data-svelte-h="svelte-1hzopix">Browse available kernels in the <a href="https://huggingface.co/kernels-community" rel="nofollow">kernels-community</a> organization.</p> <h2 class="relative group"><a id="attention-kernels" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#attention-kernels"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Attention kernels</span></h2> <p data-svelte-h="svelte-1hxy76v">Load attention kernels from the Hub with the <code>attn_implementation</code> argument.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-py "><!-- HTML_TAG_START --><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM

	model = AutoModelForCausalLM.from_pretrained(
	<span class="hljs-string">"Qwen/Qwen3-0.6B"</span>,
	attn_implementation=<span class="hljs-string">"kernels-community/flash-attn2"</span>,
	device_map=<span class="hljs-string">"cuda"</span>
	)<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-ofu76j">Note that for attention kernels, anything that is not part of the <code>kernels-community</code> repository (which is trusted - we may add more trusted repositories in the future) will require an additional <code>allow_all_kernels=True</code> kwarg to be used (similar to the <code>trust_remote_code=True</code> kwarg for non-HF models). This is because loading a kernel can lead to arbitrary code execution on the host machine, and we cannot verify every repo, so you need to explicitly allow it.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-py "><!-- HTML_TAG_START --><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM

	model = AutoModelForCausalLM.from_pretrained(
	<span class="hljs-string">"Qwen/Qwen3-0.6B"</span>,
	attn_implementation=<span class="hljs-string">"random-repo/random-attention"</span>,
	allow_all_kernels=<span class="hljs-literal">True</span>,
	device_map=<span class="hljs-string">"cuda"</span>
	)<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-1bwu2qn">Specific kernels, like attention, accept several formats.</p> <ul data-svelte-h="svelte-ix8ate"><li><code>@v2.1.0</code> pins to a specific tag or branch.</li> <li><code>@>=2.0,<3.0</code> sets semantic versioning constraints.</li></ul> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-py "><!-- HTML_TAG_START --><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM

	<span class="hljs-comment"># pin to a specific version</span>
	model = AutoModelForCausalLM.from_pretrained(
	<span class="hljs-string">"Qwen/Qwen3-0.6B"</span>,
	attn_implementation=<span class="hljs-string">"kernels-community/flash-attn2@v2.1.0"</span>,
	device_map=<span class="hljs-string">"cuda"</span>
	)
	<span class="hljs-comment"># use semantic versioning constraints</span>
	model = AutoModelForCausalLM.from_pretrained(
	<span class="hljs-string">"Qwen/Qwen3-0.6B"</span>,
	attn_implementation=<span class="hljs-string">"kernels-community/flash-attn2@>=2.0,<3.0"</span>,
	device_map=<span class="hljs-string">"cuda"</span>
	)<!-- HTML_TAG_END --></pre></div> <h2 class="relative group"><a id="mode-awareness" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#mode-awareness"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Mode-awareness</span></h2> <p data-svelte-h="svelte-1v7qpiw">Kernels automatically adapt to <a href="https://docs.pytorch.org/docs/stable/generated/torch.nn.Module.html#torch.nn.Module.train" rel="nofollow">training</a> and <a href="https://docs.pytorch.org/docs/stable/generated/torch.nn.Module.html#torch.nn.Module.eval" rel="nofollow">inference</a> modes based on PyTorch’s <code>model.training</code> state.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-py "><!-- HTML_TAG_START --><span class="hljs-keyword">import</span> torch
	<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM

	model = AutoModelForCausalLM.from_pretrained(
	<span class="hljs-string">"Qwen/Qwen3-0.6B"</span>,
	use_kernels=<span class="hljs-literal">True</span>,
	device_map=<span class="hljs-string">"cuda"</span>
	)

	<span class="hljs-comment"># Switch to inference mode - uses inference-optimized kernels</span>
	model.<span class="hljs-built_in">eval</span>()
	<span class="hljs-keyword">with</span> torch.no_grad():
	output = model.generate(input_ids, max_new_tokens=<span class="hljs-number">50</span>)

	<span class="hljs-comment"># Switch to training mode - uses training-optimized kernels with gradient support</span>
	model.train()
	loss = model(input_ids, labels=labels).loss
	loss.backward()<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-iz0dk6">Explicitly enable training and inference modes with the <code>mode</code> argument in the <a href="/docs/transformers/pr_43838/en/main_classes/kernels#transformers.kernelize">kernelize()</a> function. Training mode also supports an additional torch.compile mode.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-py "><!-- HTML_TAG_START --><span class="hljs-keyword">from</span> kernels <span class="hljs-keyword">import</span> Mode
	<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> kernelize

	<span class="hljs-comment"># inference optimized kernels</span>
	kernelize(model, mode=Mode.INFERENCE)

	<span class="hljs-comment"># training optimized kernels</span>
	kernelize(model, mode=Mode.TRAINING)

	<span class="hljs-comment"># training and torch-compile friendly kernels</span>
	kernelize(model, mode=Mode.TRAINING \| Mode.TORCH_COMPILE)<!-- HTML_TAG_END --></pre></div> <h2 class="relative group"><a id="kernelconfig" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#kernelconfig"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>KernelConfig</span></h2> <p data-svelte-h="svelte-1hm4hfw"><a href="/docs/transformers/pr_43838/en/main_classes/kernels#transformers.KernelConfig">KernelConfig</a> customizes which kernels are used in a model.</p> <p data-svelte-h="svelte-rxvdp2">The <code>:</code> separator names a specific kernel entry inside the repository and maps it to a layer.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-py "><!-- HTML_TAG_START --><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM, KernelConfig

	kernel_config = KernelConfig(
	kernel_mapping={
	<span class="hljs-string">"RMSNorm"</span>: <span class="hljs-string">"kernels-community/liger_kernels:LigerRMSNorm"</span>,
	}
	)
	model = AutoModelForCausalLM.from_pretrained(
	<span class="hljs-string">"Qwen/Qwen3-0.6B"</span>,
	attn_implementation=<span class="hljs-string">"kernels-community/flash-attn2:FlashAttention2"</span>,
	use_kernels=<span class="hljs-literal">True</span>,
	kernel_config=kernel_config,
	device_map=<span class="hljs-string">"cuda"</span>
	)<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-19rr87w">Specify different kernel implementations for each device type.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-py "><!-- HTML_TAG_START --><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> KernelConfig

	kernel_config = KernelConfig(
	kernel_mapping={
	<span class="hljs-string">"RMSNorm"</span>: {
	<span class="hljs-string">"cuda"</span>: <span class="hljs-string">"kernels-community/liger_kernels:LigerRMSNorm"</span>,
	<span class="hljs-string">"rocm"</span>: <span class="hljs-string">"kernels-community/rocm-kernels:RocmRMSNorm"</span>,
	<span class="hljs-string">"metal"</span>: <span class="hljs-string">"kernels-community/metal-kernels:MetalRMSNorm"</span>,
	<span class="hljs-string">"xpu"</span>: <span class="hljs-string">"kernels-community/xpu-kernels:XpuRMSNorm"</span>
	}
	}
	)<!-- HTML_TAG_END --></pre></div> <h2 class="relative group"><a id="module-fusion" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#module-fusion"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Module fusion</span></h2> <p data-svelte-h="svelte-y2bbhw">Fuse adjacent modules into a single kernel by passing a tuple of <code>(class_name, path_pattern)</code> pairs as the key in <a href="/docs/transformers/pr_43838/en/main_classes/kernels#transformers.KernelConfig">KernelConfig</a>. All patterns must share the same parent module. <code>*</code> matches any single path segment.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-python "><!-- HTML_TAG_START --><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM, KernelConfig

	kernel_config = KernelConfig(
	{
	(
	(<span class="hljs-string">"RMSNorm"</span>, <span class="hljs-string">"model.layers.*.post_attention_layernorm"</span>),
	(<span class="hljs-string">"MLP"</span>, <span class="hljs-string">"model.layers.*.mlp"</span>),
	): <span class="hljs-string">"owner/fused-rmsnorm-mlp:RMSNormMLP"</span>,
	}
	)
	model = AutoModelForCausalLM.from_pretrained(
	<span class="hljs-string">"Qwen/Qwen3-0.6B"</span>,
	use_kernels=<span class="hljs-literal">True</span>,
	kernel_config=kernel_config,
	device_map=<span class="hljs-string">"cuda"</span>,
	)<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-1cfj3m4">Fusion requires the kernel repo to provide a companion <code>KernelNameLayout</code> class alongside the <code>KernelName</code> class. See the <a href="./writing_kernels">Writing kernels</a> guide for how to implement one.</p> <h2 class="relative group"><a id="local-kernels" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#local-kernels"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Local kernels</span></h2> <p data-svelte-h="svelte-76isp2">Load kernels from local file paths with <code>use_local_kernel=True</code> in <a href="/docs/transformers/pr_43838/en/main_classes/kernels#transformers.KernelConfig">KernelConfig</a>. This loads from a local filesystem path instead of a Hub repository.</p> <p data-svelte-h="svelte-16axs8m">Local kernels use <code>/abs/path:layer_name</code> instead of the Hub format <code>org/repo:layer_name</code>.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-py "><!-- HTML_TAG_START --><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> KernelConfig, AutoModelForCausalLM

	kernel_mapping = {
	<span class="hljs-string">"RMSNorm"</span>: <span class="hljs-string">"/path/to/liger_kernels:LigerRMSNorm"</span>,
	}
	kernel_config = KernelConfig(kernel_mapping, use_local_kernel=<span class="hljs-literal">True</span>)

	model = AutoModelForCausalLM.from_pretrained(
	<span class="hljs-string">"Qwen/Qwen3-0.6B"</span>,
	dtype=<span class="hljs-string">"auto"</span>,
	device_map=<span class="hljs-string">"auto"</span>,
	use_kernels=<span class="hljs-literal">True</span>,
	kernel_config=kernel_config
	)<!-- HTML_TAG_END --></pre></div> <h2 class="relative group"><a id="disabling-kernels" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#disabling-kernels"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Disabling kernels</span></h2> <p data-svelte-h="svelte-169q2p2">Disable kernels for specific layers with an empty kernel mapping in <a href="/docs/transformers/pr_43838/en/main_classes/kernels#transformers.KernelConfig">KernelConfig</a>.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-py "><!-- HTML_TAG_START --><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM, KernelConfig

	kernel_config = KernelConfig(
	kernel_mapping={
	<span class="hljs-string">"RMSNorm"</span>: <span class="hljs-string">""</span>,
	}
	)

	model = AutoModelForCausalLM.from_pretrained(
	<span class="hljs-string">"Qwen/Qwen3-0.6B"</span>,
	use_kernels=<span class="hljs-literal">True</span>,
	kernel_config=kernel_config,
	device_map=<span class="hljs-string">"cuda"</span>
	)<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-16tid63">Set the environment variable to disable kernels globally.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-bash "><!-- HTML_TAG_START --><span class="hljs-built_in">export</span> USE_HUB_KERNELS=0 <span class="hljs-comment"># or OFF or NO</span><!-- HTML_TAG_END --></pre></div> <h2 class="relative group"><a id="troubleshooting" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#troubleshooting"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Troubleshooting</span></h2> <p data-svelte-h="svelte-h1zeb8">Kernel integration depends on hardware, drivers, and package versions working together. The following sections cover common failures.</p> <h3 class="relative group"><a id="installation-issues" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#installation-issues"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Installation issues</span></h3> <p data-svelte-h="svelte-yqc7im">Import errors indicate the kernels library isn’t installed.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-bash "><!-- HTML_TAG_START -->pip install -U kernels<!-- HTML_TAG_END --></pre></div> <h3 class="relative group"><a id="kernel-loading-failures" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#kernel-loading-failures"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Kernel loading failures</span></h3> <p data-svelte-h="svelte-1phe4a1">If specific kernels fail to load, try the following.</p> <ul data-svelte-h="svelte-4ayncu"><li>Check your hardware compatibility with the kernel requirements.</li> <li>Verify your CUDA/ROCm/Metal drivers are up to date.</li> <li>Consult the kernel repository documentation for known issues.</li></ul> <h3 class="relative group"><a id="device-compatibility" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#device-compatibility"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Device compatibility</span></h3> <p data-svelte-h="svelte-y6fn10">Not all kernels support all devices. The library falls back to standard PyTorch operations if a kernel is unavailable for your hardware. Check kernel repository documentation for device-specific support.</p> <h2 class="relative group"><a id="resources" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#resources"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Resources</span></h2> <ul data-svelte-h="svelte-sau9km"><li><a href="https://github.com/huggingface/kernels" rel="nofollow">Kernels</a> repository</li> <li><a href="https://huggingface.co/blog/hello-hf-kernels" rel="nofollow">Enhance Your Models in 5 Minutes with the Hugging Face Kernel Hub</a> blog post</li> <li>Discover kernels in the <a href="https://huggingface.co/kernels-community" rel="nofollow">kernels-community</a> org</li></ul> <a class="!text-gray-400 !no-underline text-sm flex items-center not-prose mt-4" href="https://github.com/huggingface/transformers/blob/main/docs/source/en/kernel_doc/loading_kernels.md" target="_blank"><svg class="mr-1" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M31,16l-7,7l-1.41-1.41L28.17,16l-5.58-5.59L24,9l7,7z"></path><path d="M1,16l7-7l1.41,1.41L3.83,16l5.58,5.59L8,23l-7-7z"></path><path d="M12.419,25.484L17.639,6.552l1.932,0.518L14.351,26.002z"></path></svg> <span data-svelte-h="svelte-zjs2n5"><span class="underline">Update</span> on GitHub</span></a> <p></p>

	<script>
	{
	__sveltekit_1o1ef6h = {
	assets: "/docs/transformers/pr_43838/en",
	base: "/docs/transformers/pr_43838/en",
	env: {}
	};

	const element = document.currentScript.parentElement;

	const data = [null,null];

	Promise.all([
	import("/docs/transformers/pr_43838/en/_app/immutable/entry/start.1bcc8411.js"),
	import("/docs/transformers/pr_43838/en/_app/immutable/entry/app.9d434a07.js")
	]).then(([kit, app]) => {
	kit.start(app, element, {
	node_ids: [0, 75],
	data,
	form: null,
	error: null
	});
	});
	}
	</script>

Xet Storage Details

Size:: 53.8 kB
Xet hash:: ede91e9760c7d02a31305e228ced5f2e10be3069a7efa27e9a4f1d648b9db487

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.