Buckets:
| <meta charset="utf-8" /><meta name="hf:doc:metadata" content="{"title":"Monkey patching (experimental feature)","local":"monkey-patching-experimental-feature","sections":[{"title":"Quick start","local":"quick-start","sections":[],"depth":2},{"title":"How it works","local":"how-it-works","sections":[],"depth":2},{"title":"Global registration","local":"global-registration","sections":[],"depth":2},{"title":"Pattern matching","local":"pattern-matching","sections":[],"depth":2},{"title":"Manual model construction","local":"manual-model-construction","sections":[],"depth":2},{"title":"Important notes","local":"important-notes","sections":[],"depth":2},{"title":"Troubleshooting","local":"troubleshooting","sections":[{"title":"My patch isn’t being applied","local":"my-patch-isnt-being-applied","sections":[],"depth":3},{"title":"How do I know if my patch is working?","local":"how-do-i-know-if-my-patch-is-working","sections":[],"depth":3},{"title":"Weight shape mismatch errors","local":"weight-shape-mismatch-errors","sections":[],"depth":3},{"title":"Cleaning up patches","local":"cleaning-up-patches","sections":[],"depth":3}],"depth":2},{"title":"Complete example","local":"complete-example","sections":[],"depth":2},{"title":"Recording and replaying MoE expert routing","local":"recording-and-replaying-moe-expert-routing","sections":[{"title":"Interop with vLLM","local":"interop-with-vllm","sections":[],"depth":3}],"depth":2},{"title":"API reference","local":"transformers.monkey_patching.register_patch_mapping","sections":[],"depth":2}],"depth":1}"> | |
| <link href="/docs/transformers/pr_36895/en/_app/immutable/assets/0.e3b0c442.css" rel="modulepreload"> | |
| <link rel="modulepreload" href="/docs/transformers/pr_36895/en/_app/immutable/entry/start.b88698fb.js"> | |
| <link rel="modulepreload" href="/docs/transformers/pr_36895/en/_app/immutable/chunks/scheduler.31fdf58d.js"> | |
| <link rel="modulepreload" href="/docs/transformers/pr_36895/en/_app/immutable/chunks/singletons.18caffe4.js"> | |
| <link rel="modulepreload" href="/docs/transformers/pr_36895/en/_app/immutable/chunks/index.252883d5.js"> | |
| <link rel="modulepreload" href="/docs/transformers/pr_36895/en/_app/immutable/chunks/paths.a82d371a.js"> | |
| <link rel="modulepreload" href="/docs/transformers/pr_36895/en/_app/immutable/entry/app.ac300a2c.js"> | |
| <link rel="modulepreload" href="/docs/transformers/pr_36895/en/_app/immutable/chunks/preload-helper.643b8c40.js"> | |
| <link rel="modulepreload" href="/docs/transformers/pr_36895/en/_app/immutable/chunks/index.2f76fdf0.js"> | |
| <link rel="modulepreload" href="/docs/transformers/pr_36895/en/_app/immutable/nodes/0.1669b2ea.js"> | |
| <link rel="modulepreload" href="/docs/transformers/pr_36895/en/_app/immutable/chunks/each.e59479a4.js"> | |
| <link rel="modulepreload" href="/docs/transformers/pr_36895/en/_app/immutable/nodes/585.a7bf1327.js"> | |
| <link rel="modulepreload" href="/docs/transformers/pr_36895/en/_app/immutable/chunks/CopyLLMTxtMenu.9cbc081e.js"> | |
| <link rel="modulepreload" href="/docs/transformers/pr_36895/en/_app/immutable/chunks/MermaidChart.svelte_svelte_type_style_lang.945f6700.js"> | |
| <link rel="modulepreload" href="/docs/transformers/pr_36895/en/_app/immutable/chunks/IconCopy.ac192424.js"> | |
| <link rel="modulepreload" href="/docs/transformers/pr_36895/en/_app/immutable/chunks/Docstring.7c34ae2a.js"> | |
| <link rel="modulepreload" href="/docs/transformers/pr_36895/en/_app/immutable/chunks/CodeBlock.e52df5d6.js"> | |
| <link rel="modulepreload" href="/docs/transformers/pr_36895/en/_app/immutable/chunks/ExampleCodeBlock.de295a13.js"><!-- HEAD_svelte-u9bgzb_START --><meta name="hf:doc:metadata" content="{"title":"Monkey patching (experimental feature)","local":"monkey-patching-experimental-feature","sections":[{"title":"Quick start","local":"quick-start","sections":[],"depth":2},{"title":"How it works","local":"how-it-works","sections":[],"depth":2},{"title":"Global registration","local":"global-registration","sections":[],"depth":2},{"title":"Pattern matching","local":"pattern-matching","sections":[],"depth":2},{"title":"Manual model construction","local":"manual-model-construction","sections":[],"depth":2},{"title":"Important notes","local":"important-notes","sections":[],"depth":2},{"title":"Troubleshooting","local":"troubleshooting","sections":[{"title":"My patch isn’t being applied","local":"my-patch-isnt-being-applied","sections":[],"depth":3},{"title":"How do I know if my patch is working?","local":"how-do-i-know-if-my-patch-is-working","sections":[],"depth":3},{"title":"Weight shape mismatch errors","local":"weight-shape-mismatch-errors","sections":[],"depth":3},{"title":"Cleaning up patches","local":"cleaning-up-patches","sections":[],"depth":3}],"depth":2},{"title":"Complete example","local":"complete-example","sections":[],"depth":2},{"title":"Recording and replaying MoE expert routing","local":"recording-and-replaying-moe-expert-routing","sections":[{"title":"Interop with vLLM","local":"interop-with-vllm","sections":[],"depth":3}],"depth":2},{"title":"API reference","local":"transformers.monkey_patching.register_patch_mapping","sections":[],"depth":2}],"depth":1}"><!-- HEAD_svelte-u9bgzb_END --> <p></p> <div class="items-center shrink-0 min-w-[100px] max-sm:min-w-[50px] justify-end ml-auto flex" style="float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"><div class="inline-flex rounded-md max-sm:rounded-sm"><button class="inline-flex items-center gap-1 h-7 max-sm:h-7 px-2 max-sm:px-1.5 text-sm font-medium text-gray-800 border border-r-0 rounded-l-md max-sm:rounded-l-sm border-gray-200 bg-white hover:shadow-inner dark:border-gray-850 dark:bg-gray-950 dark:text-gray-200 dark:hover:bg-gray-800" aria-live="polite"><span class="inline-flex items-center justify-center rounded-md p-0.5 max-sm:p-0 hover:text-gray-800 dark:hover:text-gray-200"><svg class="sm:size-3.5 size-3" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg></span> <span>Copy page</span></button> <button class="inline-flex items-center justify-center w-6 max-sm:w-5 h-7 max-sm:h-7 disabled:pointer-events-none text-sm text-gray-500 hover:text-gray-700 dark:hover:text-white rounded-r-md max-sm:rounded-r-sm border border-l transition border-gray-200 bg-white hover:shadow-inner dark:border-gray-850 dark:bg-gray-950 dark:text-gray-200 dark:hover:bg-gray-800" aria-haspopup="menu" aria-expanded="false" aria-label="Open copy menu"><svg class="transition-transform text-gray-400 overflow-visible sm:size-3.5 size-3 rotate-0" width="1em" height="1em" viewBox="0 0 12 7" fill="none" xmlns="http://www.w3.org/2000/svg"><path d="M1 1L6 6L11 1" stroke="currentColor"></path></svg></button></div> </div> <h1 class="relative group"><a id="monkey-patching-experimental-feature" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#monkey-patching-experimental-feature"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Monkey patching (experimental feature)</span></h1> <p data-svelte-h="svelte-1b6hxdz">Monkey patching allows you to replace model components globally without modifying the original model code. Once registered, patches are automatically applied when loading any model with <a href="/docs/transformers/pr_36895/en/main_classes/model#transformers.PreTrainedModel.from_pretrained">from_pretrained()</a> or <code>~PreTrainedModel.from_config</code>. This enables you to restructure models for specific requirements like quantization compatibility, apply optimizations, or experiment with architectural variants.</p> <blockquote class="warning" data-svelte-h="svelte-uqiu3"><p><strong>Monkey patching should be used as a last resort</strong> when you need to change the layout and structure of a module and or its weights. For many customization and optimization needs, try using the <a href="./attention_interface">Attention interface</a>, <a href="./experts_interface">Experts interface</a>, or <a href="./kernel_doc/overview">Kernels registry</a> instead. Only use monkey patching when you need structural changes that can’t be achieved through custom forward implementations alone (e.g., for quantization library compatibility, fusing layers, or architectural experiments).</p></blockquote> <h2 class="relative group"><a id="quick-start" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#quick-start"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Quick start</span></h2> <p data-svelte-h="svelte-ovrxb0">Here’s a simple example showing how to replace a model component:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-python "><!-- HTML_TAG_START --><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM | |
| <span class="hljs-keyword">from</span> transformers.models.llama.modeling_llama <span class="hljs-keyword">import</span> LlamaAttention | |
| <span class="hljs-keyword">from</span> transformers.monkey_patching <span class="hljs-keyword">import</span> register_patch_mapping | |
| <span class="hljs-comment"># Define your replacement class (must inherit from nn.Module)</span> | |
| <span class="hljs-keyword">class</span> <span class="hljs-title class_">CustomLlamaAttention</span>(<span class="hljs-title class_ inherited__">LlamaAttention</span>): | |
| <span class="hljs-keyword">def</span> <span class="hljs-title function_">forward</span>(<span class="hljs-params">self, *args, **kwargs</span>): | |
| <span class="hljs-comment"># Your custom implementation</span> | |
| <span class="hljs-built_in">print</span>(<span class="hljs-string">"Using custom attention!"</span>) | |
| <span class="hljs-keyword">return</span> <span class="hljs-built_in">super</span>().forward(*args, **kwargs) | |
| <span class="hljs-comment"># Register the patch globally (only applies to transformers modeling modules)</span> | |
| register_patch_mapping(mapping={<span class="hljs-string">"LlamaAttention"</span>: CustomLlamaAttention}) | |
| <span class="hljs-comment"># Load a model - the patch is automatically applied during initialization</span> | |
| model = AutoModelForCausalLM.from_pretrained(<span class="hljs-string">"meta-llama/Llama-3.2-1B"</span>) | |
| <span class="hljs-comment"># All LlamaAttention layers in the model are now CustomLlamaAttention instances</span> | |
| <span class="hljs-built_in">print</span>(<span class="hljs-built_in">type</span>(model.model.layers[<span class="hljs-number">0</span>].self_attn)) <span class="hljs-comment"># <class '__main__.CustomLlamaAttention'></span><!-- HTML_TAG_END --></pre></div> <h2 class="relative group"><a id="how-it-works" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#how-it-works"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>How it works</span></h2> <p data-svelte-h="svelte-12wb3oi">Monkey patches work through a two-step process:</p> <ol data-svelte-h="svelte-15r6z4l"><li><p><strong>Registration</strong>: Call <code>register_patch_mapping</code> to add mappings to a global registry.</p></li> <li><p><strong>Application</strong>: Patches are automatically applied during model initialization:</p> <ul><li><strong><code>from_pretrained</code> / <code>from_config</code></strong>: Patches are <strong>automatically</strong> applied via an internal context manager. No additional action needed!</li> <li><strong>Manual construction</strong> (e.g., <code>Model(config)</code>): You must use the <code>apply_patches</code> context manager manually.</li></ul></li></ol> <p data-svelte-h="svelte-3emylx">Once patches are registered, they persist and affect all subsequent model loads until you clear them with <code>clear_patch_mapping</code>.</p> <p data-svelte-h="svelte-i6g358"><strong>Important limitations</strong>:</p> <ul data-svelte-h="svelte-642p3t"><li>Only classes in <code>transformers</code> modeling modules are allowed to be patched (e.g., <code>LlamaAttention</code>, <code>LlamaMLP</code>).</li> <li>The mapping keys can be either exact class names or regular expression patterns (see <a href="#pattern-matching">Pattern matching</a> below).</li></ul> <h2 class="relative group"><a id="global-registration" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#global-registration"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Global registration</span></h2> <p data-svelte-h="svelte-15vuik4">Use <code>register_patch_mapping</code> to register mappings globally:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-python "><!-- HTML_TAG_START --><span class="hljs-keyword">from</span> transformers.monkey_patching <span class="hljs-keyword">import</span> register_patch_mapping | |
| <span class="hljs-comment"># Register a single patch</span> | |
| register_patch_mapping( | |
| mapping={<span class="hljs-string">"Qwen2MoeExperts"</span>: SequentialExperts} | |
| ) | |
| <span class="hljs-comment"># Register multiple patches at once</span> | |
| register_patch_mapping( | |
| mapping={ | |
| <span class="hljs-string">"Qwen2MoeExperts"</span>: SequentialExperts, | |
| <span class="hljs-string">"Qwen2MoeAttention"</span>: CustomAttention, | |
| }, | |
| <span class="hljs-comment"># Overwrite existing patches if they exist</span> | |
| overwrite=<span class="hljs-literal">True</span>, | |
| )<!-- HTML_TAG_END --></pre></div> <h2 class="relative group"><a id="pattern-matching" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#pattern-matching"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Pattern matching</span></h2> <p data-svelte-h="svelte-p0t3s">You can use regular expressions to match multiple classes with a single pattern:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-python "><!-- HTML_TAG_START --><span class="hljs-keyword">from</span> transformers.monkey_patching <span class="hljs-keyword">import</span> register_patch_mapping | |
| <span class="hljs-comment"># Match all classes containing "Attention"</span> | |
| register_patch_mapping( | |
| mapping={<span class="hljs-string">".*Attention"</span>: CustomAttention} | |
| ) | |
| <span class="hljs-comment"># More examples</span> | |
| register_patch_mapping( | |
| mapping={ | |
| <span class="hljs-string">".*MoeExperts$"</span>: CustomExperts, <span class="hljs-comment"># Ends with "MoeExperts"</span> | |
| <span class="hljs-string">"^Llama\\d+Attention$"</span>: CustomAttention, <span class="hljs-comment"># Llama2Attention, Llama3Attention, etc.</span> | |
| } | |
| )<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-1utt40n"><strong>Important</strong>: Exact matches take precedence over patterns. If you register both <code>"LlamaAttention"</code> and <code>".*Attention"</code>, classes named <code>LlamaAttention</code> will use the exact-match replacement, while other matching classes will use the pattern-match replacement.</p> <blockquote class="warning" data-svelte-h="svelte-u6n8qs"><p><strong>Regex patterns can silently break models.</strong> A broad pattern like <code>".*Attention"</code> will match <em>every</em> class whose name contains “Attention” — including container classes that wrap the attention you actually want to replace. For example, BERT has three attention-related classes: <code>BertSelfAttention</code> and <code>BertCrossAttention</code> (the inner attention implementations) and <code>BertAttention</code> (an outer module that <em>contains</em> one of those inner classes). Patching all three with the same custom attention layer produces a broken model because the outer <code>BertAttention</code> no longer wraps the inner one — it <em>is</em> one, eliminating expected sub-modules like <code>self</code> and <code>output</code>. Prefer narrow patterns (e.g., <code>".*SelfAttention$"</code>) or exact class names to avoid unintended matches.</p></blockquote> <p data-svelte-h="svelte-8a8nth">To unregister patches, use <code>unregister_patch_mapping</code>:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-python "><!-- HTML_TAG_START --><span class="hljs-keyword">from</span> transformers.monkey_patching <span class="hljs-keyword">import</span> unregister_patch_mapping | |
| <span class="hljs-comment"># Unregister a single patch (use exact name or pattern from registration)</span> | |
| unregister_patch_mapping(keys=[<span class="hljs-string">"Qwen2MoeExperts"</span>]) | |
| <span class="hljs-comment"># Unregister multiple patches at once</span> | |
| unregister_patch_mapping(keys=[<span class="hljs-string">"Qwen2MoeExperts"</span>, <span class="hljs-string">"Qwen2MoeAttention"</span>]) | |
| <span class="hljs-comment"># Unregister a pattern</span> | |
| unregister_patch_mapping(keys=[<span class="hljs-string">".*Attention"</span>])<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-1xpc0g">To clear all registered patches, use <code>clear_patch_mapping</code>:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-python "><!-- HTML_TAG_START --><span class="hljs-keyword">from</span> transformers.monkey_patching <span class="hljs-keyword">import</span> clear_patch_mapping | |
| clear_patch_mapping()<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-tsmdzm">To view currently registered patches, use <code>get_patch_mapping</code>:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-python "><!-- HTML_TAG_START --><span class="hljs-keyword">from</span> transformers.monkey_patching <span class="hljs-keyword">import</span> get_patch_mapping | |
| current_patches = get_patch_mapping() | |
| <span class="hljs-built_in">print</span>(current_patches)<!-- HTML_TAG_END --></pre></div> <h2 class="relative group"><a id="manual-model-construction" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#manual-model-construction"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Manual model construction</span></h2> <p data-svelte-h="svelte-1x6g5bx">The <code>apply_patches</code> context manager is only needed when you’re constructing models <strong>manually</strong> (e.g., <code>Model(config)</code>) without using <code>from_pretrained</code> or <code>from_config</code>:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-python "><!-- HTML_TAG_START --><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> LlamaModel, LlamaConfig | |
| <span class="hljs-keyword">from</span> transformers.monkey_patching <span class="hljs-keyword">import</span> register_patch_mapping, apply_patches | |
| <span class="hljs-comment"># Register patch globally</span> | |
| register_patch_mapping(mapping={<span class="hljs-string">"LlamaAttention"</span>: CustomAttention}) | |
| <span class="hljs-comment"># For manual construction, you need the context manager</span> | |
| <span class="hljs-keyword">with</span> apply_patches(): | |
| model = LlamaModel(LlamaConfig()) <span class="hljs-comment"># Uses CustomAttention</span> | |
| <span class="hljs-comment"># Without the context manager, manual construction uses original classes</span> | |
| model = LlamaModel(LlamaConfig()) <span class="hljs-comment"># Uses LlamaAttention</span> | |
| <span class="hljs-comment"># But from_pretrained and from_config will always apply registered patches</span> | |
| model = LlamaModel.from_pretrained(<span class="hljs-string">"meta-llama/Llama-3.2-1B"</span>) <span class="hljs-comment"># Uses CustomAttention</span><!-- HTML_TAG_END --></pre></div> <h2 class="relative group"><a id="important-notes" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#important-notes"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Important notes</span></h2> <ul data-svelte-h="svelte-u08e0h"><li><p><strong>Weight handling</strong>: Monkey patching only replaces classes, not weights. If your patched class has a different weights layout, you’ll need to handle <a href="./weightconverter">weight conversions</a> separately to ensure compatibility with pretrained weights. See the <a href="#complete-example">Complete example</a> below for how to combine monkey patches with weight conversion mappings.</p></li> <li><p><strong>Global effect</strong>: Patches registered with <code>register_patch_mapping</code> are applied globally to all models loaded after registration. Always use <code>clear_patch_mapping</code> to clean up when done, especially in tests, notebooks, or long-running applications.</p></li> <li><p><strong>Class validation</strong>: The API automatically validates that replacement classes are <code>nn.Module</code> subclasses. If you pass an invalid class, you’ll get a clear error message.</p></li> <li><p><strong>Thread safety</strong>: All patching operations are thread-safe. You can safely register, unregister, and apply patches from multiple threads.</p></li> <li><p><strong>Matching behavior</strong>: When using exact class names, they must exactly match the original class names as they appear in the model’s source code (case-sensitive). When using regex patterns, they are matched against class names using <code>re.search()</code>.</p></li></ul> <h2 class="relative group"><a id="troubleshooting" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#troubleshooting"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Troubleshooting</span></h2> <h3 class="relative group"><a id="my-patch-isnt-being-applied" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#my-patch-isnt-being-applied"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>My patch isn’t being applied</span></h3> <p data-svelte-h="svelte-2x33oc"><strong>Check class name or pattern</strong>: Ensure the class name or pattern in your mapping is correct:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-python "><!-- HTML_TAG_START --><span class="hljs-comment"># For exact names - must match exactly (case-sensitive)</span> | |
| register_patch_mapping(mapping={<span class="hljs-string">"LlamaAttention"</span>: CustomAttention}) | |
| <span class="hljs-comment"># For patterns - use valid regex</span> | |
| register_patch_mapping(mapping={<span class="hljs-string">".*Attention"</span>: CustomAttention})<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-ayjbin"><strong>Verify registration</strong>: Use <code>get_patch_mapping</code> to confirm your mapping is registered:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-python "><!-- HTML_TAG_START --><span class="hljs-built_in">print</span>(get_patch_mapping()) | |
| <span class="hljs-comment"># Shows all registered mappings: {'LlamaAttention': <class 'CustomAttention'>, '.*MLP': <class 'CustomMLP'>}</span><!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-1h7b3cv"><strong>Check model source</strong>: Find the exact class name in the model’s source:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-python "><!-- HTML_TAG_START --><span class="hljs-keyword">from</span> transformers.models.llama <span class="hljs-keyword">import</span> modeling_llama | |
| <span class="hljs-built_in">print</span>(<span class="hljs-built_in">dir</span>(modeling_llama)) <span class="hljs-comment"># Look for the class name</span><!-- HTML_TAG_END --></pre></div> <h3 class="relative group"><a id="how-do-i-know-if-my-patch-is-working" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#how-do-i-know-if-my-patch-is-working"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>How do I know if my patch is working?</span></h3> <p data-svelte-h="svelte-khc52g">Inspect the loaded model to verify the patch:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-python "><!-- HTML_TAG_START -->model = AutoModelForCausalLM.from_pretrained(<span class="hljs-string">"meta-llama/Llama-3.2-1B"</span>) | |
| <span class="hljs-comment"># Check the type of a specific module</span> | |
| <span class="hljs-built_in">print</span>(<span class="hljs-built_in">type</span>(model.model.layers[<span class="hljs-number">0</span>].self_attn)) <span class="hljs-comment"># Should show your custom class</span> | |
| <span class="hljs-comment"># Or iterate through all modules</span> | |
| <span class="hljs-keyword">for</span> name, module <span class="hljs-keyword">in</span> model.named_modules(): | |
| <span class="hljs-keyword">if</span> <span class="hljs-string">'attention'</span> <span class="hljs-keyword">in</span> name.lower(): | |
| <span class="hljs-built_in">print</span>(<span class="hljs-string">f"<span class="hljs-subst">{name}</span>: <span class="hljs-subst">{<span class="hljs-built_in">type</span>(module)}</span>"</span>)<!-- HTML_TAG_END --></pre></div> <h3 class="relative group"><a id="weight-shape-mismatch-errors" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#weight-shape-mismatch-errors"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Weight shape mismatch errors</span></h3> <p data-svelte-h="svelte-lvz71y">If your patched class has different weight shapes, register a weight conversion:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-python "><!-- HTML_TAG_START --><span class="hljs-keyword">from</span> transformers.conversion_mapping <span class="hljs-keyword">import</span> register_checkpoint_conversion_mapping, WeightConverter | |
| <span class="hljs-keyword">from</span> transformers.monkey_patching <span class="hljs-keyword">import</span> register_patch_mapping | |
| register_patch_mapping( | |
| mapping={ | |
| <span class="hljs-string">"LlamaAttention"</span>: LlamaFusedAttention, | |
| } | |
| ) | |
| register_checkpoint_conversion_mapping( | |
| model_type=<span class="hljs-string">"llama"</span>, | |
| mapping=[ | |
| WeightConverter( | |
| source_patterns=[<span class="hljs-string">"q_proj"</span>, <span class="hljs-string">"k_proj"</span>, <span class="hljs-string">"v_proj"</span>], | |
| target_patterns=[<span class="hljs-string">"qkv_proj"</span>], | |
| operations=[ | |
| Concatenate(dim=<span class="hljs-number">0</span>), | |
| ], | |
| ) | |
| ], | |
| overwrite=<span class="hljs-literal">True</span>, | |
| )<!-- HTML_TAG_END --></pre></div> <h3 class="relative group"><a id="cleaning-up-patches" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#cleaning-up-patches"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Cleaning up patches</span></h3> <p data-svelte-h="svelte-n1s7sk">Always clean up patches when you’re done to avoid affecting other code:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-python "><!-- HTML_TAG_START --><span class="hljs-keyword">from</span> transformers.monkey_patching <span class="hljs-keyword">import</span> register_patch_mapping, clear_patch_mapping | |
| <span class="hljs-keyword">try</span>: | |
| register_patch_mapping(mapping={<span class="hljs-string">"LlamaAttention"</span>: CustomAttention}) | |
| model = AutoModelForCausalLM.from_pretrained(<span class="hljs-string">"meta-llama/Llama-2-7b-chat-hf"</span>) | |
| <span class="hljs-comment"># ... use model ...</span> | |
| <span class="hljs-keyword">finally</span>: | |
| clear_patch_mapping() <span class="hljs-comment"># Always clean up</span><!-- HTML_TAG_END --></pre></div> <h2 class="relative group"><a id="complete-example" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#complete-example"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Complete example</span></h2> <p data-svelte-h="svelte-iaarhj">Here’s a comprehensive example showing how to restructure both the experts and attention modules in a Mixture-of-Experts model (<code>qwen2_moe</code>) for optimization and quantization compatibility. This demonstrates:</p> <ol data-svelte-h="svelte-14ajfxn"><li>Creating custom replacement classes that maintain the same interface</li> <li>Registering monkey patches for multiple components</li> <li>Handling weight conversions for the new structure</li></ol> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-python "><!-- HTML_TAG_START --><span class="hljs-keyword">from</span> typing <span class="hljs-keyword">import</span> Unpack | |
| <span class="hljs-keyword">import</span> torch | |
| <span class="hljs-keyword">import</span> torch.nn <span class="hljs-keyword">as</span> nn | |
| <span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM, Concatenate, WeightConverter | |
| <span class="hljs-keyword">from</span> transformers.activations <span class="hljs-keyword">import</span> ACT2FN | |
| <span class="hljs-keyword">from</span> transformers.cache_utils <span class="hljs-keyword">import</span> Cache | |
| <span class="hljs-keyword">from</span> transformers.conversion_mapping <span class="hljs-keyword">import</span> register_checkpoint_conversion_mapping | |
| <span class="hljs-keyword">from</span> transformers.integrations.sdpa_attention <span class="hljs-keyword">import</span> sdpa_attention_forward | |
| <span class="hljs-keyword">from</span> transformers.models.qwen2_moe.modeling_qwen2_moe <span class="hljs-keyword">import</span> apply_rotary_pos_emb | |
| <span class="hljs-keyword">from</span> transformers.monkey_patching <span class="hljs-keyword">import</span> register_patch_mapping | |
| <span class="hljs-keyword">from</span> transformers.utils.generic <span class="hljs-keyword">import</span> TransformersKwargs | |
| <span class="hljs-keyword">class</span> <span class="hljs-title class_">MoeMLP</span>(nn.Module): | |
| <span class="hljs-keyword">def</span> <span class="hljs-title function_">__init__</span>(<span class="hljs-params">self, config</span>): | |
| <span class="hljs-built_in">super</span>().__init__() | |
| self.config = config | |
| self.num_experts = config.num_experts | |
| self.hidden_size = config.hidden_size | |
| self.intermediate_size = config.moe_intermediate_size | |
| self.gate_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=<span class="hljs-literal">False</span>) | |
| self.up_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=<span class="hljs-literal">False</span>) | |
| self.down_proj = nn.Linear(self.intermediate_size, self.hidden_size, bias=<span class="hljs-literal">False</span>) | |
| self.act_fn = ACT2FN[config.hidden_act] | |
| <span class="hljs-keyword">def</span> <span class="hljs-title function_">forward</span>(<span class="hljs-params">self, x</span>): | |
| down_proj = self.down_proj(self.act_fn(self.gate_proj(x)) * self.up_proj(x)) | |
| <span class="hljs-keyword">return</span> down_proj | |
| <span class="hljs-comment"># Adapted from the original Qwen2MoeExperts</span> | |
| <span class="hljs-keyword">class</span> <span class="hljs-title class_">ModuleListExperts</span>(nn.ModuleList): | |
| <span class="hljs-keyword">def</span> <span class="hljs-title function_">__init__</span>(<span class="hljs-params">self, config</span>): | |
| <span class="hljs-built_in">super</span>().__init__() | |
| self.config = config | |
| self.num_experts = config.num_experts | |
| <span class="hljs-keyword">for</span> _ <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(self.num_experts): | |
| self.append(MoeMLP(config)) | |
| <span class="hljs-keyword">def</span> <span class="hljs-title function_">forward</span>(<span class="hljs-params"> | |
| self, hidden_states: torch.Tensor, top_k_index: torch.Tensor, top_k_weights: torch.Tensor | |
| </span>) -> torch.Tensor: | |
| final_hidden_states = torch.zeros_like(hidden_states) | |
| <span class="hljs-keyword">with</span> torch.no_grad(): | |
| expert_mask = torch.nn.functional.one_hot(top_k_index, num_classes=self.num_experts) | |
| expert_mask = expert_mask.permute(<span class="hljs-number">2</span>, <span class="hljs-number">1</span>, <span class="hljs-number">0</span>) | |
| <span class="hljs-keyword">for</span> expert_idx <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(self.num_experts): | |
| top_k_pos, token_idx = torch.where(expert_mask[expert_idx]) | |
| current_state = hidden_states[token_idx] | |
| current_hidden_states = self[expert_idx](current_state) | |
| current_hidden_states = current_hidden_states * top_k_weights[token_idx, top_k_pos, <span class="hljs-literal">None</span>] | |
| final_hidden_states.index_add_(<span class="hljs-number">0</span>, token_idx, current_hidden_states.to(final_hidden_states.dtype)) | |
| <span class="hljs-keyword">return</span> final_hidden_states | |
| <span class="hljs-comment"># Adapted from the original Qwen2MoeAttention</span> | |
| <span class="hljs-keyword">class</span> <span class="hljs-title class_">FusedQKVAttention</span>(nn.Module): | |
| <span class="hljs-keyword">def</span> <span class="hljs-title function_">__init__</span>(<span class="hljs-params">self, config, layer_idx: <span class="hljs-built_in">int</span></span>): | |
| <span class="hljs-built_in">super</span>().__init__() | |
| self.config = config | |
| self.layer_idx = layer_idx | |
| self.head_dim = <span class="hljs-built_in">getattr</span>(config, <span class="hljs-string">"head_dim"</span>, config.hidden_size // config.num_attention_heads) | |
| self.num_key_value_groups = config.num_attention_heads // config.num_key_value_heads | |
| self.scaling = self.head_dim**-<span class="hljs-number">0.5</span> | |
| self.attention_dropout = config.attention_dropout | |
| self.is_causal = <span class="hljs-literal">True</span> | |
| self.qkv_proj = nn.Linear(config.hidden_size, <span class="hljs-number">3</span> * config.num_attention_heads * self.head_dim, bias=<span class="hljs-literal">True</span>) | |
| self.o_proj = nn.Linear(config.num_attention_heads * self.head_dim, config.hidden_size, bias=<span class="hljs-literal">False</span>) | |
| <span class="hljs-keyword">if</span> self.config.layer_types[layer_idx] == <span class="hljs-string">"sliding_attention"</span>: | |
| self.sliding_window = config.sliding_window | |
| <span class="hljs-keyword">def</span> <span class="hljs-title function_">forward</span>(<span class="hljs-params"> | |
| self, | |
| hidden_states: torch.Tensor, | |
| position_embeddings: <span class="hljs-built_in">tuple</span>[torch.Tensor, torch.Tensor] | <span class="hljs-literal">None</span> = <span class="hljs-literal">None</span>, | |
| attention_mask: torch.Tensor | <span class="hljs-literal">None</span> = <span class="hljs-literal">None</span>, | |
| past_key_values: Cache | <span class="hljs-literal">None</span> = <span class="hljs-literal">None</span>, | |
| **kwargs: Unpack[TransformersKwargs], | |
| </span>) -> <span class="hljs-built_in">tuple</span>[torch.Tensor, torch.Tensor]: | |
| input_shape = hidden_states.shape[:-<span class="hljs-number">1</span>] | |
| hidden_shape = (*input_shape, -<span class="hljs-number">1</span>, self.head_dim) | |
| query_states, key_states, value_states = self.qkv_proj(hidden_states).chunk(<span class="hljs-number">3</span>, dim=-<span class="hljs-number">1</span>) | |
| query_states = query_states.view(hidden_shape).transpose(<span class="hljs-number">1</span>, <span class="hljs-number">2</span>) | |
| key_states = key_states.view(hidden_shape).transpose(<span class="hljs-number">1</span>, <span class="hljs-number">2</span>) | |
| value_states = value_states.view(hidden_shape).transpose(<span class="hljs-number">1</span>, <span class="hljs-number">2</span>) | |
| cos, sin = position_embeddings | |
| query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin) | |
| <span class="hljs-keyword">if</span> past_key_values <span class="hljs-keyword">is</span> <span class="hljs-keyword">not</span> <span class="hljs-literal">None</span>: | |
| key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx) | |
| attn_output, attn_weights = sdpa_attention_forward( | |
| self, | |
| query_states, | |
| key_states, | |
| value_states, | |
| attention_mask, | |
| dropout=<span class="hljs-number">0.0</span> <span class="hljs-keyword">if</span> <span class="hljs-keyword">not</span> self.training <span class="hljs-keyword">else</span> self.attention_dropout, | |
| scaling=self.scaling, | |
| **kwargs, | |
| ) | |
| attn_output = attn_output.reshape(*input_shape, -<span class="hljs-number">1</span>).contiguous() | |
| attn_output = self.o_proj(attn_output) | |
| <span class="hljs-keyword">return</span> attn_output, attn_weights | |
| <span class="hljs-comment"># Registering monkey patches for the new attention and experts modules.</span> | |
| register_patch_mapping( | |
| mapping={ | |
| <span class="hljs-string">"Qwen2MoeExperts"</span>: ModuleListExperts, | |
| <span class="hljs-string">"Qwen2MoeAttention"</span>: FusedQKVAttention, | |
| } | |
| ) | |
| <span class="hljs-comment"># Registering weight conversion mappings adapted for the new modules. This registration will:</span> | |
| <span class="hljs-comment"># - Override the original conversion mapping for qwen2_moe which concatenated the experts into a single parameter format.</span> | |
| <span class="hljs-comment"># - Concatenate the q_proj, k_proj, v_proj weights/biases into a single qkv_proj weight/bias for the new fused attention module.</span> | |
| register_checkpoint_conversion_mapping( | |
| model_type=<span class="hljs-string">"qwen2_moe"</span>, | |
| mapping=[ | |
| WeightConverter( | |
| source_patterns=[<span class="hljs-string">"q_proj"</span>, <span class="hljs-string">"k_proj"</span>, <span class="hljs-string">"v_proj"</span>], | |
| target_patterns=[<span class="hljs-string">"qkv_proj"</span>], | |
| operations=[Concatenate(dim=<span class="hljs-number">0</span>)], | |
| ), | |
| ], | |
| overwrite=<span class="hljs-literal">True</span>, | |
| ) | |
| model = AutoModelForCausalLM.from_pretrained(<span class="hljs-string">"Qwen/Qwen1.5-MoE-A2.7B"</span>)<!-- HTML_TAG_END --></pre></div> <h2 class="relative group"><a id="recording-and-replaying-moe-expert-routing" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#recording-and-replaying-moe-expert-routing"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Recording and replaying MoE expert routing</span></h2> <p data-svelte-h="svelte-1bo7wkt">Mixture-of-Experts training workflows like RLHF need to record which experts each token was routed to during generation, then replay that exact routing in a separate training forward pass. You can build this end-to-end with the existing monkey patching and output capturing machinery — no modeling-file changes required.</p> <p data-svelte-h="svelte-qq5zcy">The pattern has three pieces:</p> <ol data-svelte-h="svelte-15zyj04"><li>A <strong>replayable router subclass</strong> that can optionally read forced expert indices from an instance attribute.</li> <li>A <strong>context manager</strong> that sets those attributes across every router before a forward pass and clears them afterwards.</li> <li>An entry in the model’s output-capture registry so <code>output_<name>=True</code> exposes the indices through the standard <code>@capture_outputs</code> path.</li></ol> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-python "><!-- HTML_TAG_START --><span class="hljs-keyword">from</span> contextlib <span class="hljs-keyword">import</span> contextmanager | |
| <span class="hljs-keyword">import</span> torch | |
| <span class="hljs-keyword">import</span> torch.nn.functional <span class="hljs-keyword">as</span> F | |
| <span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> Qwen3MoeConfig, Qwen3MoeForCausalLM | |
| <span class="hljs-keyword">from</span> transformers.models.qwen3_moe.modeling_qwen3_moe <span class="hljs-keyword">import</span> Qwen3MoeTopKRouter | |
| <span class="hljs-keyword">from</span> transformers.monkey_patching <span class="hljs-keyword">import</span> apply_patches, register_patch_mapping | |
| <span class="hljs-keyword">from</span> transformers.utils.output_capturing <span class="hljs-keyword">import</span> _CAN_RECORD_REGISTRY, OutputRecorder | |
| <span class="hljs-keyword">class</span> <span class="hljs-title class_">ReplayableQwen3MoeTopKRouter</span>(<span class="hljs-title class_ inherited__">Qwen3MoeTopKRouter</span>): | |
| _forced_indices: torch.Tensor | <span class="hljs-literal">None</span> = <span class="hljs-literal">None</span> | |
| <span class="hljs-keyword">def</span> <span class="hljs-title function_">forward</span>(<span class="hljs-params">self, hidden_states</span>): | |
| hidden_states = hidden_states.reshape(-<span class="hljs-number">1</span>, self.hidden_dim) | |
| router_logits = F.linear(hidden_states, self.weight) | |
| router_logits = F.softmax(router_logits, dtype=torch.<span class="hljs-built_in">float</span>, dim=-<span class="hljs-number">1</span>) | |
| <span class="hljs-keyword">if</span> self._forced_indices <span class="hljs-keyword">is</span> <span class="hljs-keyword">not</span> <span class="hljs-literal">None</span>: | |
| router_indices = self._forced_indices.to(router_logits.device).long() | |
| <span class="hljs-comment"># Megatron-style replay: preserve expert path, recompute current scores</span> | |
| router_top_value = router_logits.gather(-<span class="hljs-number">1</span>, router_indices) | |
| <span class="hljs-keyword">else</span>: | |
| router_top_value, router_indices = torch.topk(router_logits, self.top_k, dim=-<span class="hljs-number">1</span>) | |
| <span class="hljs-keyword">if</span> self.norm_topk_prob: | |
| router_top_value = router_top_value / router_top_value.<span class="hljs-built_in">sum</span>(dim=-<span class="hljs-number">1</span>, keepdim=<span class="hljs-literal">True</span>) | |
| <span class="hljs-keyword">return</span> router_logits, router_top_value.to(router_logits.dtype), router_indices | |
| <span class="hljs-meta">@contextmanager</span> | |
| <span class="hljs-keyword">def</span> <span class="hljs-title function_">replay_moe_routing</span>(<span class="hljs-params">model, selected_experts_per_layer</span>): | |
| routers = [m <span class="hljs-keyword">for</span> m <span class="hljs-keyword">in</span> model.modules() <span class="hljs-keyword">if</span> <span class="hljs-built_in">isinstance</span>(m, ReplayableQwen3MoeTopKRouter)] | |
| <span class="hljs-keyword">if</span> <span class="hljs-built_in">len</span>(routers) != <span class="hljs-built_in">len</span>(selected_experts_per_layer): | |
| <span class="hljs-keyword">raise</span> ValueError(<span class="hljs-string">f"Got <span class="hljs-subst">{<span class="hljs-built_in">len</span>(routers)}</span> routers but <span class="hljs-subst">{<span class="hljs-built_in">len</span>(selected_experts_per_layer)}</span> tensors"</span>) | |
| <span class="hljs-keyword">for</span> r, t <span class="hljs-keyword">in</span> <span class="hljs-built_in">zip</span>(routers, selected_experts_per_layer): | |
| r._forced_indices = t | |
| <span class="hljs-keyword">try</span>: | |
| <span class="hljs-keyword">yield</span> | |
| <span class="hljs-keyword">finally</span>: | |
| <span class="hljs-keyword">for</span> r <span class="hljs-keyword">in</span> routers: | |
| r._forced_indices = <span class="hljs-literal">None</span> | |
| <span class="hljs-comment"># Swap the router class and construct the model</span> | |
| register_patch_mapping({<span class="hljs-string">"Qwen3MoeTopKRouter"</span>: ReplayableQwen3MoeTopKRouter}) | |
| <span class="hljs-keyword">with</span> apply_patches(): | |
| model = Qwen3MoeForCausalLM(Qwen3MoeConfig(...)).<span class="hljs-built_in">eval</span>() | |
| <span class="hljs-comment"># Expose `output_selected_experts=True` on the base model by adding an OutputRecorder</span> | |
| <span class="hljs-comment"># at runtime. Index 2 of the router's tuple output is the expert indices.</span> | |
| inner = model.model | |
| existing = _CAN_RECORD_REGISTRY.get(<span class="hljs-built_in">str</span>(inner.__class__), {}) <span class="hljs-keyword">or</span> {} | |
| _CAN_RECORD_REGISTRY[<span class="hljs-built_in">str</span>(inner.__class__)] = { | |
| **existing, | |
| <span class="hljs-string">"selected_experts"</span>: OutputRecorder(ReplayableQwen3MoeTopKRouter, index=<span class="hljs-number">2</span>), | |
| } | |
| <span class="hljs-comment"># Record</span> | |
| captured = inner(input_ids=input_ids, output_selected_experts=<span class="hljs-literal">True</span>) | |
| selected_experts = captured.selected_experts <span class="hljs-comment"># tuple of (num_tokens, top_k) LongTensors</span> | |
| <span class="hljs-comment"># Replay — same expert path regardless of current router weights</span> | |
| <span class="hljs-keyword">with</span> replay_moe_routing(inner, <span class="hljs-built_in">list</span>(selected_experts)): | |
| outputs = inner(input_ids=input_ids)<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-p15g3k">Replay preserves the exact expert indices and recomputes routing scores with the current router weights, so gradients flow through the live parameters while the expert selection stays fixed. This is the minimal replay contract used by Megatron-style MoE training.</p> <h3 class="relative group"><a id="interop-with-vllm" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#interop-with-vllm"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Interop with vLLM</span></h3> <p data-svelte-h="svelte-1bp5v33">vLLM’s <code>enable_return_routed_experts=True</code> populates <code>CompletionOutput.routed_experts</code> as an <code>(seq_len, num_layers, top_k)</code> <code>np.int32</code> array. Convert it to the per-layer list this pattern expects with a single expression:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-python "><!-- HTML_TAG_START -->selected = [ | |
| torch.from_numpy(routed_experts[:, layer, :].copy()).long() | |
| <span class="hljs-keyword">for</span> layer <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(routed_experts.shape[<span class="hljs-number">1</span>]) | |
| ] | |
| <span class="hljs-keyword">with</span> replay_moe_routing(model, selected): | |
| loss = model(input_ids=input_ids, labels=labels).loss<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-mx6mil">The same recipe applies to other MoE families — subclass the family’s <code>*TopKRouter</code>, match the original return contract (typically <code>(router_logits, router_scores, router_indices)</code>), and register the patch. See each model’s router class for the exact signature.</p> <h2 class="relative group"><a id="transformers.monkey_patching.register_patch_mapping" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#transformers.monkey_patching.register_patch_mapping"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>API reference</span></h2> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"> <div><span class="group flex space-x-1.5 items-center text-gray-800 bg-gradient-to-r rounded-tr-lg -mt-4 -ml-4 pt-3 px-2.5" id="transformers.monkey_patching.register_patch_mapping"><!-- HTML_TAG_START --><h4 class="!m-0"><span class="flex-1 rounded-xl py-0.5 break-all bg-gradient-to-r from-blue-50/60 to-white dark:from-gray-900 dark:to-gray-950 text-blue-700 dark:text-blue-300 font-medium px-2"><svg width="1em" height="1em" viewBox="0 0 32 33" class="mr-1 inline-block -mt-0.5" xmlns="http://www.w3.org/2000/svg"><path d="M5.80566 18.3545C4.90766 17.4565 4.90766 16.0005 5.80566 15.1025L14.3768 6.53142C15.2748 5.63342 16.7307 5.63342 17.6287 6.53142L26.1999 15.1025C27.0979 16.0005 27.0979 17.4565 26.1999 18.3545L17.6287 26.9256C16.7307 27.8236 15.2748 27.8236 14.3768 26.9256L5.80566 18.3545Z" fill="currentColor" fill-opacity="0.25"/><path fill-rule="evenodd" clip-rule="evenodd" d="M16.4801 13.9619C16.4801 12.9761 16.7467 12.5436 16.9443 12.3296C17.1764 12.078 17.5731 11.8517 18.2275 11.707C18.8821 11.5623 19.638 11.5342 20.4038 11.5582C20.7804 11.57 21.1341 11.5932 21.4719 11.6156L21.5263 11.6193C21.8195 11.6389 22.1626 11.6618 22.4429 11.6618V7.40825C22.3209 7.40825 22.1219 7.39596 21.7544 7.37149C21.4202 7.34925 20.9976 7.32115 20.5371 7.30672C19.6286 7.27824 18.4672 7.29779 17.3093 7.55377C16.1512 7.8098 14.8404 8.33724 13.8181 9.4452C12.7612 10.5907 12.2266 12.1236 12.2266 13.9619V15.0127H10.6836V19.2662H12.2266V26.6332H16.4801V19.2662H20.3394V15.0127H16.4801V13.9619Z" fill="currentColor"/></svg>transformers.monkey_patching.register_patch_mapping</span></h4><!-- HTML_TAG_END --> <a id="transformers.monkey_patching.register_patch_mapping" class="header-link invisible with-hover:group-hover:visible pr-2" href="#transformers.monkey_patching.register_patch_mapping"><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></a> <a class="!ml-auto !text-gray-400 !no-underline text-sm flex items-center" href="https://github.com/huggingface/transformers/blob/vr_36895/src/transformers/monkey_patching.py#L85" target="_blank"><span data-svelte-h="svelte-1kd6by1"><</span> <span class="hidden md:block mx-0.5 hover:!underline" data-svelte-h="svelte-122apf4">source</span> <span data-svelte-h="svelte-x0xyl0">></span></a></span> <p class="font-mono text-xs md:text-sm !leading-relaxed !my-6"><span data-svelte-h="svelte-8mvn6a">(</span> <span class="comma cursor-pointer"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">mapping<span class="opacity-60">: dict</span></span> </span><span class="comma cursor-pointer"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">overwrite<span class="opacity-60">: bool = False</span></span> </span> <span data-svelte-h="svelte-1jq0pl7">)</span> </p> <div class="!mb-10 relative docstring-details "> <p class="flex items-center font-semibold !mt-2 !mb-2 text-gray-800" data-svelte-h="svelte-lt6pb6">Parameters <span class="flex-auto border-t-2 border-gray-100 dark:border-gray-700 ml-3"></span></p> <ul class="px-2"><li class="text-base !pl-4 my-3 rounded "><span class="group flex space-x-1.5 items-start"><a id="transformers.monkey_patching.register_patch_mapping.mapping" class="header-link block pr-0.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#transformers.monkey_patching.register_patch_mapping.mapping"><span><svg class="text-smd" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span><!-- HTML_TAG_START --><strong>mapping</strong> (<code>Dict[str, type[nn.Module]]</code>) — | |
| Mapping from original class names (or regex patterns) to replacement classes. Supports:</p> | |
| <ul> | |
| <li>Exact class names: <code>"Qwen2MoeExperts"</code> → <code>CustomExperts</code></li> | |
| <li>Regex patterns: <code>".*Attention"</code> matches <code>LlamaAttention</code>, <code>MistralAttention</code>, etc., | |
| or <code>"^Llama\d+Attention$"</code> matches <code>Llama2Attention</code>, <code>Llama3Attention</code>, etc.</li> | |
| </ul> | |
| <p>Exact matches take precedence over patterns. Patterns are matched using <code>re.search()</code>, | |
| so they can match anywhere in the class name unless you use anchors (<code>^</code> for start, <code>$</code> for end).<!-- HTML_TAG_END --> </span></span> </li><li class="text-base !pl-4 my-3 rounded "><span class="group flex space-x-1.5 items-start"><a id="transformers.monkey_patching.register_patch_mapping.overwrite" class="header-link block pr-0.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#transformers.monkey_patching.register_patch_mapping.overwrite"><span><svg class="text-smd" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span><!-- HTML_TAG_START --><strong>overwrite</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) — | |
| Whether to overwrite existing mappings for class names that are already registered.<!-- HTML_TAG_END --> </span></span> </li></ul> </div></div> <p data-svelte-h="svelte-tqwwex">Register patch mappings to enable automatic patching during model creation using <code>from_pretrained</code>, | |
| <code>from_config</code> or within the <code>apply_patches</code> context manager.</p> <p data-svelte-h="svelte-1tfec45">Use this to register class replacements that will be automatically applied when loading any model. | |
| This is useful for quantization library compatibility, structural optimizations, and architectural | |
| experimentation. The mapping is global, can grow with multiple calls, and can be cleared entirely.</p> <div class="relative group rounded-md"><a id="transformers.monkey_patching.register_patch_mapping.example" class="header-link block pr-0.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#transformers.monkey_patching.register_patch_mapping.example"><span><svg class="text-smd" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <p data-svelte-h="svelte-11lpom8">Example:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-python "><!-- HTML_TAG_START --><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM | |
| <span class="hljs-keyword">from</span> transformers.monkey_patching <span class="hljs-keyword">import</span> register_patch_mapping | |
| <span class="hljs-comment"># Define custom expert implementation</span> | |
| <span class="hljs-keyword">class</span> <span class="hljs-title class_">SequentialExperts</span>(nn.Module): | |
| ... | |
| <span class="hljs-comment"># Register exact class name</span> | |
| register_patch_mapping( | |
| mapping={<span class="hljs-string">"Qwen2MoeExperts"</span>: SequentialExperts} | |
| ) | |
| <span class="hljs-comment"># Register with regex pattern to match multiple classes</span> | |
| register_patch_mapping( | |
| mapping={<span class="hljs-string">".*Attention"</span>: CustomAttention} <span class="hljs-comment"># Matches LlamaAttention, MistralAttention, etc.</span> | |
| ) | |
| <span class="hljs-comment"># Match specific model versions</span> | |
| register_patch_mapping( | |
| mapping={<span class="hljs-string">"^Llama\d+Attention$"</span>: CustomLlamaAttention} <span class="hljs-comment"># Matches Llama2Attention, Llama3Attention</span> | |
| ) | |
| <span class="hljs-comment"># The patch will be automatically applied during loading</span> | |
| model = AutoModelForCausalLM.from_pretrained(<span class="hljs-string">"meta-llama/Llama-3.2-1B"</span>)<!-- HTML_TAG_END --></pre></div></div> <p data-svelte-h="svelte-1f6ln1e">Note: | |
| For weight conversions, use <code>~transformers.register_checkpoint_conversion_mapping</code> instead.</p></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"> <div><span class="group flex space-x-1.5 items-center text-gray-800 bg-gradient-to-r rounded-tr-lg -mt-4 -ml-4 pt-3 px-2.5" id="transformers.monkey_patching.unregister_patch_mapping"><!-- HTML_TAG_START --><h4 class="!m-0"><span class="flex-1 rounded-xl py-0.5 break-all bg-gradient-to-r from-blue-50/60 to-white dark:from-gray-900 dark:to-gray-950 text-blue-700 dark:text-blue-300 font-medium px-2"><svg width="1em" height="1em" viewBox="0 0 32 33" class="mr-1 inline-block -mt-0.5" xmlns="http://www.w3.org/2000/svg"><path d="M5.80566 18.3545C4.90766 17.4565 4.90766 16.0005 5.80566 15.1025L14.3768 6.53142C15.2748 5.63342 16.7307 5.63342 17.6287 6.53142L26.1999 15.1025C27.0979 16.0005 27.0979 17.4565 26.1999 18.3545L17.6287 26.9256C16.7307 27.8236 15.2748 27.8236 14.3768 26.9256L5.80566 18.3545Z" fill="currentColor" fill-opacity="0.25"/><path fill-rule="evenodd" clip-rule="evenodd" d="M16.4801 13.9619C16.4801 12.9761 16.7467 12.5436 16.9443 12.3296C17.1764 12.078 17.5731 11.8517 18.2275 11.707C18.8821 11.5623 19.638 11.5342 20.4038 11.5582C20.7804 11.57 21.1341 11.5932 21.4719 11.6156L21.5263 11.6193C21.8195 11.6389 22.1626 11.6618 22.4429 11.6618V7.40825C22.3209 7.40825 22.1219 7.39596 21.7544 7.37149C21.4202 7.34925 20.9976 7.32115 20.5371 7.30672C19.6286 7.27824 18.4672 7.29779 17.3093 7.55377C16.1512 7.8098 14.8404 8.33724 13.8181 9.4452C12.7612 10.5907 12.2266 12.1236 12.2266 13.9619V15.0127H10.6836V19.2662H12.2266V26.6332H16.4801V19.2662H20.3394V15.0127H16.4801V13.9619Z" fill="currentColor"/></svg>transformers.monkey_patching.unregister_patch_mapping</span></h4><!-- HTML_TAG_END --> <a id="transformers.monkey_patching.unregister_patch_mapping" class="header-link invisible with-hover:group-hover:visible pr-2" href="#transformers.monkey_patching.unregister_patch_mapping"><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></a> <a class="!ml-auto !text-gray-400 !no-underline text-sm flex items-center" href="https://github.com/huggingface/transformers/blob/vr_36895/src/transformers/monkey_patching.py#L158" target="_blank"><span data-svelte-h="svelte-1kd6by1"><</span> <span class="hidden md:block mx-0.5 hover:!underline" data-svelte-h="svelte-122apf4">source</span> <span data-svelte-h="svelte-x0xyl0">></span></a></span> <p class="font-mono text-xs md:text-sm !leading-relaxed !my-6"><span data-svelte-h="svelte-8mvn6a">(</span> <span class="comma cursor-pointer"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">keys<span class="opacity-60">: list</span></span> </span> <span data-svelte-h="svelte-1jq0pl7">)</span> </p> <div class="!mb-10 relative docstring-details "> <p class="flex items-center font-semibold !mt-2 !mb-2 text-gray-800" data-svelte-h="svelte-lt6pb6">Parameters <span class="flex-auto border-t-2 border-gray-100 dark:border-gray-700 ml-3"></span></p> <ul class="px-2"><li class="text-base !pl-4 my-3 rounded "><span class="group flex space-x-1.5 items-start"><a id="transformers.monkey_patching.unregister_patch_mapping.keys" class="header-link block pr-0.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#transformers.monkey_patching.unregister_patch_mapping.keys"><span><svg class="text-smd" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span><!-- HTML_TAG_START --><strong>keys</strong> (<code>List[str]</code>) — | |
| List of mapping keys (class names or regex patterns) to remove from the patch mapping | |
| (e.g., <code>["Qwen2MoeExperts"]</code> or <code>[".*Attention"]</code>).<!-- HTML_TAG_END --> </span></span> </li></ul> </div></div> <p data-svelte-h="svelte-11d73sp">Unregister patch mappings to disable automatic patching.</p> <p data-svelte-h="svelte-6616ml">This removes specified mappings from the global registry, preventing them from being applied | |
| during model loading. You must provide the exact same name or pattern that was used during registration.</p> <div class="relative group rounded-md"><a id="transformers.monkey_patching.unregister_patch_mapping.example" class="header-link block pr-0.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#transformers.monkey_patching.unregister_patch_mapping.example"><span><svg class="text-smd" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <p data-svelte-h="svelte-11lpom8">Example:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-python "><!-- HTML_TAG_START --><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM | |
| <span class="hljs-keyword">from</span> transformers.monkey_patching <span class="hljs-keyword">import</span> register_patch_mapping, unregister_patch_mapping | |
| <span class="hljs-comment"># Register a patch</span> | |
| register_patch_mapping( | |
| mapping={<span class="hljs-string">"Qwen2MoeExperts"</span>: CustomExperts} | |
| ) | |
| <span class="hljs-comment"># Unregister the patch</span> | |
| unregister_patch_mapping([<span class="hljs-string">"Qwen2MoeExperts"</span>]) | |
| <span class="hljs-comment"># The patch will no longer be applied during loading</span> | |
| model = AutoModelForCausalLM.from_pretrained(<span class="hljs-string">"Qwen/Qwen1.5-MoE-A2.7B"</span>)<!-- HTML_TAG_END --></pre></div></div></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"> <div><span class="group flex space-x-1.5 items-center text-gray-800 bg-gradient-to-r rounded-tr-lg -mt-4 -ml-4 pt-3 px-2.5" id="transformers.monkey_patching.clear_patch_mapping"><!-- HTML_TAG_START --><h4 class="!m-0"><span class="flex-1 rounded-xl py-0.5 break-all bg-gradient-to-r from-blue-50/60 to-white dark:from-gray-900 dark:to-gray-950 text-blue-700 dark:text-blue-300 font-medium px-2"><svg width="1em" height="1em" viewBox="0 0 32 33" class="mr-1 inline-block -mt-0.5" xmlns="http://www.w3.org/2000/svg"><path d="M5.80566 18.3545C4.90766 17.4565 4.90766 16.0005 5.80566 15.1025L14.3768 6.53142C15.2748 5.63342 16.7307 5.63342 17.6287 6.53142L26.1999 15.1025C27.0979 16.0005 27.0979 17.4565 26.1999 18.3545L17.6287 26.9256C16.7307 27.8236 15.2748 27.8236 14.3768 26.9256L5.80566 18.3545Z" fill="currentColor" fill-opacity="0.25"/><path fill-rule="evenodd" clip-rule="evenodd" d="M16.4801 13.9619C16.4801 12.9761 16.7467 12.5436 16.9443 12.3296C17.1764 12.078 17.5731 11.8517 18.2275 11.707C18.8821 11.5623 19.638 11.5342 20.4038 11.5582C20.7804 11.57 21.1341 11.5932 21.4719 11.6156L21.5263 11.6193C21.8195 11.6389 22.1626 11.6618 22.4429 11.6618V7.40825C22.3209 7.40825 22.1219 7.39596 21.7544 7.37149C21.4202 7.34925 20.9976 7.32115 20.5371 7.30672C19.6286 7.27824 18.4672 7.29779 17.3093 7.55377C16.1512 7.8098 14.8404 8.33724 13.8181 9.4452C12.7612 10.5907 12.2266 12.1236 12.2266 13.9619V15.0127H10.6836V19.2662H12.2266V26.6332H16.4801V19.2662H20.3394V15.0127H16.4801V13.9619Z" fill="currentColor"/></svg>transformers.monkey_patching.clear_patch_mapping</span></h4><!-- HTML_TAG_END --> <a id="transformers.monkey_patching.clear_patch_mapping" class="header-link invisible with-hover:group-hover:visible pr-2" href="#transformers.monkey_patching.clear_patch_mapping"><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></a> <a class="!ml-auto !text-gray-400 !no-underline text-sm flex items-center" href="https://github.com/huggingface/transformers/blob/vr_36895/src/transformers/monkey_patching.py#L209" target="_blank"><span data-svelte-h="svelte-1kd6by1"><</span> <span class="hidden md:block mx-0.5 hover:!underline" data-svelte-h="svelte-122apf4">source</span> <span data-svelte-h="svelte-x0xyl0">></span></a></span> <p class="font-mono text-xs md:text-sm !leading-relaxed !my-6"><span data-svelte-h="svelte-8mvn6a">(</span> <span data-svelte-h="svelte-1jq0pl7">)</span> </p> <div class="!mb-10 relative docstring-details "> </div></div> <p data-svelte-h="svelte-10kaet5">Clear all registered patch mappings.</p> <p data-svelte-h="svelte-1csb6rw">This removes all registered mappings from the global registry.</p> <div class="relative group rounded-md"><a id="transformers.monkey_patching.clear_patch_mapping.example" class="header-link block pr-0.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#transformers.monkey_patching.clear_patch_mapping.example"><span><svg class="text-smd" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <p data-svelte-h="svelte-11lpom8">Example:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-python "><!-- HTML_TAG_START --><span class="hljs-keyword">from</span> transformers.monkey_patching <span class="hljs-keyword">import</span> register_patch_mapping, clear_patch_mapping | |
| <span class="hljs-comment"># Register some patches</span> | |
| register_patch_mapping( | |
| mapping={<span class="hljs-string">"Qwen2MoeExperts"</span>: CustomExperts} | |
| ) | |
| <span class="hljs-comment"># Clear all patches</span> | |
| clear_patch_mapping()<!-- HTML_TAG_END --></pre></div></div></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"> <div><span class="group flex space-x-1.5 items-center text-gray-800 bg-gradient-to-r rounded-tr-lg -mt-4 -ml-4 pt-3 px-2.5" id="transformers.monkey_patching.get_patch_mapping"><!-- HTML_TAG_START --><h4 class="!m-0"><span class="flex-1 rounded-xl py-0.5 break-all bg-gradient-to-r from-blue-50/60 to-white dark:from-gray-900 dark:to-gray-950 text-blue-700 dark:text-blue-300 font-medium px-2"><svg width="1em" height="1em" viewBox="0 0 32 33" class="mr-1 inline-block -mt-0.5" xmlns="http://www.w3.org/2000/svg"><path d="M5.80566 18.3545C4.90766 17.4565 4.90766 16.0005 5.80566 15.1025L14.3768 6.53142C15.2748 5.63342 16.7307 5.63342 17.6287 6.53142L26.1999 15.1025C27.0979 16.0005 27.0979 17.4565 26.1999 18.3545L17.6287 26.9256C16.7307 27.8236 15.2748 27.8236 14.3768 26.9256L5.80566 18.3545Z" fill="currentColor" fill-opacity="0.25"/><path fill-rule="evenodd" clip-rule="evenodd" d="M16.4801 13.9619C16.4801 12.9761 16.7467 12.5436 16.9443 12.3296C17.1764 12.078 17.5731 11.8517 18.2275 11.707C18.8821 11.5623 19.638 11.5342 20.4038 11.5582C20.7804 11.57 21.1341 11.5932 21.4719 11.6156L21.5263 11.6193C21.8195 11.6389 22.1626 11.6618 22.4429 11.6618V7.40825C22.3209 7.40825 22.1219 7.39596 21.7544 7.37149C21.4202 7.34925 20.9976 7.32115 20.5371 7.30672C19.6286 7.27824 18.4672 7.29779 17.3093 7.55377C16.1512 7.8098 14.8404 8.33724 13.8181 9.4452C12.7612 10.5907 12.2266 12.1236 12.2266 13.9619V15.0127H10.6836V19.2662H12.2266V26.6332H16.4801V19.2662H20.3394V15.0127H16.4801V13.9619Z" fill="currentColor"/></svg>transformers.monkey_patching.get_patch_mapping</span></h4><!-- HTML_TAG_END --> <a id="transformers.monkey_patching.get_patch_mapping" class="header-link invisible with-hover:group-hover:visible pr-2" href="#transformers.monkey_patching.get_patch_mapping"><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></a> <a class="!ml-auto !text-gray-400 !no-underline text-sm flex items-center" href="https://github.com/huggingface/transformers/blob/vr_36895/src/transformers/monkey_patching.py#L198" target="_blank"><span data-svelte-h="svelte-1kd6by1"><</span> <span class="hidden md:block mx-0.5 hover:!underline" data-svelte-h="svelte-122apf4">source</span> <span data-svelte-h="svelte-x0xyl0">></span></a></span> <p class="font-mono text-xs md:text-sm !leading-relaxed !my-6"><span data-svelte-h="svelte-8mvn6a">(</span> <span data-svelte-h="svelte-1jq0pl7">)</span> <span class="font-bold" data-svelte-h="svelte-1j6k10o">→</span> <span class="rounded hover:bg-gray-400 cursor-pointer"><!-- HTML_TAG_START --><script context="module">export const metadata = 'undefined';</script><span><code>Dict[str, type[nn.Module]]</code></span><!-- HTML_TAG_END --></span></p> <div class="!mb-10 relative docstring-details "> <div id="transformers.monkey_patching.get_patch_mapping.returns" class="flex items-center font-semibold space-x-3 text-base !mt-0 !mb-0 text-gray-800 rounded "><p class="text-base">Returns</p> <!-- HTML_TAG_START --><script context="module">export const metadata = 'undefined';</script> | |
| <p><code>Dict[str, type[nn.Module]]</code></p> | |
| <!-- HTML_TAG_END --> <span class="flex-auto border-t-2 border-gray-100 dark:border-gray-700"></span></div> <p class="text-base"><!-- HTML_TAG_START --><script context="module">export const metadata = 'undefined';</script> | |
| <p>Dictionary mapping class names or patterns to replacement classes.</p> | |
| <!-- HTML_TAG_END --></p> </div></div> <p data-svelte-h="svelte-sx1ktk">Get all registered patch mappings.</p></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"> <div><span class="group flex space-x-1.5 items-center text-gray-800 bg-gradient-to-r rounded-tr-lg -mt-4 -ml-4 pt-3 px-2.5" id="transformers.monkey_patching.apply_patches"><!-- HTML_TAG_START --><h4 class="!m-0"><span class="flex-1 rounded-xl py-0.5 break-all bg-gradient-to-r from-blue-50/60 to-white dark:from-gray-900 dark:to-gray-950 text-blue-700 dark:text-blue-300 font-medium px-2"><svg width="1em" height="1em" viewBox="0 0 32 33" class="mr-1 inline-block -mt-0.5" xmlns="http://www.w3.org/2000/svg"><path d="M5.80566 18.3545C4.90766 17.4565 4.90766 16.0005 5.80566 15.1025L14.3768 6.53142C15.2748 5.63342 16.7307 5.63342 17.6287 6.53142L26.1999 15.1025C27.0979 16.0005 27.0979 17.4565 26.1999 18.3545L17.6287 26.9256C16.7307 27.8236 15.2748 27.8236 14.3768 26.9256L5.80566 18.3545Z" fill="currentColor" fill-opacity="0.25"/><path fill-rule="evenodd" clip-rule="evenodd" d="M16.4801 13.9619C16.4801 12.9761 16.7467 12.5436 16.9443 12.3296C17.1764 12.078 17.5731 11.8517 18.2275 11.707C18.8821 11.5623 19.638 11.5342 20.4038 11.5582C20.7804 11.57 21.1341 11.5932 21.4719 11.6156L21.5263 11.6193C21.8195 11.6389 22.1626 11.6618 22.4429 11.6618V7.40825C22.3209 7.40825 22.1219 7.39596 21.7544 7.37149C21.4202 7.34925 20.9976 7.32115 20.5371 7.30672C19.6286 7.27824 18.4672 7.29779 17.3093 7.55377C16.1512 7.8098 14.8404 8.33724 13.8181 9.4452C12.7612 10.5907 12.2266 12.1236 12.2266 13.9619V15.0127H10.6836V19.2662H12.2266V26.6332H16.4801V19.2662H20.3394V15.0127H16.4801V13.9619Z" fill="currentColor"/></svg>transformers.monkey_patching.apply_patches</span></h4><!-- HTML_TAG_END --> <a id="transformers.monkey_patching.apply_patches" class="header-link invisible with-hover:group-hover:visible pr-2" href="#transformers.monkey_patching.apply_patches"><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></a> <a class="!ml-auto !text-gray-400 !no-underline text-sm flex items-center" href="https://github.com/huggingface/transformers/blob/vr_36895/src/transformers/monkey_patching.py#L233" target="_blank"><span data-svelte-h="svelte-1kd6by1"><</span> <span class="hidden md:block mx-0.5 hover:!underline" data-svelte-h="svelte-122apf4">source</span> <span data-svelte-h="svelte-x0xyl0">></span></a></span> <p class="font-mono text-xs md:text-sm !leading-relaxed !my-6"><span data-svelte-h="svelte-8mvn6a">(</span> <span data-svelte-h="svelte-1jq0pl7">)</span> </p> <div class="!mb-10 relative docstring-details "> </div></div> <p data-svelte-h="svelte-1w17b2r">Context manager to apply registered monkey patches within a block of code.</p> <p data-svelte-h="svelte-17wshqy">This temporarily replaces original classes with their registered replacements during the execution of the block, and restores the original classes afterward.</p> <div class="relative group rounded-md"><a id="transformers.monkey_patching.apply_patches.example" class="header-link block pr-0.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#transformers.monkey_patching.apply_patches.example"><span><svg class="text-smd" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <p data-svelte-h="svelte-11lpom8">Example:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-python "><!-- HTML_TAG_START --><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> Qwen2MoeModel, Qwen2MoeConfig | |
| <span class="hljs-keyword">from</span> transformers.monkey_patching <span class="hljs-keyword">import</span> register_patch_mapping, apply_patches | |
| <span class="hljs-comment"># Register a patch</span> | |
| register_patch_mapping( | |
| mapping={<span class="hljs-string">"Qwen2MoeExperts"</span>: CustomExperts} | |
| ) | |
| <span class="hljs-comment"># Apply patches within the context</span> | |
| <span class="hljs-keyword">with</span> apply_patches(): | |
| <span class="hljs-comment"># The model will use CustomExperts instead of Qwen2MoeExperts</span> | |
| model = Qwen2MoeModel(Qwen2MoeConfig()) | |
| <span class="hljs-comment"># Outside the context, original classes are restored</span> | |
| <span class="hljs-comment"># The model will use Qwen2MoeExperts again</span> | |
| model = Qwen2MoeModel(Qwen2MoeConfig())<!-- HTML_TAG_END --></pre></div></div></div> <a class="!text-gray-400 !no-underline text-sm flex items-center not-prose mt-4" href="https://github.com/huggingface/transformers/blob/main/docs/source/en/monkey_patching.md" target="_blank"><svg class="mr-1" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M31,16l-7,7l-1.41-1.41L28.17,16l-5.58-5.59L24,9l7,7z"></path><path d="M1,16l7-7l1.41,1.41L3.83,16l5.58,5.59L8,23l-7-7z"></path><path d="M12.419,25.484L17.639,6.552l1.932,0.518L14.351,26.002z"></path></svg> <span data-svelte-h="svelte-zjs2n5"><span class="underline">Update</span> on GitHub</span></a> <p></p> | |
| <script> | |
| { | |
| __sveltekit_1s77840 = { | |
| assets: "/docs/transformers/pr_36895/en", | |
| base: "/docs/transformers/pr_36895/en", | |
| env: {} | |
| }; | |
| const element = document.currentScript.parentElement; | |
| const data = [null,null]; | |
| Promise.all([ | |
| import("/docs/transformers/pr_36895/en/_app/immutable/entry/start.b88698fb.js"), | |
| import("/docs/transformers/pr_36895/en/_app/immutable/entry/app.ac300a2c.js") | |
| ]).then(([kit, app]) => { | |
| kit.start(app, element, { | |
| node_ids: [0, 585], | |
| data, | |
| form: null, | |
| error: null | |
| }); | |
| }); | |
| } | |
| </script> | |
Xet Storage Details
- Size:
- 120 kB
- Xet hash:
- b58680b4df68fc311511d1ad9e2547e2b74d71a13bc8a0d40b6042fadb90d9c9
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.