Buckets:
| <meta charset="utf-8" /><meta name="hf:doc:metadata" content="{"title":"Add a model with modular transformers","local":"add-a-model-with-modular-transformers","sections":[{"title":"Implementing a modular file","local":"implementing-a-modular-file","sections":[{"title":"Config","local":"config","sections":[],"depth":3},{"title":"Norm","local":"norm","sections":[],"depth":3},{"title":"Attention","local":"attention","sections":[],"depth":3},{"title":"DecoderLayer","local":"decoderlayer","sections":[],"depth":3},{"title":"Model","local":"model","sections":[],"depth":3},{"title":"Model head","local":"model-head","sections":[],"depth":3},{"title":"Other classes","local":"other-classes","sections":[],"depth":3},{"title":"Finishing the file","local":"finishing-the-file","sections":[],"depth":3}],"depth":2},{"title":"Generate the modeling files","local":"generate-the-modeling-files","sections":[],"depth":2},{"title":"Patterns for modular files","local":"patterns-for-modular-files","sections":[{"title":"Removing attributes","local":"removing-attributes","sections":[],"depth":3},{"title":"Working with super()","local":"working-with-super","sections":[{"title":"Call a grandparent class directly","local":"call-a-grandparent-class-directly","sections":[],"depth":4},{"title":"super_kwargs","local":"superkwargs","sections":[],"depth":4}],"depth":3},{"title":"Deleting unused methods","local":"deleting-unused-methods","sections":[],"depth":3},{"title":"Overriding decorated methods","local":"overriding-decorated-methods","sections":[],"depth":3},{"title":"Special naming","local":"special-naming","sections":[],"depth":3},{"title":"Config docstrings","local":"config-docstrings","sections":[],"depth":3}],"depth":2},{"title":"Checkpoint conversion","local":"checkpoint-conversion","sections":[{"title":"Write a conversion script","local":"write-a-conversion-script","sections":[],"depth":3},{"title":"Runtime conversion mapping","local":"runtime-conversion-mapping","sections":[],"depth":3}],"depth":2},{"title":"Next steps","local":"next-steps","sections":[],"depth":2}],"depth":1}"> | |
| <link href="/docs/transformers/pr_43838/en/_app/immutable/assets/0.e3b0c442.css" rel="modulepreload"> | |
| <link rel="modulepreload" href="/docs/transformers/pr_43838/en/_app/immutable/entry/start.1bcc8411.js"> | |
| <link rel="modulepreload" href="/docs/transformers/pr_43838/en/_app/immutable/chunks/scheduler.31fdf58d.js"> | |
| <link rel="modulepreload" href="/docs/transformers/pr_43838/en/_app/immutable/chunks/singletons.49522dd1.js"> | |
| <link rel="modulepreload" href="/docs/transformers/pr_43838/en/_app/immutable/chunks/index.252883d5.js"> | |
| <link rel="modulepreload" href="/docs/transformers/pr_43838/en/_app/immutable/chunks/paths.91497448.js"> | |
| <link rel="modulepreload" href="/docs/transformers/pr_43838/en/_app/immutable/entry/app.9d434a07.js"> | |
| <link rel="modulepreload" href="/docs/transformers/pr_43838/en/_app/immutable/chunks/preload-helper.f02a9c49.js"> | |
| <link rel="modulepreload" href="/docs/transformers/pr_43838/en/_app/immutable/chunks/index.2f76fdf0.js"> | |
| <link rel="modulepreload" href="/docs/transformers/pr_43838/en/_app/immutable/nodes/0.b44e4c87.js"> | |
| <link rel="modulepreload" href="/docs/transformers/pr_43838/en/_app/immutable/chunks/each.e59479a4.js"> | |
| <link rel="modulepreload" href="/docs/transformers/pr_43838/en/_app/immutable/nodes/610.a1864c99.js"> | |
| <link rel="modulepreload" href="/docs/transformers/pr_43838/en/_app/immutable/chunks/CopyLLMTxtMenu.f658c69c.js"> | |
| <link rel="modulepreload" href="/docs/transformers/pr_43838/en/_app/immutable/chunks/MermaidChart.svelte_svelte_type_style_lang.0d9e7a9c.js"> | |
| <link rel="modulepreload" href="/docs/transformers/pr_43838/en/_app/immutable/chunks/IconCopy.ac192424.js"> | |
| <link rel="modulepreload" href="/docs/transformers/pr_43838/en/_app/immutable/chunks/CodeBlock.e52df5d6.js"><!-- HEAD_svelte-u9bgzb_START --><meta name="hf:doc:metadata" content="{"title":"Add a model with modular transformers","local":"add-a-model-with-modular-transformers","sections":[{"title":"Implementing a modular file","local":"implementing-a-modular-file","sections":[{"title":"Config","local":"config","sections":[],"depth":3},{"title":"Norm","local":"norm","sections":[],"depth":3},{"title":"Attention","local":"attention","sections":[],"depth":3},{"title":"DecoderLayer","local":"decoderlayer","sections":[],"depth":3},{"title":"Model","local":"model","sections":[],"depth":3},{"title":"Model head","local":"model-head","sections":[],"depth":3},{"title":"Other classes","local":"other-classes","sections":[],"depth":3},{"title":"Finishing the file","local":"finishing-the-file","sections":[],"depth":3}],"depth":2},{"title":"Generate the modeling files","local":"generate-the-modeling-files","sections":[],"depth":2},{"title":"Patterns for modular files","local":"patterns-for-modular-files","sections":[{"title":"Removing attributes","local":"removing-attributes","sections":[],"depth":3},{"title":"Working with super()","local":"working-with-super","sections":[{"title":"Call a grandparent class directly","local":"call-a-grandparent-class-directly","sections":[],"depth":4},{"title":"super_kwargs","local":"superkwargs","sections":[],"depth":4}],"depth":3},{"title":"Deleting unused methods","local":"deleting-unused-methods","sections":[],"depth":3},{"title":"Overriding decorated methods","local":"overriding-decorated-methods","sections":[],"depth":3},{"title":"Special naming","local":"special-naming","sections":[],"depth":3},{"title":"Config docstrings","local":"config-docstrings","sections":[],"depth":3}],"depth":2},{"title":"Checkpoint conversion","local":"checkpoint-conversion","sections":[{"title":"Write a conversion script","local":"write-a-conversion-script","sections":[],"depth":3},{"title":"Runtime conversion mapping","local":"runtime-conversion-mapping","sections":[],"depth":3}],"depth":2},{"title":"Next steps","local":"next-steps","sections":[],"depth":2}],"depth":1}"><!-- HEAD_svelte-u9bgzb_END --> <p></p> <div class="items-center shrink-0 min-w-[100px] max-sm:min-w-[50px] justify-end ml-auto flex" style="float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"><div class="inline-flex rounded-md max-sm:rounded-sm"><button class="inline-flex items-center gap-1 h-7 max-sm:h-7 px-2 max-sm:px-1.5 text-sm font-medium text-gray-800 border border-r-0 rounded-l-md max-sm:rounded-l-sm border-gray-200 bg-white hover:shadow-inner dark:border-gray-850 dark:bg-gray-950 dark:text-gray-200 dark:hover:bg-gray-800" aria-live="polite"><span class="inline-flex items-center justify-center rounded-md p-0.5 max-sm:p-0 hover:text-gray-800 dark:hover:text-gray-200"><svg class="sm:size-3.5 size-3" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg></span> <span>Copy page</span></button> <button class="inline-flex items-center justify-center w-6 max-sm:w-5 h-7 max-sm:h-7 disabled:pointer-events-none text-sm text-gray-500 hover:text-gray-700 dark:hover:text-white rounded-r-md max-sm:rounded-r-sm border border-l transition border-gray-200 bg-white hover:shadow-inner dark:border-gray-850 dark:bg-gray-950 dark:text-gray-200 dark:hover:bg-gray-800" aria-haspopup="menu" aria-expanded="false" aria-label="Open copy menu"><svg class="transition-transform text-gray-400 overflow-visible sm:size-3.5 size-3 rotate-0" width="1em" height="1em" viewBox="0 0 12 7" fill="none" xmlns="http://www.w3.org/2000/svg"><path d="M1 1L6 6L11 1" stroke="currentColor"></path></svg></button></div> </div> <h1 class="relative group"><a id="add-a-model-with-modular-transformers" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#add-a-model-with-modular-transformers"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Add a model with modular transformers</span></h1> <p data-svelte-h="svelte-ndn82s">Modular transformers reduces the code needed to add a model by allowing imports and inheritance, in contrast to the <a href="https://huggingface.co/blog/transformers-design-philosophy" rel="nofollow">single model, single file</a> policy. Instead of repeating model components across files, add a <em>modular</em> file to your model folder and inherit from existing classes.</p> <p data-svelte-h="svelte-1xvicrc">A converter generates standalone files from the modular file. Users get the same single-file interface they already know.</p> <blockquote class="note" data-svelte-h="svelte-pgurfo"><p>Modular transformers isn’t meant to replace the <a href="./add_new_model">legacy modeling code</a>. If your model isn’t based on an existing model, add a <code>modeling.py</code> file manually. The same applies to configuration, tokenization, or processing files that can’t cleanly inherit from a similar file.</p> <p>There’s no single right order either. Some contributors write the modular file first and generate from it. Others start with a hand-written <code>modeling.py</code> and refactor it into a modular file later. Both approaches work.</p></blockquote> <h2 class="relative group"><a id="implementing-a-modular-file" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#implementing-a-modular-file"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Implementing a modular file</span></h2> <p data-svelte-h="svelte-1q05wbt">Start by finding a model in Transformers similar to yours. Good starting points are <a href="./model_doc/mistral">Mistral</a>, <a href="./model_doc/qwen2">Qwen2</a>, <a href="./model_doc/cohere">Cohere</a> and <a href="./model_doc/cohere2">Cohere2</a>, and <a href="./model_doc/llama">Llama</a>. The table below maps common components to models you can inherit from.</p> <table data-svelte-h="svelte-10o9t2n"><thead><tr><th>Component</th> <th>Model</th></tr></thead> <tbody><tr><td>Mixture of experts</td> <td>Mixtral or Qwen2-MoE</td></tr> <tr><td>Interleaved (and/or partial) rotary embedding</td> <td>GLM, Phi</td></tr> <tr><td>State space models</td> <td>Jamba, Bamba, Zamba, Mamba2</td></tr> <tr><td>Recurrent hidden states</td> <td>Gemma2</td></tr> <tr><td>Sliding window attention/full attention patterns per layer</td> <td>Gemma2, Cohere2</td></tr> <tr><td>QKV clipping</td> <td>Olmo</td></tr> <tr><td>QK normalization</td> <td>Olmo2, Cohere</td></tr> <tr><td>Fused QKV (not recommended)</td> <td>Phi3</td></tr></tbody></table> <blockquote class="tip" data-svelte-h="svelte-173zbl3"><p>Use the <a href="https://huggingface.co/spaces/Molbap/modular-detector-v2" rel="nofollow">modular-detector-v2</a> tool to find existing implementations to inherit from. Paste a code snippet and it returns the most similar methods already in Transformers, so you can identify the best parent class before you start writing.</p></blockquote> <p data-svelte-h="svelte-c01g80">Don’t modify an existing model just to make inheritance work for your new one. If renaming or subclassing a parent class is too awkward, copy the relevant code directly instead.</p> <p data-svelte-h="svelte-ee47me">Create <code>src/transformers/models/<name>/modular_<name>.py</code>, where <code><name></code> matches the snake_case model directory name. This section walks you through implementing <a href="./model_doc/olmo2">Olmo2</a> from <a href="./model_doc/olmo">Olmo</a> with the modular approach (refer to the original <a href="../../../src/transformers/models/olmo2/modular_olmo2">modular_olmo2.py</a> file).</p> <h3 class="relative group"><a id="config" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#config"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Config</span></h3> <p data-svelte-h="svelte-1ij7183">There are two points where <a href="/docs/transformers/pr_43838/en/model_doc/olmo2#transformers.Olmo2Config">Olmo2Config</a> differs from <a href="/docs/transformers/pr_43838/en/model_doc/olmo#transformers.OlmoConfig">OlmoConfig</a>.</p> <ol data-svelte-h="svelte-zxeld0"><li>There is a new argument, <code>rms_norm_eps</code>.</li> <li>The <code>clip_qkv</code> argument is no longer used.</li></ol> <p data-svelte-h="svelte-1lnzmqz">Declare new arguments as class-level type annotations with a default value. For removed arguments, assign <code>AttributeError()</code> to suppress the inherited attribute in the generated file (see <a href="#removing-attributes">Removing attributes</a>).</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-diff "><!-- HTML_TAG_START --><span class="hljs-deletion">- @auto_docstring(checkpoint="allenai/OLMo-7B-hf")</span> | |
| <span class="hljs-addition">+ @auto_docstring(checkpoint="allenai/Olmo2-7B-1124-hf")</span> | |
| <span class="hljs-addition">+ @strict</span> | |
| <span class="hljs-deletion">- class OlmoConfig(PreTrainedConfig):</span> | |
| <span class="hljs-addition">+ class Olmo2Config(OlmoConfig):</span> | |
| ... | |
| <span class="hljs-deletion">- model_type = "olmo"</span> | |
| <span class="hljs-addition">+ model_type = "olmo2"</span> | |
| ... | |
| <span class="hljs-addition">+ rms_norm_eps: float = 1e-5</span> | |
| <span class="hljs-deletion">- clip_qkv: float | None = None</span> | |
| <span class="hljs-addition">+ clip_qkv = AttributeError()</span><!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-1v77pe2"><code>@auto_docstring</code> generates standard argument docs automatically (see the <a href="./auto_docstring">@auto_docstring</a> guide). <code>@strict</code> rejects unknown kwargs at instantiation time, catching typos and stale arguments early. Add both to every config class because the decorators aren’t inherited from the parent. Declare them explicitly even if the parent config already has them.</p> <p data-svelte-h="svelte-k8rze1">To set a derived attribute or handle backward-compatibility logic, use <code>__post_init__</code> instead of <code>__init__</code>. For example, Cohere2 computes <code>head_dim</code> and derives <code>layer_types</code> at init time.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-py "><!-- HTML_TAG_START --><span class="hljs-keyword">def</span> <span class="hljs-title function_">__post_init__</span>(<span class="hljs-params">self, **kwargs</span>): | |
| <span class="hljs-keyword">if</span> self.num_key_value_heads <span class="hljs-keyword">is</span> <span class="hljs-literal">None</span>: | |
| self.num_key_value_heads = self.num_attention_heads | |
| self.head_dim = self.hidden_size // self.num_attention_heads | |
| <span class="hljs-built_in">super</span>().__post_init__(**kwargs)<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-udgu1q">For models with tensor or pipeline parallelism support, define <code>base_model_tp_plan</code> and <code>base_model_pp_plan</code> as class-level dictionaries on the config. Both dictionaries define how to shard the model across devices. See existing configs like <a href="../../../src/transformers/models/olmo2/modular_olmo2">Olmo2</a> or <a href="../../../src/transformers/models/cohere2/modular_cohere2">Cohere2</a> for examples.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-py "><!-- HTML_TAG_START --><span class="hljs-keyword">class</span> <span class="hljs-title class_">MyNewModelConfig</span>(<span class="hljs-title class_ inherited__">PreTrainedConfig</span>): | |
| model_type = <span class="hljs-string">"my_new_model"</span> | |
| <span class="hljs-comment"># Tensor parallelism: maps layer name patterns to sharding strategies.</span> | |
| <span class="hljs-comment"># Use "colwise" / "rowwise" for standard sharding, or the "gather_output" /</span> | |
| <span class="hljs-comment"># "split_input" variants when an extra op (e.g. a QK norm) prevents fusing.</span> | |
| base_model_tp_plan = { | |
| <span class="hljs-string">"layers.*.self_attn.q_proj"</span>: <span class="hljs-string">"colwise"</span>, | |
| <span class="hljs-string">"layers.*.self_attn.k_proj"</span>: <span class="hljs-string">"colwise"</span>, | |
| <span class="hljs-string">"layers.*.self_attn.v_proj"</span>: <span class="hljs-string">"colwise"</span>, | |
| <span class="hljs-string">"layers.*.self_attn.o_proj"</span>: <span class="hljs-string">"rowwise"</span>, | |
| <span class="hljs-string">"layers.*.mlp.gate_proj"</span>: <span class="hljs-string">"colwise"</span>, | |
| <span class="hljs-string">"layers.*.mlp.up_proj"</span>: <span class="hljs-string">"colwise"</span>, | |
| <span class="hljs-string">"layers.*.mlp.down_proj"</span>: <span class="hljs-string">"rowwise"</span>, | |
| } | |
| <span class="hljs-comment"># Pipeline parallelism: maps submodule names to their (input, output) tensor names.</span> | |
| base_model_pp_plan = { | |
| <span class="hljs-string">"embed_tokens"</span>: ([<span class="hljs-string">"input_ids"</span>], [<span class="hljs-string">"inputs_embeds"</span>]), | |
| <span class="hljs-string">"layers"</span>: ([<span class="hljs-string">"hidden_states"</span>, <span class="hljs-string">"attention_mask"</span>], [<span class="hljs-string">"hidden_states"</span>]), | |
| <span class="hljs-string">"norm"</span>: ([<span class="hljs-string">"hidden_states"</span>], [<span class="hljs-string">"hidden_states"</span>]), | |
| }<!-- HTML_TAG_END --></pre></div> <h3 class="relative group"><a id="norm" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#norm"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Norm</span></h3> <p data-svelte-h="svelte-rg3ytm">To copy a parent class without changes, inherit with <code>pass</code>. The linter copies the parent’s content and renames all references to match the new model.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-py "><!-- HTML_TAG_START --><span class="hljs-keyword">from</span> ..olmo.modeling_olmo <span class="hljs-keyword">import</span> OlmoRotaryEmbedding | |
| <span class="hljs-keyword">class</span> <span class="hljs-title class_">Olmo2RotaryEmbedding</span>(<span class="hljs-title class_ inherited__">OlmoRotaryEmbedding</span>): | |
| <span class="hljs-keyword">pass</span><!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-q1l9l9">To change specific behavior, inherit and override only what differs. <code>Olmo2RMSNorm</code> differs from <code>LlamaRMSNorm</code> on one line. The multiply happens <em>before</em> casting back to the input dtype, not after.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-diff "><!-- HTML_TAG_START --> from ..llama.modeling_llama import LlamaRMSNorm | |
| class Olmo2RMSNorm(LlamaRMSNorm): | |
| def forward(self, hidden_states): | |
| input_dtype = hidden_states.dtype | |
| hidden_states = hidden_states.to(torch.float32) | |
| variance = hidden_states.pow(2).mean(-1, keepdim=True) | |
| hidden_states = hidden_states * torch.rsqrt(variance + self.variance_epsilon) | |
| <span class="hljs-deletion">- return self.weight * hidden_states.to(input_dtype)</span> | |
| <span class="hljs-addition">+ return (self.weight * hidden_states).to(input_dtype)</span><!-- HTML_TAG_END --></pre></div> <h3 class="relative group"><a id="attention" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#attention"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Attention</span></h3> <p data-svelte-h="svelte-1kjhak9">Olmo2’s attention is identical to Olmo’s except it applies <code>RMSNorm</code> to the queries and keys, and removes qkv clipping. <code>super().__init__(...)</code> copies the parent body and appends the two new norm lines. The <code>forward</code> is fully redefined because queries and keys now pass through norms before projection. The linter also pulls in any imported functions into the generated file, including <code>apply_rotary_pos_emb</code>, <code>eager_attention_forward</code>, and their dependencies.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-diff "><!-- HTML_TAG_START --> class Olmo2Attention(OlmoAttention): | |
| def __init__(self, config: Olmo2Config, layer_idx: int | None = None): | |
| super().__init__(config, layer_idx=layer_idx) | |
| <span class="hljs-addition">+ self.q_norm = Olmo2RMSNorm(config.num_attention_heads * self.head_dim, config.rms_norm_eps)</span> | |
| <span class="hljs-addition">+ self.k_norm = Olmo2RMSNorm(config.num_key_value_heads * self.head_dim, config.rms_norm_eps)</span> | |
| def forward(self, ...): | |
| ... | |
| <span class="hljs-deletion">- query_states = self.q_proj(hidden_states)</span> | |
| <span class="hljs-deletion">- key_states = self.k_proj(hidden_states)</span> | |
| <span class="hljs-addition">+ query_states = self.q_norm(self.q_proj(hidden_states))</span> | |
| <span class="hljs-addition">+ key_states = self.k_norm(self.k_proj(hidden_states))</span> | |
| value_states = self.v_proj(hidden_states) | |
| <span class="hljs-deletion">- if self.config.clip_qkv is not None:</span> | |
| <span class="hljs-deletion">- query_states.clamp_(min=-self.config.clip_qkv, max=self.config.clip_qkv)</span> | |
| <span class="hljs-deletion">- key_states.clamp_(min=-self.config.clip_qkv, max=self.config.clip_qkv)</span> | |
| <span class="hljs-deletion">- value_states.clamp_(min=-self.config.clip_qkv, max=self.config.clip_qkv)</span> | |
| <span class="hljs-deletion">-</span> | |
| ...<!-- HTML_TAG_END --></pre></div> <h3 class="relative group"><a id="decoderlayer" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#decoderlayer"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>DecoderLayer</span></h3> <p data-svelte-h="svelte-is69d7">After <code>super().__init__(...)</code>, overwrite the norm attributes with <code>Olmo2RMSNorm</code> instances and reassign <code>self.self_attn</code> to the new <code>Olmo2Attention</code> class. The <code>del self.input_layernorm</code> removes the parent’s <code>input_layernorm</code> assignment since Olmo2 applies the norm <em>after</em> attention rather than before. See <a href="#removing-attributes">Removing attributes</a> for details on what <code>del</code> does and doesn’t remove.</p> <p data-svelte-h="svelte-lma1ur">The <code>forward</code> is rewritten to reflect the post-attention norm placement. A <code>forward</code> rewrite is only needed when an attribute is renamed, not when only its type changes.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-diff "><!-- HTML_TAG_START --> class Olmo2DecoderLayer(OlmoDecoderLayer): | |
| def __init__(self, config: Olmo2Config, layer_idx: int): | |
| super().__init__(config, layer_idx=layer_idx) | |
| <span class="hljs-deletion">- self.self_attn = OlmoAttention(config=config, layer_idx=layer_idx)</span> | |
| <span class="hljs-deletion">- self.input_layernorm = OlmoLayerNorm(config.hidden_size)</span> | |
| <span class="hljs-deletion">- self.post_attention_layernorm = OlmoLayerNorm(config.hidden_size)</span> | |
| <span class="hljs-addition">+ self.self_attn = Olmo2Attention(config=config, layer_idx=layer_idx)</span> | |
| <span class="hljs-addition">+ del self.input_layernorm</span> | |
| <span class="hljs-addition">+ self.post_attention_layernorm = Olmo2RMSNorm(config.hidden_size, eps=config.rms_norm_eps)</span> | |
| <span class="hljs-addition">+ self.post_feedforward_layernorm = Olmo2RMSNorm(config.hidden_size, eps=config.rms_norm_eps)</span> | |
| def forward(self, ...): | |
| residual = hidden_states | |
| <span class="hljs-deletion">- hidden_states = self.input_layernorm(hidden_states)</span> | |
| # Self Attention | |
| hidden_states, _ = self.self_attn(...) | |
| <span class="hljs-deletion">- hidden_states = residual + hidden_states</span> | |
| <span class="hljs-addition">+ hidden_states = self.post_attention_layernorm(hidden_states)</span> | |
| <span class="hljs-addition">+ hidden_states = residual + hidden_states</span> | |
| # Fully Connected | |
| residual = hidden_states | |
| <span class="hljs-deletion">- hidden_states = self.post_attention_layernorm(hidden_states)</span> | |
| hidden_states = self.mlp(hidden_states) | |
| <span class="hljs-deletion">- hidden_states = residual + hidden_states</span> | |
| <span class="hljs-addition">+ hidden_states = self.post_feedforward_layernorm(hidden_states)</span> | |
| <span class="hljs-addition">+ hidden_states = residual + hidden_states</span> | |
| return hidden_states<!-- HTML_TAG_END --></pre></div> <h3 class="relative group"><a id="model" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#model"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Model</span></h3> <p data-svelte-h="svelte-xa7jz5">Only the type of <code>self.norm</code> changes here. The <code>forward</code> method is identical to the parent’s, so the linter carries it over automatically.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-diff "><!-- HTML_TAG_START --> class Olmo2Model(OlmoModel): | |
| def __init__(self, config: Olmo2Config): | |
| super().__init__(config) | |
| <span class="hljs-deletion">- self.layers = nn.ModuleList(</span> | |
| <span class="hljs-deletion">- [OlmoDecoderLayer(config, layer_idx) for layer_idx in range(config.num_hidden_layers)]</span> | |
| <span class="hljs-deletion">- )</span> | |
| <span class="hljs-deletion">- self.norm = OlmoLayerNorm(config.hidden_size)</span> | |
| <span class="hljs-addition">+ self.norm = Olmo2RMSNorm(config.hidden_size, eps=config.rms_norm_eps)</span> | |
| <span class="hljs-addition">+ self.layers = nn.ModuleList(</span> | |
| <span class="hljs-addition">+ [Olmo2DecoderLayer(config, layer_idx) for layer_idx in range(config.num_hidden_layers)]</span> | |
| <span class="hljs-addition">+ )</span><!-- HTML_TAG_END --></pre></div> <h3 class="relative group"><a id="model-head" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#model-head"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Model head</span></h3> <p data-svelte-h="svelte-19exvqe">The logic is identical to <a href="/docs/transformers/pr_43838/en/model_doc/olmo#transformers.OlmoForCausalLM">OlmoForCausalLM</a>, so no changes are needed.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-py "><!-- HTML_TAG_START --><span class="hljs-keyword">from</span> ..olmo.modeling_olmo <span class="hljs-keyword">import</span> OlmoForCausalLM | |
| <span class="hljs-keyword">class</span> <span class="hljs-title class_">Olmo2ForCausalLM</span>(<span class="hljs-title class_ inherited__">OlmoForCausalLM</span>): | |
| <span class="hljs-keyword">pass</span><!-- HTML_TAG_END --></pre></div> <h3 class="relative group"><a id="other-classes" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#other-classes"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Other classes</span></h3> <p data-svelte-h="svelte-14tatwp">The <a href="../../../src/transformers/models/olmo2/modeling_olmo2">modeling_olmo2.py</a> generated by the linter also contains classes (<code>Olmo2MLP</code>, <code>Olmo2RotaryEmbedding</code>, <code>Olmo2PreTrainedModel</code>) that weren’t explicitly defined in <code>modular_olmo2.py</code>.</p> <p data-svelte-h="svelte-1478hcz">The linter pulls in any class an inherited class depends on, unless you explicitly redefine it. Imported functions like <code>apply_rotary_pos_emb</code> follow the same rule.</p> <p data-svelte-h="svelte-1s6wrjk">For example, <code>OlmoDecoderLayer</code> has <code>self.mlp = OlmoMLP(config)</code>. <code>Olmo2MLP</code> was never defined in the modular file, so the linter creates it automatically, equivalent to using <code>pass</code>.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-py "><!-- HTML_TAG_START --><span class="hljs-keyword">from</span> ..olmo.modeling_olmo <span class="hljs-keyword">import</span> OlmoMLP | |
| <span class="hljs-keyword">class</span> <span class="hljs-title class_">Olmo2MLP</span>(<span class="hljs-title class_ inherited__">OlmoMLP</span>): | |
| <span class="hljs-keyword">pass</span><!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-b5vvgh">If you want <code>Olmo2MLP</code> to inherit from a different model instead, be explicit.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-py "><!-- HTML_TAG_START --><span class="hljs-comment"># switch to Mistral definition</span> | |
| <span class="hljs-keyword">from</span> ..mistral.modeling_mistral <span class="hljs-keyword">import</span> MistralMLP | |
| <span class="hljs-keyword">class</span> <span class="hljs-title class_">Olmo2MLP</span>(<span class="hljs-title class_ inherited__">MistralMLP</span>): | |
| <span class="hljs-keyword">pass</span><!-- HTML_TAG_END --></pre></div> <h3 class="relative group"><a id="finishing-the-file" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#finishing-the-file"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Finishing the file</span></h3> <p data-svelte-h="svelte-qd3d36">Every modular file must declare a <code>logger</code> and an <code>__all__</code> list at the module level.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-py "><!-- HTML_TAG_START -->logger = logging.get_logger(__name__) | |
| __all__ = [ | |
| <span class="hljs-string">"Olmo2Config"</span>, | |
| <span class="hljs-string">"Olmo2ForCausalLM"</span>, | |
| <span class="hljs-string">"Olmo2Model"</span>, | |
| <span class="hljs-string">"Olmo2PreTrainedModel"</span>, | |
| ]<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-juxip6"><code>__all__</code> must list every public class in the file. The converter and downstream imports depend on it. A class missing from <code>__all__</code> won’t be exported correctly.</p> <h2 class="relative group"><a id="generate-the-modeling-files" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#generate-the-modeling-files"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Generate the modeling files</span></h2> <p data-svelte-h="svelte-17xh6b1">The <code>modular_model_converter.py</code> script generates standalone <code>modeling.py</code>, <code>configuration.py</code>, and other files from your modular file. For each inherited class, it copies the parent body into the child, renames all references to match the new model, and pulls in any helper functions or classes those parents depend on.</p> <p data-svelte-h="svelte-52smj8">The output files contain no cross-model imports and no inheritance from other model directories. The linter flattens inheritance to a single level. If <code>Olmo2Attention</code> inherits from <code>OlmoAttention</code>, the generated <code>Olmo2Attention</code> is fully self-contained. But if <code>OlmoAttention</code> itself inherited from something else, the linter doesn’t inline that grandparent.</p> <p data-svelte-h="svelte-15tnuoy">Run the command below to generate files from a modular file.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-bash "><!-- HTML_TAG_START -->python utils/modular_model_converter.py your_model<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-o2u1os">Never edit the generated files directly because any changes will be overwritten on the next run.</p> <h2 class="relative group"><a id="patterns-for-modular-files" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#patterns-for-modular-files"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Patterns for modular files</span></h2> <p data-svelte-h="svelte-16zela2">The sections below document common usage patterns, such as removing attributes or overriding decorated methods, when working with a modular file.</p> <h3 class="relative group"><a id="removing-attributes" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#removing-attributes"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Removing attributes</span></h3> <p data-svelte-h="svelte-1i0czbv">Removing an inherited attribute depends on whether you’re working with a config class or an <code>nn.Module</code> subclass.</p> <p data-svelte-h="svelte-1eld04b">For a config class, assign <code>AttributeError()</code> to the attribute at the class level.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-py "><!-- HTML_TAG_START --><span class="hljs-keyword">class</span> <span class="hljs-title class_">MyNewConfig</span>(<span class="hljs-title class_ inherited__">ParentConfig</span>): | |
| removed_attr = AttributeError()<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-tdzysx">The linter removes the attribute declaration from the generated config file entirely. Config classes use a dataclass-style layout with no <code>__init__</code>, so assigning <code>AttributeError()</code> at the class level is the correct approach.</p> <p data-svelte-h="svelte-1klsbkq">For an <code>nn.Module</code> subclass, use <code>del self.attribute</code> after <code>super().__init__(...)</code>.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-py "><!-- HTML_TAG_START --><span class="hljs-keyword">class</span> <span class="hljs-title class_">MyNewModel</span>(<span class="hljs-title class_ inherited__">ParentModel</span>): | |
| <span class="hljs-keyword">def</span> <span class="hljs-title function_">__init__</span>(<span class="hljs-params">self, config: MyNewConfig</span>): | |
| <span class="hljs-built_in">super</span>().__init__(config) | |
| <span class="hljs-keyword">del</span> self.attribute<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-27hjag"><code>del self.attribute</code> removes only the <code>self.attribute = ...</code> assignment line from the copied parent body. It doesn’t remove any other lines that reference <code>self.attribute</code>. If the parent’s <code>forward</code> or other methods also reference the attribute, override those methods too.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-py "><!-- HTML_TAG_START --><span class="hljs-keyword">class</span> <span class="hljs-title class_">DummyModel</span>(nn.Module): | |
| <span class="hljs-keyword">def</span> <span class="hljs-title function_">__init__</span>(<span class="hljs-params">self, config: DummyConfig</span>): | |
| <span class="hljs-built_in">super</span>().__init__() | |
| self.attribute = config.attribute | |
| <span class="hljs-keyword">if</span> self.attribute: | |
| <span class="hljs-comment"># do more stuff with `self.attribute` here</span> | |
| ... | |
| <span class="hljs-keyword">class</span> <span class="hljs-title class_">MyNewDummyModel</span>(<span class="hljs-title class_ inherited__">DummyModel</span>): | |
| <span class="hljs-keyword">def</span> <span class="hljs-title function_">__init__</span>(<span class="hljs-params">self, config: MyNewDummyConfig</span>): | |
| <span class="hljs-built_in">super</span>().__init__(config) | |
| <span class="hljs-keyword">del</span> self.attribute | |
| <span class="hljs-comment"># 'self.attribute = config.attribute' is removed, but the 'if self.attribute:' block remains.</span> | |
| <span class="hljs-comment"># Override forward() or any other method that references self.attribute.</span><!-- HTML_TAG_END --></pre></div> <h3 class="relative group"><a id="working-with-super" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#working-with-super"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Working with super()</span></h3> <p data-svelte-h="svelte-sokvve"><code>super().__init__(config)</code> tells the converter to copy the parent body into the child. Two patterns let you override this behavior.</p> <ul data-svelte-h="svelte-dang2q"><li>Call a specific parent class directly when you need the generated output to call a grandparent (<code>nn.Module.__init__</code>) rather than the modular parent.</li> <li>Use <code>**super_kwargs</code> to inherit a parent method’s full signature while adding a custom docstring or swapping a decorator.</li></ul> <h4 class="relative group"><a id="call-a-grandparent-class-directly" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#call-a-grandparent-class-directly"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Call a grandparent class directly</span></h4> <p data-svelte-h="svelte-lsrcct">Be explicit about which class you’re calling when you need <code>super()</code> to target the generated class parent rather than the modular parent. The example below calls <code>nn.Module.__init__(self)</code> directly. <code>DummyModule</code> is itself an <code>nn.Module</code>, so the converter writes it as <code>super().__init__()</code> in the generated <code>MyNewDummyModule</code>.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-py "><!-- HTML_TAG_START --><span class="hljs-keyword">class</span> <span class="hljs-title class_">MyNewDummyModule</span>(<span class="hljs-title class_ inherited__">DummyModule</span>): | <span class="hljs-keyword">class</span> <span class="hljs-title class_">MyNewDummyModule</span>(nn.Module): | |
| | | |
| <span class="hljs-keyword">def</span> <span class="hljs-title function_">__init__</span>(<span class="hljs-params">self</span>): | <span class="hljs-keyword">def</span> <span class="hljs-title function_">__init__</span>(<span class="hljs-params">self</span>): | |
| nn.Module.__init__(self) | <span class="hljs-built_in">super</span>().__init__() | |
| self.foo = config.foo | self.foo = config.foo | |
| ... | ...<!-- HTML_TAG_END --></pre></div> <h4 class="relative group"><a id="superkwargs" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#superkwargs"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>super_kwargs</span></h4> <p data-svelte-h="svelte-qiq8gp">Use <code>**super_kwargs</code> to inherit a parent method’s full signature while adding a custom docstring or swapping a decorator. In the overridden signature, it tells the linter to expand all parent arguments in the generated output.</p> <p data-svelte-h="svelte-1e1iu38">The most common use is adding a model-specific docstring, like documenting the <code>labels</code> argument, without rewriting the full signature.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-py "><!-- HTML_TAG_START --><span class="hljs-comment"># modular_gemma.py</span> | |
| <span class="hljs-keyword">class</span> <span class="hljs-title class_">GemmaForCausalLM</span>(<span class="hljs-title class_ inherited__">LlamaForCausalLM</span>): | |
| <span class="hljs-keyword">def</span> <span class="hljs-title function_">forward</span>(<span class="hljs-params">**super_kwargs</span>): | |
| <span class="hljs-string">r""" | |
| Example: | |
| ```python | |
| >>> from transformers import AutoTokenizer, GemmaForCausalLM | |
| >>> model = GemmaForCausalLM.from_pretrained("google/gemma-7b") | |
| ... | |
| ```"""</span> | |
| <span class="hljs-keyword">return</span> <span class="hljs-built_in">super</span>().forward(**super_kwargs)<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-qgs0dy">The generated <code>GemmaForCausalLM.forward</code> has the full <code>LlamaForCausalLM</code> signature with no manual copying needed.</p> <p data-svelte-h="svelte-1w4v5sw"><code>**super_kwargs</code> is a shortcut for niche cases. If you’re changing behavior, write the full signature instead.</p> <h3 class="relative group"><a id="deleting-unused-methods" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#deleting-unused-methods"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Deleting unused methods</span></h3> <p data-svelte-h="svelte-1qdb3yt">Remove a parent method by overriding it with a <code>raise AttributeError("")</code> statement. The linter removes the method from the generated file.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-py "><!-- HTML_TAG_START --><span class="hljs-keyword">class</span> <span class="hljs-title class_">GemmaTokenizer</span>(<span class="hljs-title class_ inherited__">LlamaTokenizer</span>): | |
| ... | |
| <span class="hljs-keyword">def</span> <span class="hljs-title function_">get_spm_processor</span>(<span class="hljs-params">self</span>): | |
| <span class="hljs-keyword">raise</span> AttributeError(<span class="hljs-string">"Not needed for Gemma"</span>) | |
| <span class="hljs-keyword">def</span> <span class="hljs-title function_">unk_token_length</span>(<span class="hljs-params">self</span>): | |
| <span class="hljs-keyword">raise</span> AttributeError(<span class="hljs-string">"Not needed for Gemma"</span>)<!-- HTML_TAG_END --></pre></div> <h3 class="relative group"><a id="overriding-decorated-methods" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#overriding-decorated-methods"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Overriding decorated methods</span></h3> <p data-svelte-h="svelte-21ok5k">When you override a decorated parent method, the parent’s decorator carries over automatically. If you add your own decorator, it replaces the parent’s.</p> <p data-svelte-h="svelte-1lcsrav">Two decorators appear throughout the library, one for <a href="./model_output_tracing">capturing model intermediate outputs</a> and one for <a href="./auto_docstring">auto-generating docstrings</a>.</p> <p data-svelte-h="svelte-1p6i709">In the example below, a subclass overrides a decorated parent method without adding its own decorator. The parent’s decorator carries over.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-py "><!-- HTML_TAG_START --><span class="hljs-keyword">class</span> <span class="hljs-title class_">NewModel</span>(<span class="hljs-title class_ inherited__">DummyModel</span>): | <span class="hljs-keyword">class</span> <span class="hljs-title class_">NewModel</span>(nn.Module): | |
| ... | ... | |
| | | |
| <span class="hljs-keyword">def</span> <span class="hljs-title function_">forward</span>(<span class="hljs-params">...</span>): | @decorator(...) | |
| ... | <span class="hljs-keyword">def</span> <span class="hljs-title function_">forward</span>(<span class="hljs-params">...</span>): | |
| | ...<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-1jl7653">If you add a new decorator, your decorator replaces the parent’s.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-py "><!-- HTML_TAG_START --><span class="hljs-keyword">class</span> <span class="hljs-title class_">NewModel</span>(<span class="hljs-title class_ inherited__">DummyModel</span>): | <span class="hljs-keyword">class</span> <span class="hljs-title class_">NewModel</span>(nn.Module): | |
| ... | ... | |
| | | |
| <span class="hljs-meta"> @my_new_decorator(<span class="hljs-params">...</span>) | @my_new_decorator(<span class="hljs-params">...</span>)</span> | |
| <span class="hljs-keyword">def</span> <span class="hljs-title function_">forward</span>(<span class="hljs-params">...</span>): | <span class="hljs-keyword">def</span> <span class="hljs-title function_">forward</span>(<span class="hljs-params">...</span>): | |
| ... | ...<!-- HTML_TAG_END --></pre></div> <h3 class="relative group"><a id="special-naming" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#special-naming"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Special naming</span></h3> <p data-svelte-h="svelte-mqlf69">The linter automatically renames everything when inheriting from a class. Use the same class name prefix across all classes in the same file.</p> <p data-svelte-h="svelte-re4mwy">Avoid mixing prefixes like in the example below. <code>MyModelIncredibleMLP</code> breaks naming conventions, and the linter won’t know whether to use <code>MyModelIncredible</code> or <code>MyModel</code> when renaming higher-order dependencies.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-py "><!-- HTML_TAG_START --><span class="hljs-keyword">class</span> <span class="hljs-title class_">MyModelIncredibleMLP</span>(<span class="hljs-title class_ inherited__">LlamaMLP</span>): | |
| ... | |
| <span class="hljs-keyword">class</span> <span class="hljs-title class_">MyModelDecoderLayer</span>(<span class="hljs-title class_ inherited__">LlamaDecoderLayer</span>): | |
| ...<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-l688ur">With no <a href="#other-classes">implicit dependencies</a>, you can rename a single class locally. Explicitly redefine every other mention of that class with the new name pattern. Otherwise, the linter adds an unwanted <code>MyModelMLP</code> class alongside <code>MyModelIncredibleMLP</code>.</p> <p data-svelte-h="svelte-42nacl">The linter raises a warning when it detects an ambiguous prefix.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-text "><!-- HTML_TAG_START -->We detected multiple prefix names when inheriting from transformers.models.llama.modeling_llama: ('Emu3Text', 'Emu3'). We will only use the most used 'Emu3' prefix when grabbing args and dependencies. Make sure to subclass the intermediate classes with the prefix you want (if different from 'Emu3') or use a single prefix in all the modular (best).<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-3m7a91">Ambiguous prefixes are most common in multimodal models where class names include a modality qualifier like <code>Text</code>. To give a dependency a specific prefix, explicitly rename it with a <code>pass</code>.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-py "><!-- HTML_TAG_START --><span class="hljs-keyword">class</span> <span class="hljs-title class_">Emu3TextMLP</span>(<span class="hljs-title class_ inherited__">LlamaMLP</span>): | |
| <span class="hljs-keyword">pass</span><!-- HTML_TAG_END --></pre></div> <h3 class="relative group"><a id="config-docstrings" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#config-docstrings"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Config docstrings</span></h3> <p data-svelte-h="svelte-qch5sw">The linter doesn’t support partial docstring inheritance yet. When adding or removing config attributes, add the full docstring directly in the modular file under the class definition.</p> <h2 class="relative group"><a id="checkpoint-conversion" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#checkpoint-conversion"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Checkpoint conversion</span></h2> <p data-svelte-h="svelte-kxb6bh">Once you’ve generated your modeling files, verify that real weights load correctly. Write a conversion script to translate the upstream checkpoint format into a Transformers-compatible one, then save it to the Hub.</p> <h3 class="relative group"><a id="write-a-conversion-script" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#write-a-conversion-script"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Write a conversion script</span></h3> <p data-svelte-h="svelte-1ltd09j">Add a <code>convert_<model>_to_hf.py</code> file to <code>src/transformers/models/<model>/</code>. The script loads the upstream weights, renames and reshapes keys to match your module’s parameter names, and saves the result with <a href="/docs/transformers/pr_43838/en/main_classes/model#transformers.PreTrainedModel.save_pretrained">save_pretrained()</a>.</p> <blockquote class="tip" data-svelte-h="svelte-pbw7je"><p>Look for an existing script to copy and adapt. Models under <code>src/transformers/models/</code> include a <code>convert_*_to_hf.py</code> you can use as a starting point.</p></blockquote> <p data-svelte-h="svelte-isxgjz">After running the script, load the saved checkpoint with <a href="/docs/transformers/pr_43838/en/main_classes/model#transformers.PreTrainedModel.from_pretrained">from_pretrained()</a> and confirm every expected weight loaded correctly. Unused checkpoint keys indicate mismatched names, so print them to catch problems early.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-py "><!-- HTML_TAG_START -->model = YourModelForTask.from_pretrained(<span class="hljs-string">"path/to/output/"</span>)<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-1jkbetc">Check shape and name matches when iterating over keys. Shape mismatches typically mean a parameter in your config is wrong, the architecture differs from the original, or a weight needs to be transposed.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-py "><!-- HTML_TAG_START --><span class="hljs-keyword">for</span> key, tensor <span class="hljs-keyword">in</span> original_state_dict.items(): | |
| hf_tensor = hf_model.state_dict().get(mapped_key) | |
| <span class="hljs-keyword">assert</span> hf_tensor.shape == tensor.shape, ( | |
| <span class="hljs-string">f"Shape mismatch for <span class="hljs-subst">{key}</span>: expected <span class="hljs-subst">{tensor.shape}</span>, got <span class="hljs-subst">{hf_tensor.shape}</span>"</span> | |
| )<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-12pxdag">Fix any issues by iterating between your modular file, the generated modeling file, and the conversion script until all weights load cleanly.</p> <p data-svelte-h="svelte-1itjzhn">Once the checkpoint loads cleanly, push it to the Hub using <a href="/docs/transformers/pr_43838/en/main_classes/model#transformers.utils.PushToHubMixin.push_to_hub">push_to_hub()</a>. Refer to the <a href="./model_sharing">model sharing</a> guide for more details.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-py "><!-- HTML_TAG_START -->model.push_to_hub(<span class="hljs-string">"username/your-model-name"</span>)<!-- HTML_TAG_END --></pre></div> <h3 class="relative group"><a id="runtime-conversion-mapping" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#runtime-conversion-mapping"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Runtime conversion mapping</span></h3> <p data-svelte-h="svelte-fg8b4e">Add a runtime mapping to <code>src/transformers/conversion_mapping.py</code> when the published weights don’t match your module’s parameter layout. Common cases include fused weights stored separately and MoE expert tensors that need stacking. The mapping lets <a href="/docs/transformers/pr_43838/en/main_classes/model#transformers.PreTrainedModel.from_pretrained">from_pretrained()</a> load the Hub checkpoint without a separate export step.</p> <p data-svelte-h="svelte-120n6ne">Refer to the <a href="./weightconverter">dynamic weight loading</a> guide for how to write <code>WeightRenaming</code> and <a href="/docs/transformers/pr_43838/en/internal/modeling_utils#transformers.WeightConverter">WeightConverter</a> rules and register them for your <code>model_type</code>.</p> <h2 class="relative group"><a id="next-steps" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#next-steps"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Next steps</span></h2> <ul data-svelte-h="svelte-1cf99oj"><li><a href="./modeling_rules">Model structure rules</a> are static rules enforced on all <code>modeling_*.py</code>, <code>modular_*.py</code>, and <code>configuration_*.py</code> files. Run <code>make typing</code> to check them before opening a PR.</li> <li><a href="./add_vision_processing_components">Add vision processing components</a> walks through adding an image processor, video processor, and processor for a multimodal model.</li> <li><a href="./auto_docstring">Auto-generating docstrings</a> shows how to use <code>@auto_docstring</code> so you don’t have to hand-write argument docs for shared model APIs.</li> <li><a href="./testing">Writing model tests</a> covers how to write integration tests for your new model and run it locally.</li> <li><a href="./pr_checks">Pull request checks</a> explains the CI checks your PR has to pass before it can be merged, and how to reproduce and fix them locally.</li></ul> <a class="!text-gray-400 !no-underline text-sm flex items-center not-prose mt-4" href="https://github.com/huggingface/transformers/blob/main/docs/source/en/modular_transformers.md" target="_blank"><svg class="mr-1" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M31,16l-7,7l-1.41-1.41L28.17,16l-5.58-5.59L24,9l7,7z"></path><path d="M1,16l7-7l1.41,1.41L3.83,16l5.58,5.59L8,23l-7-7z"></path><path d="M12.419,25.484L17.639,6.552l1.932,0.518L14.351,26.002z"></path></svg> <span data-svelte-h="svelte-zjs2n5"><span class="underline">Update</span> on GitHub</span></a> <p></p> | |
| <script> | |
| { | |
| __sveltekit_1o1ef6h = { | |
| assets: "/docs/transformers/pr_43838/en", | |
| base: "/docs/transformers/pr_43838/en", | |
| env: {} | |
| }; | |
| const element = document.currentScript.parentElement; | |
| const data = [null,null]; | |
| Promise.all([ | |
| import("/docs/transformers/pr_43838/en/_app/immutable/entry/start.1bcc8411.js"), | |
| import("/docs/transformers/pr_43838/en/_app/immutable/entry/app.9d434a07.js") | |
| ]).then(([kit, app]) => { | |
| kit.start(app, element, { | |
| node_ids: [0, 610], | |
| data, | |
| form: null, | |
| error: null | |
| }); | |
| }); | |
| } | |
| </script> | |
Xet Storage Details
- Size:
- 108 kB
- Xet hash:
- b46608673df97f7fb8ffbbf2bce45e538d4d17d4f0e29b95a7f8cc39c647910d
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.