transformers/pr_40546/en/heterogeneous_configurations.html

<meta charset="utf-8" /><meta name="hf:doc:metadata" content="{&quot;title&quot;:&quot;Heterogeneous model configurations&quot;,&quot;local&quot;:&quot;heterogeneous-model-configurations&quot;,&quot;sections&quot;:[{&quot;title&quot;:&quot;Define per-layer overrides&quot;,&quot;local&quot;:&quot;define-per-layer-overrides&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Global attribute access&quot;,&quot;local&quot;:&quot;global-attribute-access&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Serialization&quot;,&quot;local&quot;:&quot;serialization&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2}],&quot;depth&quot;:1}"/>
		<link href="/docs/transformers/main/en/_app/immutable/entry/start.DdJkiArM.js" rel="modulepreload">
		<link href="/docs/transformers/main/en/_app/immutable/chunks/DREL0biP.js" rel="modulepreload">
		<link href="/docs/transformers/main/en/_app/immutable/chunks/Q6Z1sYnD.js" rel="modulepreload">
		<link href="/docs/transformers/main/en/_app/immutable/entry/app.Bx8MAX1s.js" rel="modulepreload">
		<link href="/docs/transformers/main/en/_app/immutable/chunks/Cj1VzlLG.js" rel="modulepreload">
		<link href="/docs/transformers/main/en/_app/immutable/chunks/BRRD8tqb.js" rel="modulepreload">
		<link href="/docs/transformers/main/en/_app/immutable/chunks/DsnmJJEf.js" rel="modulepreload">
		<link href="/docs/transformers/main/en/_app/immutable/chunks/CkyaH76N.js" rel="modulepreload">
		<link href="/docs/transformers/main/en/_app/immutable/chunks/Cku66UBC.js" rel="modulepreload">
		<link href="/docs/transformers/main/en/_app/immutable/nodes/0.BzNzWC0H.js" rel="modulepreload">
		<link href="/docs/transformers/main/en/_app/immutable/chunks/957ErxVI.js" rel="modulepreload">
		<link href="/docs/transformers/main/en/_app/immutable/nodes/2.BDO4BcOu.js" rel="modulepreload">
		<link href="/docs/transformers/main/en/_app/immutable/chunks/MFkQE-9Z.js" rel="modulepreload">
		<!--pkalzc--><meta name="hf:doc:metadata" content="{&quot;title&quot;:&quot;Heterogeneous model configurations&quot;,&quot;local&quot;:&quot;heterogeneous-model-configurations&quot;,&quot;sections&quot;:[{&quot;title&quot;:&quot;Define per-layer overrides&quot;,&quot;local&quot;:&quot;define-per-layer-overrides&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Global attribute access&quot;,&quot;local&quot;:&quot;global-attribute-access&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Serialization&quot;,&quot;local&quot;:&quot;serialization&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2}],&quot;depth&quot;:1}"/><!---->
		<link href="/docs/transformers/main/en/_app/immutable/assets/0.tn0RQdqM.css" rel="modulepreload"> <!--[--><!--[0--><!--[--><!--[0--><!--[--><p></p> <div class="items-center shrink-0 min-w-[100px] max-sm:min-w-[50px] justify-end ml-auto flex" style="float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"><div class="inline-flex rounded-md max-sm:rounded-sm"><button class="inline-flex items-center gap-1 h-7 max-sm:h-7 px-2 max-sm:px-1.5 text-sm font-medium text-gray-800 border border-r-0 rounded-l-md max-sm:rounded-l-sm border-gray-200 bg-white hover:shadow-inner dark:border-gray-850 dark:bg-gray-950 dark:text-gray-200 dark:hover:bg-gray-800" aria-live="polite"><span class="inline-flex items-center justify-center rounded-md p-0.5 max-sm:p-0 hover:text-gray-800 dark:hover:text-gray-200"><svg class="sm:size-3.5 size-3" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----></span> <span>Copy page</span></button> <button class="inline-flex items-center justify-center w-6 max-sm:w-5 h-7 max-sm:h-7 disabled:pointer-events-none text-sm text-gray-500 hover:text-gray-700 dark:hover:text-white rounded-r-md max-sm:rounded-r-sm border border-l transition border-gray-200 bg-white hover:shadow-inner dark:border-gray-850 dark:bg-gray-950 dark:text-gray-200 dark:hover:bg-gray-800" aria-haspopup="menu" aria-expanded="false" aria-label="Open copy menu"><svg class="transition-transform text-gray-400 overflow-visible sm:size-3.5 size-3 rotate-0" width="1em" height="1em" viewBox="0 0 12 7" fill="none" xmlns="http://www.w3.org/2000/svg"><path d="M1 1L6 6L11 1" stroke="currentColor"></path></svg><!----></button></div> <!--[-1--><!--]--></div><!----> <!--[0--><h1 class="relative group"><a id="heterogeneous-model-configurations" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#heterogeneous-model-configurations"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>Heterogeneous model configurations</span></h1><!--]--><!----> <p>Most model configurations in Transformers describe a homogeneous stack: each layer has the same dimensions and contains
the same submodules. Some checkpoints do not follow this pattern. For example, a model may use a smaller MLP in one
layer, fewer key-value heads in another layer, or omit a submodule such as attention or the MLP from selected layers.</p> <p><code>per_layer_config</code> represents these layer-specific differences directly in the model configuration. Each entry stores
only the attributes that differ from the global configuration. Attributes that are not overridden inherit their value
from the global configuration.</p> <p>This is useful for checkpoints that remain close to an existing architecture but are no longer layer-uniform, such as
pruned, distilled, or NAS-derived (Neural Architecture Search) models. Instead of defining a new architecture for every
such variant, <code>per_layer_config</code> records the layer-level differences in a few lines of config, at little to no
config-side cost.</p> <blockquote class="note"><p>Heterogeneous configurations are a power feature. If a heterogeneous layout becomes a common or prominent
architecture, we will strive to model it explicitly in the architecture implementation rather than rely on <code>per_layer_config</code>. Prefer the explicit architecture when one exists.</p></blockquote> <p>Examples of heterogeneous checkpoints include:</p> <table><thead><tr><th>Model</th><th>Derived from</th></tr></thead><tbody><tr><td><a href="https://huggingface.co/nvidia/Llama-3_3-Nemotron-Super-49B-v1_5" rel="nofollow"><code>nvidia/Llama-3_3-Nemotron-Super-49B-v1_5</code></a></td><td><a href="https://huggingface.co/meta-llama/Llama-3.3-70B-Instruct" rel="nofollow"><code>meta-llama/Llama-3.3-70B-Instruct</code></a></td></tr><tr><td><a href="https://huggingface.co/nvidia/Llama-3_1-Nemotron-Ultra-253B-v1" rel="nofollow"><code>nvidia/Llama-3_1-Nemotron-Ultra-253B-v1</code></a></td><td><a href="https://huggingface.co/meta-llama/Llama-3.1-405B-Instruct" rel="nofollow"><code>meta-llama/Llama-3.1-405B-Instruct</code></a></td></tr><tr><td><a href="https://huggingface.co/nvidia/gpt-oss-puzzle-88B" rel="nofollow"><code>nvidia/gpt-oss-puzzle-88B</code></a></td><td><a href="https://huggingface.co/openai/gpt-oss-120b" rel="nofollow"><code>openai/gpt-oss-120b</code></a></td></tr><tr><td><a href="https://huggingface.co/nvidia/NVIDIA-Nemotron-Labs-3-Puzzle-75B-A9B-BF16" rel="nofollow"><code>nvidia/NVIDIA-Nemotron-Labs-3-Puzzle-75B-A9B-BF16</code></a></td><td><a href="https://huggingface.co/nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16" rel="nofollow"><code>nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16</code></a></td></tr></tbody></table> <!--[1--><h2 class="relative group"><a id="define-per-layer-overrides" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#define-per-layer-overrides"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>Define per-layer overrides</span></h2><!--]--><!----> <p>Pass <code>per_layer_config</code> to a configuration as a mapping from layer indices to attribute overrides. Layer indices are
zero-based. Only attributes that differ from the global configuration need to be specified.</p> <p>The following example overrides four layers: layer 5 uses a smaller MLP, layer 11 uses
fewer key-value heads, layer 23 skips the MLP, and layer 27 skips attention.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5   text-gray-600  " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-py "><!----><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> LlamaConfig


config = LlamaConfig(
    hidden_size=<span class="hljs-number">4096</span>,
    intermediate_size=<span class="hljs-number">14336</span>,
    num_hidden_layers=<span class="hljs-number">32</span>,
    num_attention_heads=<span class="hljs-number">32</span>,
    num_key_value_heads=<span class="hljs-number">8</span>,
    per_layer_config={
        <span class="hljs-comment"># Use a smaller MLP in one layer.</span>
        <span class="hljs-number">5</span>: {<span class="hljs-string">&quot;intermediate_size&quot;</span>: <span class="hljs-number">8192</span>},

        <span class="hljs-comment"># Use fewer key-value heads in another layer.</span>
        <span class="hljs-number">11</span>: {<span class="hljs-string">&quot;num_key_value_heads&quot;</span>: <span class="hljs-number">4</span>},

        <span class="hljs-comment"># Omit the MLP from a selected layer.</span>
        <span class="hljs-number">23</span>: {<span class="hljs-string">&quot;skip&quot;</span>: [<span class="hljs-string">&quot;mlp&quot;</span>]},

        <span class="hljs-comment"># Omit attention from a selected layer.</span>
        <span class="hljs-number">27</span>: {<span class="hljs-string">&quot;skip&quot;</span>: [<span class="hljs-string">&quot;attention&quot;</span>]},
    },
)<!----></pre></div><!----> <p>The submodules that can be skipped (for example, <code>"mlp"</code> and <code>"attention"</code>) are defined per architecture. <code>skip</code> accepts a list, so a layer can omit more than one submodule.</p> <p>Accessing <code>config.per_layer_config[layer_idx]</code> returns a resolved layer configuration. The resolved configuration
combines the global configuration with the overrides for that layer.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5   text-gray-600  " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-py "><!----><span class="hljs-comment"># Layer 0 does not define overrides, so it inherits the global values.</span>
config.per_layer_config[<span class="hljs-number">0</span>].intermediate_size
<span class="hljs-comment"># 14336</span>

config.per_layer_config[<span class="hljs-number">0</span>].num_key_value_heads
<span class="hljs-comment"># 8</span>

<span class="hljs-comment"># Layer 5 overrides the MLP intermediate size.</span>
config.per_layer_config[<span class="hljs-number">5</span>].intermediate_size
<span class="hljs-comment"># 8192</span>

<span class="hljs-comment"># Layer 11 overrides the number of key-value heads.</span>
config.per_layer_config[<span class="hljs-number">11</span>].num_key_value_heads
<span class="hljs-comment"># 4</span>

<span class="hljs-comment"># Layer 23 skips the MLP.</span>
config.per_layer_config[<span class="hljs-number">23</span>].skip
<span class="hljs-comment"># [&quot;mlp&quot;]</span>

<span class="hljs-comment"># Layer 27 skips attention.</span>
config.per_layer_config[<span class="hljs-number">27</span>].skip
<span class="hljs-comment"># [&quot;attention&quot;]</span><!----></pre></div><!----> <p>Configurations that use <code>per_layer_config</code> support the same <a href="/docs/transformers/pr_40546/en/main_classes/configuration#transformers.PreTrainedConfig.save_pretrained">save_pretrained()</a> and <a href="/docs/transformers/pr_40546/en/main_classes/configuration#transformers.PreTrainedConfig.from_pretrained">from_pretrained()</a> round trip as other configurations.</p> <p>Each architecture defines in its code which attributes are used at the layer level. <code>per_layer_config</code> provides the mechanism for
recording those layer-level differences and resolving them against the global config.</p> <!--[1--><h2 class="relative group"><a id="global-attribute-access" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#global-attribute-access"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>Global attribute access</span></h2><!--]--><!----> <p>In a heterogeneous configuration, an attribute with per-layer overrides no longer has a single model-wide value.
For example, <code>num_key_value_heads</code> may be <code>8</code> for most layers and <code>4</code> for selected layers, so reading <code>config.num_key_value_heads</code> outside a layer-specific context is not well-defined.</p> <p>This matters because consumers that read such an attribute globally would silently apply the wrong value to the
overridden layers. Code that allocates a key-value cache from a global <code>num_key_value_heads</code>, for instance,
would be incorrect for the layers that override it.</p> <p>By default, an <code>AmbiguousGlobalPerLayerAttributeError</code> will be raised for this access pattern, directing callers to use <code>config.per_layer_config[layer_idx]</code> instead. We raise this error instead of <code>AttributeError</code> because the attribute
exists on the global config, but reading it there is ambiguous without layer-specific context.</p> <p>Set <code>allow_global_per_layer_attribute_access=True</code> only when the caller intentionally needs the global fallback value
and can safely handle heterogeneous configurations. In that case, global access is allowed, but a warning will be emitted once.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5   text-gray-600  " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-py "><!---->config = LlamaConfig(
    hidden_size=<span class="hljs-number">4096</span>,
    intermediate_size=<span class="hljs-number">14336</span>,
    num_hidden_layers=<span class="hljs-number">32</span>,
    num_attention_heads=<span class="hljs-number">32</span>,
    num_key_value_heads=<span class="hljs-number">8</span>,
    allow_global_per_layer_attribute_access=<span class="hljs-literal">True</span>,
    per_layer_config={
        <span class="hljs-number">11</span>: {<span class="hljs-string">&quot;num_key_value_heads&quot;</span>: <span class="hljs-number">4</span>},
    },
)

config.num_key_value_heads
<span class="hljs-comment"># 8</span>
<span class="hljs-comment"># Emits a warning_once message because num_key_value_heads has a per-layer override.</span><!----></pre></div><!----> <!--[1--><h2 class="relative group"><a id="serialization" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#serialization"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>Serialization</span></h2><!--]--><!----> <p><code>per_layer_config</code> is serialized sparsely by default. Layers without overrides are omitted, and overridden attributes
that match the global value are also omitted.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5   text-gray-600  " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-py "><!----><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> LlamaConfig


config = LlamaConfig(
    hidden_size=<span class="hljs-number">4096</span>,
    intermediate_size=<span class="hljs-number">14336</span>,
    num_hidden_layers=<span class="hljs-number">4</span>,
    num_attention_heads=<span class="hljs-number">32</span>,
    num_key_value_heads=<span class="hljs-number">8</span>,
    per_layer_config={
        <span class="hljs-number">0</span>: {<span class="hljs-string">&quot;num_key_value_heads&quot;</span>: <span class="hljs-number">8</span>},
        <span class="hljs-number">2</span>: {<span class="hljs-string">&quot;num_key_value_heads&quot;</span>: <span class="hljs-number">4</span>},
    },
)

config.to_dict()[<span class="hljs-string">&quot;per_layer_config&quot;</span>]
<span class="hljs-comment"># {&quot;2&quot;: {&quot;num_key_value_heads&quot;: 4}}</span><!----></pre></div><!----> <p>Set <code>serialize_explicit_per_layer_config=True</code> when the serialized configuration should include every layer for the
attributes represented in <code>per_layer_config</code>. This can make the layer layout easier to inspect, even when some values
match the global configuration.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5   text-gray-600  " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-py "><!---->explicit_config = LlamaConfig(
    hidden_size=<span class="hljs-number">4096</span>,
    intermediate_size=<span class="hljs-number">14336</span>,
    num_hidden_layers=<span class="hljs-number">4</span>,
    num_attention_heads=<span class="hljs-number">32</span>,
    num_key_value_heads=<span class="hljs-number">8</span>,
    serialize_explicit_per_layer_config=<span class="hljs-literal">True</span>,
    per_layer_config={
        <span class="hljs-number">0</span>: {<span class="hljs-string">&quot;num_key_value_heads&quot;</span>: <span class="hljs-number">8</span>},
        <span class="hljs-number">2</span>: {<span class="hljs-string">&quot;num_key_value_heads&quot;</span>: <span class="hljs-number">4</span>},
    },
)

serialized_per_layer_config = explicit_config.to_dict()[<span class="hljs-string">&quot;per_layer_config&quot;</span>]

serialized_per_layer_config
<span class="hljs-comment"># {</span>
<span class="hljs-comment">#     &quot;0&quot;: {&quot;num_key_value_heads&quot;: 8},</span>
<span class="hljs-comment">#     &quot;1&quot;: {&quot;num_key_value_heads&quot;: 8},</span>
<span class="hljs-comment">#     &quot;2&quot;: {&quot;num_key_value_heads&quot;: 4},</span>
<span class="hljs-comment">#     &quot;3&quot;: {&quot;num_key_value_heads&quot;: 8},</span>
<span class="hljs-comment"># }</span><!----></pre></div><!----> <p>Use sparse serialization for compact configs. Use explicit serialization when readability or downstream tooling benefits
from seeing the full per-layer layout.</p> <a class="!text-gray-400 !no-underline text-sm flex items-center not-prose mt-4" href="https://github.com/huggingface/transformers/blob/main/docs/source/en/heterogeneous_configurations.md" target="_blank"><svg class="mr-1" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M31,16l-7,7l-1.41-1.41L28.17,16l-5.58-5.59L24,9l7,7z"></path><path d="M1,16l7-7l1.41,1.41L3.83,16l5.58,5.59L8,23l-7-7z"></path><path d="M12.419,25.484L17.639,6.552l1.932,0.518L14.351,26.002z"></path></svg><!----> <span><span class="underline">Update</span> on GitHub</span></a><!----> <p></p><!--]--><!----><!--]--><!--]--><!--]--> <!--[-1--><!--]--><!--]-->
			
			<script>
				{
					__sveltekit_1atfz6d = {
						base: "/docs/transformers/main/en",
						assets: "/docs/transformers/main/en"
					};

					const element = document.currentScript.parentElement;

					Promise.all([
						import("/docs/transformers/main/en/_app/immutable/entry/start.DdJkiArM.js"),
						import("/docs/transformers/main/en/_app/immutable/entry/app.Bx8MAX1s.js")
					]).then(([kit, app]) => {
						kit.start(app, element, {
							node_ids: [0, 2],
							data: [null,null],
							form: null,
							error: null
						});
					});
				}
			</script>