Buckets:

hf-doc-build
/

doc-dev

Files

xet

hf-doc-build/doc-dev / peft /main /en /developer_guides /troubleshooting.html

rtrm

about 1 month ago

download

raw

54.2 kB

	<meta charset="utf-8" /><meta name="hf:doc:metadata" content="{"title":"Troubleshooting","local":"troubleshooting","sections":[{"title":"Examples don’t work","local":"examples-dont-work","sections":[],"depth":2},{"title":"ValueError: Attempting to unscale FP16 gradients","local":"valueerror-attempting-to-unscale-fp16-gradients","sections":[],"depth":2},{"title":"Bad results from a loaded PEFT model","local":"bad-results-from-a-loaded-peft-model","sections":[{"title":"Random deviations","local":"random-deviations","sections":[],"depth":3},{"title":"Incorrectly loaded model","local":"incorrectly-loaded-model","sections":[],"depth":3},{"title":"Randomly initialized layers","local":"randomly-initialized-layers","sections":[],"depth":3},{"title":"Extending the vocabulary","local":"extending-the-vocabulary","sections":[],"depth":3},{"title":"Check layer and model status","local":"check-layer-and-model-status","sections":[],"depth":3}],"depth":2},{"title":"Reproducibility","local":"reproducibility","sections":[{"title":"Models using batch norm","local":"models-using-batch-norm","sections":[],"depth":3}],"depth":2}],"depth":1}">
	<link href="/docs/peft/main/en/_app/immutable/assets/0.e3b0c442.css" rel="modulepreload">
	<link rel="modulepreload" href="/docs/peft/main/en/_app/immutable/entry/start.b1913684.js">
	<link rel="modulepreload" href="/docs/peft/main/en/_app/immutable/chunks/scheduler.c57aa7ef.js">
	<link rel="modulepreload" href="/docs/peft/main/en/_app/immutable/chunks/singletons.6b6c8b9d.js">
	<link rel="modulepreload" href="/docs/peft/main/en/_app/immutable/chunks/index.c4ac7298.js">
	<link rel="modulepreload" href="/docs/peft/main/en/_app/immutable/chunks/paths.62c814b0.js">
	<link rel="modulepreload" href="/docs/peft/main/en/_app/immutable/entry/app.ca8eed50.js">
	<link rel="modulepreload" href="/docs/peft/main/en/_app/immutable/chunks/index.c50cb18e.js">
	<link rel="modulepreload" href="/docs/peft/main/en/_app/immutable/nodes/0.f0f7697b.js">
	<link rel="modulepreload" href="/docs/peft/main/en/_app/immutable/chunks/each.e59479a4.js">
	<link rel="modulepreload" href="/docs/peft/main/en/_app/immutable/nodes/17.9d12bfc6.js">
	<link rel="modulepreload" href="/docs/peft/main/en/_app/immutable/chunks/Tip.9268b0ca.js">
	<link rel="modulepreload" href="/docs/peft/main/en/_app/immutable/chunks/CodeBlock.34f0a53d.js">
	<link rel="modulepreload" href="/docs/peft/main/en/_app/immutable/chunks/EditOnGithub.958a8a49.js"><!-- HEAD_svelte-u9bgzb_START --><meta name="hf:doc:metadata" content="{"title":"Troubleshooting","local":"troubleshooting","sections":[{"title":"Examples don’t work","local":"examples-dont-work","sections":[],"depth":2},{"title":"ValueError: Attempting to unscale FP16 gradients","local":"valueerror-attempting-to-unscale-fp16-gradients","sections":[],"depth":2},{"title":"Bad results from a loaded PEFT model","local":"bad-results-from-a-loaded-peft-model","sections":[{"title":"Random deviations","local":"random-deviations","sections":[],"depth":3},{"title":"Incorrectly loaded model","local":"incorrectly-loaded-model","sections":[],"depth":3},{"title":"Randomly initialized layers","local":"randomly-initialized-layers","sections":[],"depth":3},{"title":"Extending the vocabulary","local":"extending-the-vocabulary","sections":[],"depth":3},{"title":"Check layer and model status","local":"check-layer-and-model-status","sections":[],"depth":3}],"depth":2},{"title":"Reproducibility","local":"reproducibility","sections":[{"title":"Models using batch norm","local":"models-using-batch-norm","sections":[],"depth":3}],"depth":2}],"depth":1}"><!-- HEAD_svelte-u9bgzb_END --> <p></p> <h1 class="relative group"><a id="troubleshooting" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#troubleshooting"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Troubleshooting</span></h1> <p data-svelte-h="svelte-wwms7a">If you encounter any issue when using PEFT, please check the following list of common issues and their solutions.</p> <h2 class="relative group"><a id="examples-dont-work" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#examples-dont-work"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Examples don’t work</span></h2> <p data-svelte-h="svelte-18nlz3r">Examples often rely on the most recent package versions, so please ensure they’re up-to-date. In particular, check the following package versions:</p> <ul data-svelte-h="svelte-ccv654"><li><code>peft</code></li> <li><code>transformers</code></li> <li><code>accelerate</code></li> <li><code>torch</code></li></ul> <p data-svelte-h="svelte-1cuqjyb">In general, you can update the package version by running this command inside your Python environment:</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->python -m pip install -U <package_name><!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-h5ykrw">Installing PEFT from source is useful for keeping up with the latest developments:</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->python -m pip install git+https://github.com/huggingface/peft<!-- HTML_TAG_END --></pre></div> <h2 class="relative group"><a id="valueerror-attempting-to-unscale-fp16-gradients" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#valueerror-attempting-to-unscale-fp16-gradients"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>ValueError: Attempting to unscale FP16 gradients</span></h2> <p data-svelte-h="svelte-oa0e54">This error probably occurred because the model was loaded with <code>torch_dtype=torch.float16</code> and then used in an automatic mixed precision (AMP) context, e.g. by setting <code>fp16=True</code> in the <a href="https://huggingface.co/docs/transformers/main/en/main_classes/trainer#transformers.Trainer" rel="nofollow">Trainer</a> class from 🤗 Transformers. The reason is that when using AMP, trainable weights should never use fp16. To make this work without loading the whole model in fp32, add the following to your code:</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->peft_model = get_peft_model(...)

	<span class="hljs-comment"># add this:</span>
	<span class="hljs-keyword">for</span> param <span class="hljs-keyword">in</span> model.parameters():
	<span class="hljs-keyword">if</span> param.requires_grad:
	param.data = param.data.<span class="hljs-built_in">float</span>()

	<span class="hljs-comment"># proceed as usual</span>
	trainer = Trainer(model=peft_model, fp16=<span class="hljs-literal">True</span>, ...)
	trainer.train()<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-p7trz1">Alternatively, you can use the <a href="/docs/peft/main/en/package_reference/peft_model#peft.cast_mixed_precision_params">cast_mixed_precision_params()</a> function to correctly cast the weights:</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-keyword">from</span> peft <span class="hljs-keyword">import</span> cast_mixed_precision_params

	peft_model = get_peft_model(...)
	cast_mixed_precision_params(peft_model, dtype=torch.float16)

	<span class="hljs-comment"># proceed as usual</span>
	trainer = Trainer(model=peft_model, fp16=<span class="hljs-literal">True</span>, ...)
	trainer.train()<!-- HTML_TAG_END --></pre></div> <div class="course-tip bg-gradient-to-br dark:bg-gradient-to-r before:border-green-500 dark:before:border-green-800 from-green-50 dark:from-gray-900 to-white dark:to-gray-950 border border-green-50 text-green-700 dark:text-gray-400"><p data-svelte-h="svelte-y2m5zt">Starting from PEFT verion v0.12.0, PEFT automatically promotes the dtype of adapter weights from <code>torch.float16</code> and <code>torch.bfloat16</code> to <code>torch.float32</code> where appropriate. To <em>prevent</em> this behavior, you can pass <code>autocast_adapter_dtype=False</code> to <a href="/docs/peft/main/en/package_reference/peft_model#peft.get_peft_model">~get_peft_model()</a>, to <a href="/docs/peft/main/en/package_reference/peft_model#peft.PeftModel.from_pretrained">from_pretrained()</a>, and to <a href="/docs/peft/main/en/package_reference/peft_model#peft.PeftModel.load_adapter">load_adapter()</a>.</p></div> <h2 class="relative group"><a id="bad-results-from-a-loaded-peft-model" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#bad-results-from-a-loaded-peft-model"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Bad results from a loaded PEFT model</span></h2> <p data-svelte-h="svelte-ns8zue">There can be several reasons for getting a poor result from a loaded PEFT model which are listed below. If you’re still unable to troubleshoot the problem, see if anyone else had a similar <a href="https://github.com/huggingface/peft/issues" rel="nofollow">issue</a> on GitHub, and if you can’t find any, open a new issue.</p> <p data-svelte-h="svelte-lfc2ft">When opening an issue, it helps a lot if you provide a minimal code example that reproduces the issue. Also, please report if the loaded model performs at the same level as the model did before fine-tuning, if it performs at a random level, or if it is only slightly worse than expected. This information helps us identify the problem more quickly.</p> <h3 class="relative group"><a id="random-deviations" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#random-deviations"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Random deviations</span></h3> <p data-svelte-h="svelte-16feh">If your model outputs are not exactly the same as previous runs, there could be an issue with random elements. For example:</p> <ol data-svelte-h="svelte-1fvhlks"><li>please ensure it is in <code>.eval()</code> mode, which is important, for instance, if the model uses dropout</li> <li>if you use <a href="https://huggingface.co/docs/transformers/main/en/main_classes/text_generation#transformers.GenerationMixin.generate" rel="nofollow">generate</a> on a language model, there could be random sampling, so obtaining the same result requires setting a random seed</li> <li>if you used quantization and merged the weights, small deviations are expected due to rounding errors</li></ol> <h3 class="relative group"><a id="incorrectly-loaded-model" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#incorrectly-loaded-model"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Incorrectly loaded model</span></h3> <p data-svelte-h="svelte-q1qqrd">Please ensure that you load the model correctly. A common error is trying to load a <em>trained</em> model with <a href="/docs/peft/main/en/package_reference/peft_model#peft.get_peft_model">get_peft_model()</a> which is incorrect. Instead, the loading code should look like this:</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-keyword">from</span> peft <span class="hljs-keyword">import</span> PeftModel, PeftConfig

	base_model = ... <span class="hljs-comment"># to load the base model, use the same code as when you trained it</span>
	config = PeftConfig.from_pretrained(peft_model_id)
	peft_model = PeftModel.from_pretrained(base_model, peft_model_id)<!-- HTML_TAG_END --></pre></div> <h3 class="relative group"><a id="randomly-initialized-layers" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#randomly-initialized-layers"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Randomly initialized layers</span></h3> <p data-svelte-h="svelte-5heszj">For some tasks, it is important to correctly configure <code>modules_to_save</code> in the config to account for randomly initialized layers.</p> <p data-svelte-h="svelte-c4824l">As an example, this is necessary if you use LoRA to fine-tune a language model for sequence classification because 🤗 Transformers adds a randomly initialized classification head on top of the model. If you do not add this layer to <code>modules_to_save</code>, the classification head won’t be saved. The next time you load the model, you’ll get a <em>different</em> randomly initialized classification head, resulting in completely different results.</p> <p data-svelte-h="svelte-1ou3c35">PEFT tries to correctly guess the <code>modules_to_save</code> if you provide the <code>task_type</code> argument in the config. This should work for transformers models that follow the standard naming scheme. It is always a good idea to double check though because we can’t guarantee all models follow the naming scheme.</p> <p data-svelte-h="svelte-1g4txs2">When you load a transformers model that has randomly initialized layers, you should see a warning along the lines of:</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-keyword">Some</span> weights <span class="hljs-keyword">of</span> <MODEL> were <span class="hljs-keyword">not</span> initialized <span class="hljs-keyword">from</span> the model <span class="hljs-keyword">checkpoint</span> at <ID> <span class="hljs-keyword">and</span> are newly initialized: [<LAYER_NAMES>].
	You should probably TRAIN this model <span class="hljs-keyword">on</span> a down-stream task <span class="hljs-keyword">to</span> be able <span class="hljs-keyword">to</span> use it <span class="hljs-keyword">for</span> predictions <span class="hljs-keyword">and</span> inference.<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-1fbeum6">The mentioned layers should be added to <code>modules_to_save</code> in the config to avoid the described problem.</p> <h3 class="relative group"><a id="extending-the-vocabulary" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#extending-the-vocabulary"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Extending the vocabulary</span></h3> <p data-svelte-h="svelte-15igmai">For many language fine-tuning tasks, extending the model’s vocabulary is necessary since new tokens are being introduced. This requires extending the embedding layer to account for the new tokens and also storing the embedding layer in addition to the adapter weights when saving the adapter.</p> <p data-svelte-h="svelte-1cgzlph">Save the embedding layer by adding it to the <code>target_modules</code> of the config. The embedding layer name must follow the standard naming scheme from Transformers. For example, the Mistral config could look like this:</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->config = LoraConfig(..., target_modules=[<span class="hljs-string">"embed_tokens"</span>, <span class="hljs-string">"lm_head"</span>, <span class="hljs-string">"q_proj"</span>, <span class="hljs-string">"v_proj"</span>])<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-kwgcwv">Once added to <code>target_modules</code>, PEFT automatically stores the embedding layer when saving the adapter if the model has the <a href="https://huggingface.co/docs/transformers/main/en/main_classes/model#transformers.PreTrainedModel.get_input_embeddings" rel="nofollow">get_input_embeddings</a> and <a href="https://huggingface.co/docs/transformers/main/en/main_classes/model#transformers.PreTrainedModel.get_output_embeddings" rel="nofollow">get_output_embeddings</a>. This is generally the case for Transformers models.</p> <p data-svelte-h="svelte-ucua6z">If the model’s embedding layer doesn’t follow the Transformer’s naming scheme, you can still save it by manually passing <code>save_embedding_layers=True</code> when saving the adapter:</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->model = get_peft_model(...)
	<span class="hljs-comment"># train the model</span>
	model.save_pretrained(<span class="hljs-string">"my_adapter"</span>, save_embedding_layers=<span class="hljs-literal">True</span>)<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-nkv7o7">For inference, load the base model first and resize it the same way you did before you trained the model. After you’ve resized the base model, you can load the PEFT checkpoint.</p> <p data-svelte-h="svelte-a0ibfa">For a complete example, please check out <a href="https://github.com/huggingface/peft/blob/main/examples/causal_language_modeling/peft_lora_clm_with_additional_tokens.ipynb" rel="nofollow">this notebook</a>.</p> <h3 class="relative group"><a id="check-layer-and-model-status" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#check-layer-and-model-status"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Check layer and model status</span></h3> <p data-svelte-h="svelte-1f0hxdh">Sometimes a PEFT model can end up in a bad state, especially when handling multiple adapters. There can be some confusion around what adapters exist, which one is active, which one is merged, etc. To help investigate this issue, call the <a href="/docs/peft/main/en/package_reference/peft_model#peft.PeftModel.get_layer_status">get_layer_status()</a> and the <a href="/docs/peft/main/en/package_reference/peft_model#peft.PeftModel.get_model_status">get_model_status()</a> methods.</p> <p data-svelte-h="svelte-1ygf8mz">The <a href="/docs/peft/main/en/package_reference/peft_model#peft.PeftModel.get_layer_status">get_layer_status()</a> method gives you a detailed overview of each targeted layer’s active, merged, and available adapters.</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-meta">>>> </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModel
	<span class="hljs-meta">>>> </span><span class="hljs-keyword">from</span> peft <span class="hljs-keyword">import</span> get_peft_model, LoraConfig

	<span class="hljs-meta">>>> </span>model_id = <span class="hljs-string">"google/flan-t5-small"</span>
	<span class="hljs-meta">>>> </span>model = AutoModel.from_pretrained(model_id)
	<span class="hljs-meta">>>> </span>model = get_peft_model(model, LoraConfig())

	<span class="hljs-meta">>>> </span>model.get_layer_status()
	[TunerLayerStatus(name=<span class="hljs-string">'model.encoder.block.0.layer.0.SelfAttention.q'</span>,
	module_type=<span class="hljs-string">'lora.Linear'</span>,
	enabled=<span class="hljs-literal">True</span>,
	active_adapters=[<span class="hljs-string">'default'</span>],
	merged_adapters=[],
	requires_grad={<span class="hljs-string">'default'</span>: <span class="hljs-literal">True</span>},
	available_adapters=[<span class="hljs-string">'default'</span>]),
	TunerLayerStatus(name=<span class="hljs-string">'model.encoder.block.0.layer.0.SelfAttention.v'</span>,
	module_type=<span class="hljs-string">'lora.Linear'</span>,
	enabled=<span class="hljs-literal">True</span>,
	active_adapters=[<span class="hljs-string">'default'</span>],
	merged_adapters=[],
	requires_grad={<span class="hljs-string">'default'</span>: <span class="hljs-literal">True</span>},
	available_adapters=[<span class="hljs-string">'default'</span>]),
	...]

	<span class="hljs-meta">>>> </span>model.get_model_status()
	TunerModelStatus(
	base_model_type=<span class="hljs-string">'T5Model'</span>,
	adapter_model_type=<span class="hljs-string">'LoraModel'</span>,
	peft_types={<span class="hljs-string">'default'</span>: <span class="hljs-string">'LORA'</span>},
	trainable_params=<span class="hljs-number">344064</span>,
	total_params=<span class="hljs-number">60855680</span>,
	num_adapter_layers=<span class="hljs-number">48</span>,
	enabled=<span class="hljs-literal">True</span>,
	active_adapters=[<span class="hljs-string">'default'</span>],
	merged_adapters=[],
	requires_grad={<span class="hljs-string">'default'</span>: <span class="hljs-literal">True</span>},
	available_adapters=[<span class="hljs-string">'default'</span>],
	)<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-1gb1r7d">In the model state output, you should look out for entries that say <code>"irregular"</code>. This means PEFT detected an inconsistent state in the model. For instance, if <code>merged_adapters="irregular"</code>, it means that for at least one adapter, it was merged on some target modules but not on others. The inference results will most likely be incorrect as a result.</p> <p data-svelte-h="svelte-19sfzdf">The best way to resolve this issue is to reload the whole model and adapter checkpoint(s). Ensure that you don’t perform any incorrect operations on the model, e.g. manually merging adapters on some modules but not others.</p> <p data-svelte-h="svelte-18r4lni">Convert the layer status into a pandas <code>DataFrame</code> for an easier visual inspection.</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-keyword">from</span> dataclasses <span class="hljs-keyword">import</span> asdict
	<span class="hljs-keyword">import</span> pandas <span class="hljs-keyword">as</span> pd

	df = pd.DataFrame(asdict(layer) <span class="hljs-keyword">for</span> layer <span class="hljs-keyword">in</span> model.get_layer_status())<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-17rnlhh">It is possible to get this information for non-PEFT models if they are using PEFT layers under the hood, but some information like the <code>base_model_type</code> or the <code>peft_types</code> cannot be determined in that case. As an example, you can call this on a <a href="https://huggingface.co/docs/diffusers/index" rel="nofollow">diffusers</a> model like so:</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-meta">>>> </span><span class="hljs-keyword">import</span> torch
	<span class="hljs-meta">>>> </span><span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> StableDiffusionPipeline
	<span class="hljs-meta">>>> </span><span class="hljs-keyword">from</span> peft <span class="hljs-keyword">import</span> get_model_status, get_layer_status

	<span class="hljs-meta">>>> </span>path = <span class="hljs-string">"runwayml/stable-diffusion-v1-5"</span>
	<span class="hljs-meta">>>> </span>lora_id = <span class="hljs-string">"takuma104/lora-test-text-encoder-lora-target"</span>
	<span class="hljs-meta">>>> </span>pipe = StableDiffusionPipeline.from_pretrained(path, torch_dtype=torch.float16)
	<span class="hljs-meta">>>> </span>pipe.load_lora_weights(lora_id, adapter_name=<span class="hljs-string">"adapter-1"</span>)
	<span class="hljs-meta">>>> </span>pipe.load_lora_weights(lora_id, adapter_name=<span class="hljs-string">"adapter-2"</span>)
	<span class="hljs-meta">>>> </span>pipe.set_lora_device([<span class="hljs-string">"adapter-2"</span>], <span class="hljs-string">"cuda"</span>)
	<span class="hljs-meta">>>> </span>get_layer_status(pipe.text_encoder)
	[TunerLayerStatus(name=<span class="hljs-string">'text_model.encoder.layers.0.self_attn.k_proj'</span>,
	module_type=<span class="hljs-string">'lora.Linear'</span>,
	enabled=<span class="hljs-literal">True</span>,
	active_adapters=[<span class="hljs-string">'adapter-2'</span>],
	merged_adapters=[],
	requires_grad={<span class="hljs-string">'adapter-1'</span>: <span class="hljs-literal">False</span>, <span class="hljs-string">'adapter-2'</span>: <span class="hljs-literal">True</span>},
	available_adapters=[<span class="hljs-string">'adapter-1'</span>, <span class="hljs-string">'adapter-2'</span>],
	devices={<span class="hljs-string">'adapter-1'</span>: [<span class="hljs-string">'cpu'</span>], <span class="hljs-string">'adapter-2'</span>: [<span class="hljs-string">'cuda'</span>]}),
	TunerLayerStatus(name=<span class="hljs-string">'text_model.encoder.layers.0.self_attn.v_proj'</span>,
	module_type=<span class="hljs-string">'lora.Linear'</span>,
	enabled=<span class="hljs-literal">True</span>,
	active_adapters=[<span class="hljs-string">'adapter-2'</span>],
	merged_adapters=[],
	requires_grad={<span class="hljs-string">'adapter-1'</span>: <span class="hljs-literal">False</span>, <span class="hljs-string">'adapter-2'</span>: <span class="hljs-literal">True</span>},
	devices={<span class="hljs-string">'adapter-1'</span>: [<span class="hljs-string">'cpu'</span>], <span class="hljs-string">'adapter-2'</span>: [<span class="hljs-string">'cuda'</span>]}),
	...]

	<span class="hljs-meta">>>> </span>get_model_status(pipe.unet)
	TunerModelStatus(
	base_model_type=<span class="hljs-string">'other'</span>,
	adapter_model_type=<span class="hljs-string">'None'</span>,
	peft_types={},
	trainable_params=<span class="hljs-number">797184</span>,
	total_params=<span class="hljs-number">861115332</span>,
	num_adapter_layers=<span class="hljs-number">128</span>,
	enabled=<span class="hljs-literal">True</span>,
	active_adapters=[<span class="hljs-string">'adapter-2'</span>],
	merged_adapters=[],
	requires_grad={<span class="hljs-string">'adapter-1'</span>: <span class="hljs-literal">False</span>, <span class="hljs-string">'adapter-2'</span>: <span class="hljs-literal">True</span>},
	available_adapters=[<span class="hljs-string">'adapter-1'</span>, <span class="hljs-string">'adapter-2'</span>],
	devices={<span class="hljs-string">'adapter-1'</span>: [<span class="hljs-string">'cpu'</span>], <span class="hljs-string">'adapter-2'</span>: [<span class="hljs-string">'cuda'</span>]},
	)<!-- HTML_TAG_END --></pre></div> <h2 class="relative group"><a id="reproducibility" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#reproducibility"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Reproducibility</span></h2> <h3 class="relative group"><a id="models-using-batch-norm" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#models-using-batch-norm"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Models using batch norm</span></h3> <p data-svelte-h="svelte-yt67c7">When loading a trained PEFT model where the base model uses batch norm (e.g. <code>torch.nn.BatchNorm1d</code> or <code>torch.nn.BatchNorm2d</code>), you may find that you cannot reproduce the exact same outputs. This is because the batch norm layers keep track of running stats during training, but these stats are not part of the PEFT checkpoint. Therefore, when you load the PEFT model, the running stats of the base model will be used (i.e. from before training with PEFT).</p> <p data-svelte-h="svelte-wblxho">Depending on your use case, this may not be a big deal. If, however, you need your outputs to be 100% reproducible, you can achieve this by adding the batch norm layers to <code>modules_to_save</code>. Below is an example of this using resnet and LoRA. Notice that we set <code>modules_to_save=["classifier", "normalization"]</code>. We need the <code>"classifier"</code> argument because our task is image classification, and we add the <code>"normalization"</code> argument to ensure that the batch norm layers are saved in the PEFT checkpoint.</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForImageClassification
	<span class="hljs-keyword">from</span> peft <span class="hljs-keyword">import</span> LoraConfig, get_peft_model

	model_id = <span class="hljs-string">"microsoft/resnet-18"</span>
	base_model = AutoModelForImageClassification.from_pretrained(self.model_id)
	config = LoraConfig(
	target_modules=[<span class="hljs-string">"convolution"</span>],
	modules_to_save=[<span class="hljs-string">"classifier"</span>, <span class="hljs-string">"normalization"</span>],
	),<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-1e3stb1">Depending on the type of model you use, the batch norm layers could have different names than <code>"normalization"</code>, so please ensure that the name matches your model architecture.</p> <a class="!text-gray-400 !no-underline text-sm flex items-center not-prose mt-4" href="https://github.com/huggingface/peft/blob/main/docs/source/developer_guides/troubleshooting.md" target="_blank"><span data-svelte-h="svelte-1kd6by1"><</span> <span data-svelte-h="svelte-x0xyl0">></span> <span data-svelte-h="svelte-1dajgef"><span class="underline ml-1.5">Update</span> on GitHub</span></a> <p></p>

	<script>
	{
	__sveltekit_nv4ir6 = {
	assets: "/docs/peft/main/en",
	base: "/docs/peft/main/en",
	env: {}
	};

	const element = document.currentScript.parentElement;

	const data = [null,null];

	Promise.all([
	import("/docs/peft/main/en/_app/immutable/entry/start.b1913684.js"),
	import("/docs/peft/main/en/_app/immutable/entry/app.ca8eed50.js")
	]).then(([kit, app]) => {
	kit.start(app, element, {
	node_ids: [0, 17],
	data,
	form: null,
	error: null
	});
	});
	}
	</script>

Xet Storage Details

Size:: 54.2 kB
Xet hash:: e03831b26f56b3d1dfa45aaa99170158bab2774eac9ba4e58fc3b6d0e56250a9

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.