Buckets:

hf-doc-build
/

doc-dev

Files

xet

hf-doc-build/doc-dev / peft /pr_3205 /en /conceptual_guides /adapter.html

HuggingFaceDocBuilder

4 months ago

download

raw

38.8 kB

	<meta charset="utf-8" /><meta name="hf:doc:metadata" content="{"title":"Adapters","local":"adapters","sections":[{"title":"Low-Rank Adaptation (LoRA)","local":"low-rank-adaptation-lora","sections":[],"depth":2},{"title":"Mixture of LoRA Experts (X-LoRA)","local":"mixture-of-lora-experts-x-lora","sections":[],"depth":2},{"title":"Low-Rank Hadamard Product (LoHa)","local":"low-rank-hadamard-product-loha","sections":[],"depth":2},{"title":"Low-Rank Kronecker Product (LoKr)","local":"low-rank-kronecker-product-lokr","sections":[],"depth":2},{"title":"Orthogonal Finetuning (OFT)","local":"orthogonal-finetuning-oft","sections":[],"depth":2},{"title":"Orthogonal Butterfly (BOFT)","local":"orthogonal-butterfly-boft","sections":[],"depth":2},{"title":"Adaptive Low-Rank Adaptation (AdaLoRA)","local":"adaptive-low-rank-adaptation-adalora","sections":[],"depth":2},{"title":"Llama-Adapter","local":"llama-adapter","sections":[],"depth":2},{"title":"Householder Reflection Adaptation (HRA)","local":"householder-reflection-adaptation-hra","sections":[],"depth":2},{"title":"Bone","local":"bone","sections":[],"depth":2},{"title":"MiSS","local":"miss","sections":[],"depth":2}],"depth":1}">
	<link href="/docs/peft/pr_3205/en/_app/immutable/assets/0.e3b0c442.css" rel="modulepreload">
	<link rel="modulepreload" href="/docs/peft/pr_3205/en/_app/immutable/entry/start.6e4f7154.js">
	<link rel="modulepreload" href="/docs/peft/pr_3205/en/_app/immutable/chunks/scheduler.78382b47.js">
	<link rel="modulepreload" href="/docs/peft/pr_3205/en/_app/immutable/chunks/singletons.fa874020.js">
	<link rel="modulepreload" href="/docs/peft/pr_3205/en/_app/immutable/chunks/index.fadd215c.js">
	<link rel="modulepreload" href="/docs/peft/pr_3205/en/_app/immutable/chunks/paths.ca1902b3.js">
	<link rel="modulepreload" href="/docs/peft/pr_3205/en/_app/immutable/entry/app.968b774b.js">
	<link rel="modulepreload" href="/docs/peft/pr_3205/en/_app/immutable/chunks/preload-helper.db85a81e.js">
	<link rel="modulepreload" href="/docs/peft/pr_3205/en/_app/immutable/chunks/index.6dd35eb6.js">
	<link rel="modulepreload" href="/docs/peft/pr_3205/en/_app/immutable/nodes/0.02e4cb39.js">
	<link rel="modulepreload" href="/docs/peft/pr_3205/en/_app/immutable/chunks/each.e59479a4.js">
	<link rel="modulepreload" href="/docs/peft/pr_3205/en/_app/immutable/nodes/4.79fac136.js">
	<link rel="modulepreload" href="/docs/peft/pr_3205/en/_app/immutable/chunks/MermaidChart.svelte_svelte_type_style_lang.184d38d3.js"><!-- HEAD_svelte-u9bgzb_START --><meta name="hf:doc:metadata" content="{"title":"Adapters","local":"adapters","sections":[{"title":"Low-Rank Adaptation (LoRA)","local":"low-rank-adaptation-lora","sections":[],"depth":2},{"title":"Mixture of LoRA Experts (X-LoRA)","local":"mixture-of-lora-experts-x-lora","sections":[],"depth":2},{"title":"Low-Rank Hadamard Product (LoHa)","local":"low-rank-hadamard-product-loha","sections":[],"depth":2},{"title":"Low-Rank Kronecker Product (LoKr)","local":"low-rank-kronecker-product-lokr","sections":[],"depth":2},{"title":"Orthogonal Finetuning (OFT)","local":"orthogonal-finetuning-oft","sections":[],"depth":2},{"title":"Orthogonal Butterfly (BOFT)","local":"orthogonal-butterfly-boft","sections":[],"depth":2},{"title":"Adaptive Low-Rank Adaptation (AdaLoRA)","local":"adaptive-low-rank-adaptation-adalora","sections":[],"depth":2},{"title":"Llama-Adapter","local":"llama-adapter","sections":[],"depth":2},{"title":"Householder Reflection Adaptation (HRA)","local":"householder-reflection-adaptation-hra","sections":[],"depth":2},{"title":"Bone","local":"bone","sections":[],"depth":2},{"title":"MiSS","local":"miss","sections":[],"depth":2}],"depth":1}"><!-- HEAD_svelte-u9bgzb_END --> <p></p> <div class="items-center shrink-0 min-w-[100px] max-sm:min-w-[50px] justify-end ml-auto flex" style="float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"><div class="inline-flex rounded-md max-sm:rounded-sm"><button class="inline-flex items-center gap-1 h-7 max-sm:h-7 px-2 max-sm:px-1.5 text-sm font-medium text-gray-800 border border-r-0 rounded-l-md max-sm:rounded-l-sm border-gray-200 bg-white hover:shadow-inner dark:border-gray-850 dark:bg-gray-950 dark:text-gray-200 dark:hover:bg-gray-800" aria-live="polite"><span class="inline-flex items-center justify-center rounded-md p-0.5 max-sm:p-0 hover:text-gray-800 dark:hover:text-gray-200"><svg class="sm:size-3.5 size-3" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg></span> <span>Copy page</span></button> <button class="inline-flex items-center justify-center w-6 max-sm:w-5 h-7 max-sm:h-7 disabled:pointer-events-none text-sm text-gray-500 hover:text-gray-700 dark:hover:text-white rounded-r-md max-sm:rounded-r-sm border border-l transition border-gray-200 bg-white hover:shadow-inner dark:border-gray-850 dark:bg-gray-950 dark:text-gray-200 dark:hover:bg-gray-800" aria-haspopup="menu" aria-expanded="false" aria-label="Open copy menu"><svg class="transition-transform text-gray-400 overflow-visible sm:size-3.5 size-3 rotate-0" width="1em" height="1em" viewBox="0 0 12 7" fill="none" xmlns="http://www.w3.org/2000/svg"><path d="M1 1L6 6L11 1" stroke="currentColor"></path></svg></button></div> </div> <h1 class="relative group"><a id="adapters" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#adapters"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Adapters</span></h1> <p data-svelte-h="svelte-1udysl8">Adapter-based methods add extra trainable parameters after the attention and fully-connected layers of a frozen pretrained model to reduce memory-usage and speed up training. The method varies depending on the adapter, it could simply be an extra added layer or it could be expressing the weight updates ∆W as a low-rank decomposition of the weight matrix. Either way, the adapters are typically small but demonstrate comparable performance to a fully finetuned model and enable training larger models with fewer resources.</p> <p data-svelte-h="svelte-2ih5z0">This guide will give you a brief overview of the adapter methods supported by PEFT (if you’re interested in learning more details about a specific method, take a look at the linked paper).</p> <h2 class="relative group"><a id="low-rank-adaptation-lora" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#low-rank-adaptation-lora"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Low-Rank Adaptation (LoRA)</span></h2> <blockquote class="tip" data-svelte-h="svelte-1wx7gg2"><p>LoRA is one of the most popular PEFT methods and a good starting point if you’re just getting started with PEFT. It was originally developed for large language models but it is a tremendously popular training method for diffusion models because of its efficiency and effectiveness.</p></blockquote> <p data-svelte-h="svelte-1jx2sye">As mentioned briefly earlier, <a href="https://hf.co/papers/2106.09685" rel="nofollow">LoRA</a> is a technique that accelerates finetuning large models while consuming less memory.</p> <p data-svelte-h="svelte-1h9l7gl">LoRA represents the weight updates ∆W with two smaller matrices (called <em>update matrices</em>) through low-rank decomposition. These new matrices can be trained to adapt to the new data while keeping the overall number of parameters low. The original weight matrix remains frozen and doesn’t receive any further updates. To produce the final results, the original and extra adapted weights are combined. You could also merge the adapter weights with the base model to eliminate inference latency.</p> <div class="flex justify-center" data-svelte-h="svelte-1yiq52m"><img src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/peft/lora_animated.gif"></div> <p data-svelte-h="svelte-tt8pol">This approach has a number of advantages:</p> <ul data-svelte-h="svelte-18hcxvt"><li>LoRA makes finetuning more efficient by drastically reducing the number of trainable parameters.</li> <li>The original pretrained weights are kept frozen, which means you can have multiple lightweight and portable LoRA models for various downstream tasks built on top of them.</li> <li>LoRA is orthogonal to other parameter-efficient methods and can be combined with many of them.</li> <li>Performance of models finetuned using LoRA is comparable to the performance of fully finetuned models.</li></ul> <p data-svelte-h="svelte-j2a6td">In principle, LoRA can be applied to any subset of weight matrices in a neural network to reduce the number of trainable parameters. However, for simplicity and further parameter efficiency, LoRA is typically only applied to the attention blocks in Transformer models. The resulting number of trainable parameters in a LoRA model depends on the size of the update matrices, which is determined mainly by the rank <code>r</code> and the shape of the original weight matrix.</p> <div class="flex justify-center" data-svelte-h="svelte-oil2ou"><img src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/peft/lora.png"></div> <small data-svelte-h="svelte-1py15yj"><a href="https://hf.co/papers/2103.10385">Navigating Text-To-Image Customization: From LyCORIS Fine-Tuning to Model Evaluation</a></small> <h2 class="relative group"><a id="mixture-of-lora-experts-x-lora" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#mixture-of-lora-experts-x-lora"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Mixture of LoRA Experts (X-LoRA)</span></h2> <p data-svelte-h="svelte-1nc77nj"><a href="https://huggingface.co/papers/2402.07148" rel="nofollow">X-LoRA</a> is a mixture of experts method for LoRA which works by using dense or sparse gating to dynamically activate LoRA experts. The LoRA experts as well as the base model are frozen during training, resulting in a low parameter count as only the gating layers must be trained. In particular, the gating layers output scalings which (depending on config) are granular on the layer and token level. Additionally, during inference, X-LoRA dynamically activates LoRA adapters to recall knowledge and effectively mix them:</p> <p data-svelte-h="svelte-1ehagof">The below graphic demonstrates how the scalings change for different prompts for each token. This highlights the activation of different adapters as the generation progresses and the sequence creates new context.</p> <p data-svelte-h="svelte-bww8jk"><img src="https://github.com/EricLBuehler/xlora/raw/master/res/token_by_token_scalings.gif" alt="Token-by-token scalings"></p> <p data-svelte-h="svelte-rgrq61">For each step, X-LoRA requires the base model to be run twice: first, to get hidden states without any LoRA adapters, and secondly, the hidden states are used to calculate scalings which are applied to the LoRA adapters and the model is run a second time. The output of the second run is the result of the model step.</p> <p data-svelte-h="svelte-skx0xh">Ultimately, X-LoRA allows the model to reflect upon its knowledge because of the dual forward pass scheme, and dynamically reconfigure the architecture.</p> <h2 class="relative group"><a id="low-rank-hadamard-product-loha" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#low-rank-hadamard-product-loha"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Low-Rank Hadamard Product (LoHa)</span></h2> <p data-svelte-h="svelte-26ytw">Low-rank decomposition can impact performance because the weight updates are limited to the low-rank space, which can constrain a model’s expressiveness. However, you don’t necessarily want to use a larger rank because it increases the number of trainable parameters. To address this, <a href="https://huggingface.co/papers/2108.06098" rel="nofollow">LoHa</a> (a method originally developed for computer vision) was applied to diffusion models where the ability to generate diverse images is an important consideration. LoHa should also work with general model types, but the embedding layers aren’t currently implemented in PEFT.</p> <p data-svelte-h="svelte-2t2yyu">LoHa uses the <a href="https://en.wikipedia.org/wiki/Hadamard_product_(matrices)" rel="nofollow">Hadamard product</a> (element-wise product) instead of the matrix product. ∆W is represented by four smaller matrices instead of two - like in LoRA - and each pair of these low-rank matrices are combined with the Hadamard product. As a result, ∆W can have the same number of trainable parameters but a higher rank and expressivity.</p> <h2 class="relative group"><a id="low-rank-kronecker-product-lokr" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#low-rank-kronecker-product-lokr"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Low-Rank Kronecker Product (LoKr)</span></h2> <p data-svelte-h="svelte-kr8vx3"><a href="https://hf.co/papers/2309.14859" rel="nofollow">LoKr</a> is very similar to LoRA and LoHa, and it is also mainly applied to diffusion models, though you could also use it with other model types. LoKr replaces the matrix product with the <a href="https://en.wikipedia.org/wiki/Kronecker_product" rel="nofollow">Kronecker product</a> instead. The Kronecker product decomposition creates a block matrix which preserves the rank of the original weight matrix. Another benefit of the Kronecker product is that it can be vectorized by stacking the matrix columns. This can speed up the process because you’re avoiding fully reconstructing ∆W.</p> <h2 class="relative group"><a id="orthogonal-finetuning-oft" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#orthogonal-finetuning-oft"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Orthogonal Finetuning (OFT)</span></h2> <div class="flex justify-center" data-svelte-h="svelte-yxp5w7"><img src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/peft/oft.png"></div> <small data-svelte-h="svelte-1bvbr61"><a href="https://hf.co/papers/2306.07280">Controlling Text-to-Image Diffusion by Orthogonal Finetuning</a></small> <p data-svelte-h="svelte-11bv21u"><a href="https://hf.co/papers/2306.07280" rel="nofollow">OFT</a> is a method that primarily focuses on preserving a pretrained model’s generative performance in the finetuned model. It tries to maintain the same cosine similarity (hyperspherical energy) between all pairwise neurons in a layer because this better captures the semantic information among neurons. This means OFT is more capable at preserving the subject and it is better for controllable generation (similar to <a href="https://huggingface.co/docs/diffusers/using-diffusers/controlnet" rel="nofollow">ControlNet</a>).</p> <p data-svelte-h="svelte-1pykz3m">OFT preserves the hyperspherical energy by learning an orthogonal transformation for neurons to keep the cosine similarity between them unchanged. In practice, this means taking the matrix product of an orthogonal matrix with the pretrained weight matrix. However, to be parameter-efficient, the orthogonal matrix is represented as a block-diagonal matrix with rank <code>r</code> blocks. Whereas LoRA reduces the number of trainable parameters with low-rank structures, OFT reduces the number of trainable parameters with a sparse block-diagonal matrix structure.</p> <h2 class="relative group"><a id="orthogonal-butterfly-boft" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#orthogonal-butterfly-boft"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Orthogonal Butterfly (BOFT)</span></h2> <p data-svelte-h="svelte-jxh68"><a href="https://hf.co/papers/2311.06243" rel="nofollow">BOFT</a> is an improved orthogonal finetuning method that focuses on preserving a pretrained model’s generative capabilities while being significantly more parameter-efficient than standard OFT. Like OFT, BOFT maintains the same cosine similarity (hyperspherical energy) between all pairwise neurons in a layer by applying an orthogonal transformation to the pretrained weight matrix, ensuring the semantic relationships among neurons are preserved.</p> <p data-svelte-h="svelte-zpkz7u">Instead of using a block-diagonal orthogonal matrix, BOFT factorizes the orthogonal transformation into a product of <strong>sparse butterfly matrices</strong> (originally introduced in the <a href="https://en.wikipedia.org/wiki/Cooley%E2%80%93Tukey_FFT_algorithm" rel="nofollow">Cooley–Tukey FFT</a>). Unlike OFT’s block-diagonal rotations, which only mix inputs within each block, the butterfly structure guarantees that every input can influence every output, producing a <strong>dense connectivity</strong> with just <code>O(d log d)</code> parameters. This factorization preserves expressivity while drastically reducing the parameter count compared to OFT (at the expense of computation time).</p> <p data-svelte-h="svelte-bhzbhm">In practice, BOFT multiplies each pretrained weight matrix by a sequence of butterfly-structured orthogonal factors, enabling efficient and expressive neuron rotations. This makes BOFT well-suited for controllable generation and tasks where maintaining the pretrained model’s subject representation is critical, while also scaling to larger models with lower memory and compute overhead.</p> <h2 class="relative group"><a id="adaptive-low-rank-adaptation-adalora" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#adaptive-low-rank-adaptation-adalora"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Adaptive Low-Rank Adaptation (AdaLoRA)</span></h2> <p data-svelte-h="svelte-1343mpt"><a href="https://hf.co/papers/2303.10512" rel="nofollow">AdaLoRA</a> manages the parameter budget introduced from LoRA by allocating more parameters - in other words, a higher rank <code>r</code> - for important weight matrices that are better adapted for a task and pruning less important ones. The rank is controlled by a method similar to singular value decomposition (SVD). The ∆W is parameterized with two orthogonal matrices and a diagonal matrix which contains singular values. This parametrization method avoids iteratively applying SVD which is computationally expensive. Based on this method, the rank of ∆W is adjusted according to an importance score. ∆W is divided into triplets and each triplet is scored according to its contribution to model performance. Triplets with low importance scores are pruned and triplets with high importance scores are kept for finetuning.</p> <p data-svelte-h="svelte-1gjwbd2">Training with AdaLoRA has three phases: the init phase, the budgeting phase and the final phase. In the initial phase, no budgeting is applied, therefore the ranks are not touched. During the budgeting phase the process described above is applied and the rank is redistributed according to a budget, aiming to give more important adapters more rank and less important layers less. When reaching the final phase, budgeting has ended, the ranks are redistributed but we may continue training for a while with the redistributed ranks to further improve performance.</p> <h2 class="relative group"><a id="llama-adapter" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#llama-adapter"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Llama-Adapter</span></h2> <p data-svelte-h="svelte-vxhqjn"><a href="https://hf.co/papers/2303.16199" rel="nofollow">Llama-Adapter</a> is a method for adapting Llama into an instruction-following model. To help adapt the model for instruction-following, the adapter is trained with a 52K instruction-output dataset.</p> <p data-svelte-h="svelte-1h31mxk">A set of learnable adaption prompts are prefixed to the input instruction tokens. These are inserted into the upper layers of the model because it is better to learn with the higher-level semantics of the pretrained model. The instruction-output tokens prefixed to the input guide the adaption prompt to generate a contextual response.</p> <div class="flex justify-center" data-svelte-h="svelte-3f0tu3"><img src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/peft/llama-adapter.png"></div> <small data-svelte-h="svelte-1npun8x"><a href="https://hf.co/papers/2303.16199">LLaMA-Adapter: Efficient Fine-tuning of Language Models with Zero-init Attention</a></small> <p data-svelte-h="svelte-8urzuj">To avoid adding noise to the tokens, the adapter uses zero-initialized attention. On top of this, the adapter adds a learnable gating factor (initialized with zeros) to progressively add information to the model during training. This prevents overwhelming the model’s pretrained knowledge with the newly learned instructions.</p> <h2 class="relative group"><a id="householder-reflection-adaptation-hra" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#householder-reflection-adaptation-hra"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Householder Reflection Adaptation (HRA)</span></h2> <p data-svelte-h="svelte-tjdvth"><a href="https://huggingface.co/papers/2405.17484" rel="nofollow">HRA</a> provides a new perspective connecting LoRA to OFT, which means it can harness the advantages of both strategies, reduce parameters and computation costs while penalizing the loss of pre-training knowledge.</p> <div class="flex justify-center" data-svelte-h="svelte-adr3x9"><img src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/peft/hra.png"></div> <small data-svelte-h="svelte-ovly7j"><a href="https://huggingface.co/papers/2405.17484">Bridging The Gap between Low-rank and Orthogonal Adaptation via Householder Reflection Adaptation</a></small> <p data-svelte-h="svelte-1quqg8s">HRA constructs a chain of <code>r</code> trainable Householder reflections (HRs). Because the Householder reflection matrix is an orthogonal matrix and the product of orthogonal matrices is also an orthogonal matrix, HRA satisfies the theoretical guarantee of Orthogonal Finetuning (OFT). Meanwhile, HRA can also be viewed as a low-rank fine-tuning adapter by rewriting formula.</p> <p>The higher <code data-svelte-h="svelte-104cvww">r</code>, the more trainable parameters, resulting in a larger model capacity and better performance. Besides, due to the chain structure, the orthogonality of HR planes impacts the capacity and regularity of HRA. To achieve a trade-off between the model capacity and regularity, an orthogonality regularizer of the HR planes is added to the loss function. The weight<!-- HTML_TAG_START --><span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mi>λ</mi></mrow><annotation encoding="application/x-tex">\lambda</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.6944em;"></span><span class="mord mathnormal">λ</span></span></span></span><!-- HTML_TAG_END --> can control the strength of the regularizer.</p> <h2 class="relative group"><a id="bone" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#bone"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Bone</span></h2> <p data-svelte-h="svelte-11jt869">Bone was deprecated and removed in PEFT v0.19.0 in favor of <a href="https://huggingface.co/papers/2409.15371" rel="nofollow">MiSS</a> (new version of paper: “MiSS: Balancing LoRA Performance and Efficiency with Simple Shard Sharing”). If you already have a Bone checkpoint, you can use <code>/scripts/convert-bone-to-miss.py</code> to convert it into a MiSS checkpoint and proceed with training using MiSS.</p> <h2 class="relative group"><a id="miss" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#miss"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>MiSS</span></h2> <p data-svelte-h="svelte-l9dge0"><a href="https://github.com/Joluck/MiSS" rel="nofollow">MiSS</a> Matrix Shard Sharing is a novel Parameter-Efficient Fine-Tuning (PEFT) method designed to address the trade-off between adaptability and efficiency in Large Language Models. The core approach of MiSS involves a simple shard-sharing mechanism. It achieves low-rank adaptation by decomposing a weight matrix into multiple fragments and then utilizing a shared, trainable “common fragment.” The final low-rank update matrix is constructed by replicating these shared, partitioned shards. (MiSS is a novel PEFT method that adopts a low-rank structure, requires only a single trainable matrix, and introduces a new update mechanism distinct from LoRA, achieving an excellent balance between performance and efficiency.)</p> <small data-svelte-h="svelte-ak6q3w"><a href="https://arxiv.org/abs/2409.15371">MiSS: Revisiting the Trade-off in LoRA with an Efficient Shard-Sharing Structure</a></small> <p data-svelte-h="svelte-12ubopk">Intuitively, the shape of a single trainable matrix in MiSS is consistent with <code>lora_B</code>, so the <code>r</code> parameter in MiSS is less than the <code>r</code> in LoRA by (<code>in_feature * r</code>).</p> <p data-svelte-h="svelte-1ls6z8x">Note: Bat’s r (b) is special and requires that weight W satisfies the conditions <code>in_features % r == 0</code> and <code>out_features % r == 0</code>. Additionally, when <code>in_features == out_features</code> and MiSS-r equals LoRA-r, MiSS’s number of trainable parameters is only half that of LoRA.</p> <p data-svelte-h="svelte-dkmp2o">Although the nonlinear updates of Bat bring some performance improvements, they also increase computational overhead. Its main purpose is to provide researchers with a direction for improvement. Therefore, we recommend fine-tuning the comprehensive MiSS model instead.</p> <a class="!text-gray-400 !no-underline text-sm flex items-center not-prose mt-4" href="https://github.com/huggingface/peft/blob/main/docs/source/conceptual_guides/adapter.md" target="_blank"><svg class="mr-1" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M31,16l-7,7l-1.41-1.41L28.17,16l-5.58-5.59L24,9l7,7z"></path><path d="M1,16l7-7l1.41,1.41L3.83,16l5.58,5.59L8,23l-7-7z"></path><path d="M12.419,25.484L17.639,6.552l1.932,0.518L14.351,26.002z"></path></svg> <span data-svelte-h="svelte-zjs2n5"><span class="underline">Update</span> on GitHub</span></a> <p></p>

	<script>
	{
	__sveltekit_k1ait3 = {
	assets: "/docs/peft/pr_3205/en",
	base: "/docs/peft/pr_3205/en",
	env: {}
	};

	const element = document.currentScript.parentElement;

	const data = [null,null];

	Promise.all([
	import("/docs/peft/pr_3205/en/_app/immutable/entry/start.6e4f7154.js"),
	import("/docs/peft/pr_3205/en/_app/immutable/entry/app.968b774b.js")
	]).then(([kit, app]) => {
	kit.start(app, element, {
	node_ids: [0, 4],
	data,
	form: null,
	error: null
	});
	});
	}
	</script>

Xet Storage Details

Size:: 38.8 kB
Xet hash:: 07fe720abc41143470f2f30e6a7fd7d87f566e616f36c6cfc015c03b76e7c4d0

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.