peft/pr_3111/en/_app/immutable/nodes/2.CwGSFXb4.js

import"../chunks/DsnmJJEf.js";import{i as O,h as D,C as z,H as t,b as f,a as u,D as l,E as V,s as X}from"../chunks/C6Httu04.js";import{p as Y,o as S,s as a,f as b,a as i,b as H,c as o,d as _,n as d,r as s}from"../chunks/jDjavuwI.js";import{E as I}from"../chunks/Br76-P0a.js";const P='{"title":"AdaLoRA","local":"adalora","sections":[{"title":"Benchmark overview","local":"benchmark-overview","sections":[],"depth":2},{"title":"Usage","local":"usage","sections":[],"depth":2}],"depth":1}';var K=_('<meta name="hf:doc:metadata"/>'),Z=_("<p>Example:</p> <!>",1),$=_(`<p></p> <!> <!> <p><a href="https://hf.co/papers/2303.10512" rel="nofollow">AdaLoRA</a> (Adaptive LoRA) is a method for optimizing the number of trainable parameters to assign to weight matrices and layers, unlike LoRA, which distributes parameters evenly across all modules. More parameters are budgeted for important weight matrices and layers while less important ones receive fewer parameters. You can control the average desired <em>rank</em> or <code>r</code> of the matrices, and which modules to apply AdaLoRA to with <code>target_modules</code>. Other important parameters to set are <code>lora_alpha</code> (scaling factor), and <code>modules_to_save</code> (the modules apart from the AdaLoRA layers to be trained and saved). All of these parameters - and more - are found in the <a href="/docs/peft/pr_3111/en/package_reference/adalora#peft.AdaLoraConfig">AdaLoraConfig</a>.</p> <p>The abstract from the paper is:</p> <p><em>Fine-tuning large pre-trained language models on downstream tasks has become an important paradigm in NLP. However, common practice fine-tunes all of the parameters in a pre-trained model, which becomes prohibitive when a large number of downstream tasks are present. Therefore, many fine-tuning methods are proposed to learn incremental updates of pre-trained weights in a parameter efficient way, e.g., low-rank increments. These methods often evenly distribute the budget of incremental updates across all pre-trained weight matrices, and overlook the varying importance of different weight parameters. As a consequence, the fine-tuning performance is suboptimal. To bridge this gap, we propose AdaLoRA, which adaptively allocates the parameter budget among weight matrices according to their importance score. In particular, AdaLoRA parameterizes the incremental updates in the form of singular value decomposition. Such a novel approach allows us to effectively prune the singular values of unimportant updates, which is essentially to reduce their parameter budget but circumvent intensive exact SVD computations. We conduct extensive experiments with several pre-trained models on natural language processing, question answering, and natural language generation to validate the effectiveness of AdaLoRA. Results demonstrate that AdaLoRA manifests notable improvement over baselines, especially in the low budget settings. Our code is publicly available at <a href="https://github.com/QingruZhang/AdaLoRA" rel="nofollow">https://github.com/QingruZhang/AdaLoRA</a></em>.</p> <blockquote class="warning"><p>AdaLoRA has an <a href="/docs/peft/pr_3111/en/package_reference/adalora#peft.AdaLoraModel.update_and_allocate">update_and_allocate()</a> method that should be called at each training step to update the parameter budget and mask, otherwise the adaptation step is not performed. This requires writing a custom training loop or subclassing the <a href="https://huggingface.co/docs/transformers/main/en/main_classes/trainer#transformers.Trainer" rel="nofollow">Trainer</a> to incorporate this method. As an example, take a look at this <a href="https://github.com/huggingface/peft/blob/912ad41e96e03652cabf47522cd876076f7a0c4f/examples/conditional_generation/peft_adalora_seq2seq.py#L120" rel="nofollow">custom training loop</a>.</p></blockquote> <p>AdaLoRA manages the parameter budget introduced from LoRA by allocating more parameters - in other words, a higher rank <code>r</code> - for important weight matrices that are better adapted for a task and pruning less important ones. The rank is controlled by a method similar to singular value decomposition (SVD). The <!> is parameterized with two orthogonal matrices and a diagonal matrix which contains singular values. This parametrization method avoids iteratively applying SVD which is computationally expensive. Based on this method, the rank of <!> is adjusted according to an importance score. <!> is divided into triplets and each triplet is scored according to its contribution to model performance. Triplets with low importance scores are pruned and triplets with high importance scores are kept for finetuning.</p> <p>Training with AdaLoRA has three phases: the init phase, the budgeting phase and the final phase. In the initial phase, no budgeting is applied, therefore the ranks are not touched. During the budgeting phase the process described above is applied and the rank is redistributed according to a budget, aiming to give more important adapters more rank and less important layers less. When reaching the final phase, budgeting has ended, the ranks are redistributed but we may continue training for a while with the redistributed ranks to further improve performance.</p> <blockquote class="note"><p><strong>Contributions welcome</strong>: This section needs clarification.</p> <p>It is unclear how importance is measured. The explanations are also a bit redundant and could benefit from consolidation.
See <a href="../developer_guides/contributing#documentation-improvements">here</a> on how to contribute.</p></blockquote> <!> <iframe src="https://peft-internal-testing-peft-method-comparison-embed.hf.space/?highlight[type]=ADALORA" frameborder="0" width="850" height="1000"></iframe> <!> <!> <!> <!> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>This is the configuration class to store the configuration of a <a href="/docs/peft/pr_3111/en/package_reference/adalora#peft.AdaLoraModel">AdaLoraModel</a>.</p> <p>AdaLoRA has three phases defined by <code>tinit</code>, <code>tfinal</code> and <code>total_step</code>.</p> <p>The initial phase can be understood as a step for pre-training the adapters so that when reducing their rank, there
is already some information encoded that can be reduced instead of random matrices. This phase is defined by
supplying <code>tinit</code>.</p> <p>After the initial phase is over (<code>tinit</code> steps have passed) and the final phase has not begun, AdaLoRA reduces the
budget of how much rank each layer is allowed to have with each step. This is where the reduction of rank is
happening. This goes on until <code>total_step - tfinal</code> steps are reached.</p> <p>The last phase, beginning once <code>total_step - tfinal</code> steps are reached, does not change the layer ranks anymore but
fine-tunes the reduced-rank layers that resulted from the previous phase.</p> <p>A practical example: <code>tinit</code> is 10, <code>tfinal</code> is 20, <code>total_step</code> is 100. We spend 10 steps doing pre-training
without rank reduction because our budget is constant (init phase), then we spend 80 (100-20) steps in the
reduction phase where our budget decreases step-wise and, finally, 20 steps in the final fine-tuning stage without
reduction.</p></div> <!> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Creates AdaLoRA (Adaptive LoRA) model from a pretrained transformers model. Paper: <a href="https://openreview.net/forum?id=lq62uWRJjiY" rel="nofollow">https://openreview.net/forum?id=lq62uWRJjiY</a></p> <!> <p><strong>Attributes</strong>:</p> <ul><li><strong>model</strong> (<a href="https://huggingface.co/docs/transformers/main/en/main_classes/model#transformers.PreTrainedModel" rel="nofollow">transformers.PreTrainedModel</a>) — The model to be adapted.</li> <li><strong>peft_config</strong> (<a href="/docs/peft/pr_3111/en/package_reference/adalora#peft.AdaLoraConfig">AdaLoraConfig</a>): The configuration of the AdaLora model.</li></ul> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>This method is not supported for AdaLoRA, use LoRA instead.</p></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>This method updates Adalora budget and mask.</p> <p>This should be called in every training step after <code>loss.backward()</code> and before <code>zero_grad()</code>.</p> <p><code>tinit</code>, <code>tfinal</code> and <code>deltaT</code> are handled with in the method.</p> <!></div></div> <!> <p></p>`,1);function oa(W,E){Y(E,!1),S(()=>{new URLSearchParams(window.location.search).get("fw")}),O();var v=$();D("g3wh8m",e=>{var r=K();X(r,"content",P),i(e,r)});var M=a(b(v),2);z(M,{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"});var w=a(M,2);t(w,{title:"AdaLoRA",local:"adalora",headingTag:"h1"});var p=a(w,10),y=a(o(p),3);f(y,()=>'<span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mi mathvariant="normal">Δ</mi><mi>W</mi></mrow><annotation encoding="application/x-tex">\\Delta W</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.6833em;"></span><span class="mord">Δ</span><span class="mord mathnormal" style="margin-right:0.1389em;">W</span></span></span></span>');var T=a(y,2);f(T,()=>'<span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mi mathvariant="normal">Δ</mi><mi>W</mi></mrow><annotation encoding="application/x-tex">\\Delta W</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.6833em;"></span><span class="mord">Δ</span><span class="mord mathnormal" style="margin-right:0.1389em;">W</span></span></span></span>');var B=a(T,2);f(B,()=>'<span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mi mathvariant="normal">Δ</mi><mi>W</mi></mrow><annotation encoding="application/x-tex">\\Delta W</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.6833em;"></span><span class="mord">Δ</span><span class="mord mathnormal" style="margin-right:0.1389em;">W</span></span></span></span>'),d(),s(p);var A=a(p,6);t(A,{title:"Benchmark overview",local:"benchmark-overview",headingTag:"h2"});var k=a(A,4);t(k,{title:"Usage",local:"usage",headingTag:"h2"});var j=a(k,2);u(j,{code:"ZnJvbSUyMHBlZnQlMjBpbXBvcnQlMjBBZGFMb3JhQ29uZmlnJTJDJTIwZ2V0X3BlZnRfbW9kZWwlMEElMEFjb25maWclMjAlM0QlMjBBZGFMb3JhQ29uZmlnKCUwQSUyMCUyMCUyMCUyMHIlM0Q4JTJDJTBBJTIwJTIwJTIwJTIwaW5pdF9yJTNEMTIlMkMlMEElMjAlMjAlMjAlMjB0aW5pdCUzRDIwMCUyQyUwQSUyMCUyMCUyMCUyMHRmaW5hbCUzRDEwMDAlMkMlMEElMjAlMjAlMjAlMjBkZWx0YVQlM0QxMCUyQyUwQSUyMCUyMCUyMCUyMHRhcmdldF9tb2R1bGVzJTNEJTVCJTIycXVlcnklMjIlMkMlMjAlMjJ2YWx1ZSUyMiU1RCUyQyUwQSUyMCUyMCUyMCUyMG1vZHVsZXNfdG9fc2F2ZSUzRCU1QiUyMmNsYXNzaWZpZXIlMjIlNUQlMkMlMEEpJTBBbW9kZWwlMjAlM0QlMjBnZXRfcGVmdF9tb2RlbChtb2RlbCUyQyUyMGNvbmZpZyklMEFtb2RlbC5wcmludF90cmFpbmFibGVfcGFyYW1ldGVycygpJTBBJTIydHJhaW5hYmxlJTIwcGFyYW1zJTNBJTIwNTIwJTJDMzI1JTIwJTdDJTdDJTIwYWxsJTIwcGFyYW1zJTNBJTIwODclMkM2MTQlMkM3MjIlMjAlN0MlN0MlMjB0cmFpbmFibGUlMjUlM0ElMjAwLjU5Mzg3ODUwMzY2MDYwNjIlMjIlMEElMEElNUIuLi4lMjB0cmFpbmluZyUyMGNvZGUlMjAuLi4lNUQlMEElMEFtb2RlbC51cGRhdGVfYW5kX2FsbG9jYXRlKHN0ZXBfaWR4KQ==",highlighted:`<span class="hljs-keyword">from</span> peft <span class="hljs-keyword">import</span> AdaLoraConfig, get_peft_model

config = AdaLoraConfig(
    r=<span class="hljs-number">8</span>,
    init_r=<span class="hljs-number">12</span>,
    tinit=<span class="hljs-number">200</span>,
    tfinal=<span class="hljs-number">1000</span>,
    deltaT=<span class="hljs-number">10</span>,
    target_modules=[<span class="hljs-string">&quot;query&quot;</span>, <span class="hljs-string">&quot;value&quot;</span>],
    modules_to_save=[<span class="hljs-string">&quot;classifier&quot;</span>],
)
model = get_peft_model(model, config)
model.print_trainable_parameters()
<span class="hljs-string">&quot;trainable params: 520,325 || all params: 87,614,722 || trainable%: 0.5938785036606062&quot;</span>

[... training code ...]

model.update_and_allocate(step_idx)`,lang:"py",wrap:!1});var J=a(j,2);t(J,{title:"API",local:"api",headingTag:"h1"});var L=a(J,2);t(L,{title:"AdaLoraConfig",local:"peft.AdaLoraConfig",headingTag:"h2"});var c=a(L,2),F=o(c);l(F,{name:"class peft.AdaLoraConfig",anchor:"peft.AdaLoraConfig",source:"https://github.com/huggingface/peft/blob/vr_3111/src/peft/tuners/adalora/config.py#L24",parameters:[{name:"task_type",val:": Optional[Union[str, TaskType]] = None"},{name:"peft_type",val:": Optional[Union[str, PeftType]] = None"},{name:"auto_mapping",val:": Optional[dict] = None"},{name:"peft_version",val:": Optional[str] = None"},{name:"base_model_name_or_path",val:": Optional[str] = None"},{name:"revision",val:": Optional[str] = None"},{name:"inference_mode",val:": bool = False"},{name:"r",val:": int = 8"},{name:"target_modules",val:": Optional[Union[list[str], str]] = None"},{name:"exclude_modules",val:": Optional[Union[list[str], str]] = None"},{name:"lora_alpha",val:": int = 8"},{name:"lora_dropout",val:": float = 0.0"},{name:"fan_in_fan_out",val:": bool = False"},{name:"bias",val:": Literal['none', 'all', 'lora_only'] = 'none'"},{name:"use_rslora",val:": bool = False"},{name:"modules_to_save",val:": Optional[list[str]] = None"},{name:"init_lora_weights",val:": bool | Literal['gaussian', 'eva', 'olora', 'pissa', 'pissa_niter_[number of iters]', 'corda', 'loftq', 'orthogonal', 'mica'] = True"},{name:"layers_to_transform",val:": Optional[Union[list[int], int]] = None"},{name:"layers_pattern",val:": Optional[Union[list[str], str]] = None"},{name:"rank_pattern",val:": typing.Optional[dict] = None"},{name:"alpha_pattern",val:": Optional[dict] = <factory>"},{name:"megatron_config",val:": Optional[dict] = None"},{name:"megatron_core",val:": Optional[str] = 'megatron.core'"},{name:"trainable_token_indices",val:": Optional[Union[list[int], dict[str, list[int]]]] = None"},{name:"loftq_config",val:": Union[LoftQConfig, dict] = <factory>"},{name:"eva_config",val:": Optional[EvaConfig] = None"},{name:"corda_config",val:": Optional[CordaConfig] = None"},{name:"lora_ga_config",val:": Optional[LoraGAConfig] = None"},{name:"use_dora",val:": bool = False"},{name:"velora_config",val:": Optional[Union[VeloraConfig, dict]] = None"},{name:"alora_invocation_tokens",val:": Optional[list[int]] = None"},{name:"use_qalora",val:": bool = False"},{name:"qalora_group_size",val:": int = 16"},{name:"monteclora_config",val:": Optional[MontecloraConfig] = None"},{name:"layer_replication",val:": Optional[list[tuple[int, int]]] = None"},{name:"runtime_config",val:": LoraRuntimeConfig = <factory>"},{name:"lora_bias",val:": bool = False"},{name:"target_parameters",val:": Optional[list[str]] = None"},{name:"use_bdlora",val:": Optional[BdLoraConfig] = None"},{name:"arrow_config",val:": Optional[ArrowConfig] = None"},{name:"kasa_config",val:": Optional[KasaConfig] = None"},{name:"ensure_weight_tying",val:": bool = False"},{name:"target_r",val:": int = 8"},{name:"init_r",val:": int = 12"},{name:"tinit",val:": int = 0"},{name:"tfinal",val:": int = 0"},{name:"deltaT",val:": int = 1"},{name:"beta1",val:": float = 0.85"},{name:"beta2",val:": float = 0.85"},{name:"orth_reg_weight",val:": float = 0.5"},{name:"total_step",val:": typing.Optional[int] = None"}],parametersDescription:[{anchor:"peft.AdaLoraConfig.target_r",description:"<strong>target_r</strong> (<code>int</code>) &#x2014; The target average rank of incremental matrix.",name:"target_r"},{anchor:"peft.AdaLoraConfig.init_r",description:"<strong>init_r</strong> (<code>int</code>) &#x2014; The initial rank for each incremental matrix.",name:"init_r"},{anchor:"peft.AdaLoraConfig.tinit",description:"<strong>tinit</strong> (<code>int</code>) &#x2014; The steps of initial fine-tuning warmup.",name:"tinit"},{anchor:"peft.AdaLoraConfig.tfinal",description:"<strong>tfinal</strong> (<code>int</code>) &#x2014; The number of steps of final fine-tuning.",name:"tfinal"},{anchor:"peft.AdaLoraConfig.deltaT",description:"<strong>deltaT</strong> (<code>int</code>) &#x2014; The time internval between two budget allocations.",name:"deltaT"},{anchor:"peft.AdaLoraConfig.beta1",description:"<strong>beta1</strong> (<code>float</code>) &#x2014; The hyperparameter of EMA for sensitivity smoothing.",name:"beta1"},{anchor:"peft.AdaLoraConfig.beta2",description:"<strong>beta2</strong> (<code>float</code>) &#x2014; The hyperparameter of EMA for undertainty quantification.",name:"beta2"},{anchor:"peft.AdaLoraConfig.orth_reg_weight",description:"<strong>orth_reg_weight</strong> (<code>float</code>) &#x2014; The coefficient of orthogonal regularization.",name:"orth_reg_weight"},{anchor:"peft.AdaLoraConfig.total_step",description:"<strong>total_step</strong> (<code>int</code>) &#x2014; The total training steps that should be specified before training.",name:"total_step"},{anchor:"peft.AdaLoraConfig.rank_pattern",description:"<strong>rank_pattern</strong> (<code>list</code>) &#x2014; The allocated rank for each weight matrix by RankAllocator.",name:"rank_pattern"}]}),d(12),s(c);var U=a(c,2);t(U,{title:"AdaLoraModel",local:"peft.AdaLoraModel",headingTag:"h2"});var m=a(U,2),C=o(m);l(C,{name:"class peft.AdaLoraModel",anchor:"peft.AdaLoraModel",source:"https://github.com/huggingface/peft/blob/vr_3111/src/peft/tuners/adalora/model.py#L36",parameters:[{name:"model",val:""},{name:"config",val:""},{name:"adapter_name",val:""},{name:"**kwargs",val:""}],parametersDescription:[{anchor:"peft.AdaLoraModel.model",description:'<strong>model</strong> (<a href="https://huggingface.co/docs/transformers/main/en/main_classes/model#transformers.PreTrainedModel" rel="nofollow">transformers.PreTrainedModel</a>) &#x2014; The model to be adapted.',name:"model"},{anchor:"peft.AdaLoraModel.config",description:'<strong>config</strong> (<a href="/docs/peft/pr_3111/en/package_reference/adalora#peft.AdaLoraConfig">AdaLoraConfig</a>) &#x2014; The configuration of the AdaLora model.',name:"config"},{anchor:"peft.AdaLoraModel.adapter_name",description:"<strong>adapter_name</strong> (<code>str</code>) &#x2014; The name of the adapter, defaults to <code>&quot;default&quot;</code>.",name:"adapter_name"},{anchor:"peft.AdaLoraModel.low_cpu_mem_usage",description:`<strong>low_cpu_mem_usage</strong> (<code>bool</code>, <code>optional</code>, defaults to <code>False</code>) &#x2014;
Create empty adapter weights on meta device. Useful to speed up the loading process.`,name:"low_cpu_mem_usage"}],returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p>The AdaLora model.</p>
`,returnType:`<script context="module">export const metadata = 'undefined';<\/script>


<p><code>torch.nn.Module</code></p>
`});var x=a(C,4);I(x,{anchor:"peft.AdaLoraModel.example",children:(e,r)=>{var n=Z(),g=a(b(n),2);u(g,{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Nb2RlbEZvclNlcTJTZXFMTSUwQWZyb20lMjBwZWZ0JTIwaW1wb3J0JTIwQWRhTG9yYUNvbmZpZyUyQyUyMGdldF9wZWZ0X21vZGVsJTBBJTBBY29uZmlnJTIwJTNEJTIwQWRhTG9yYUNvbmZpZyglMEElMjAlMjAlMjAlMjBwZWZ0X3R5cGUlM0QlMjJBREFMT1JBJTIyJTJDJTBBJTIwJTIwJTIwJTIwdGFza190eXBlJTNEJTIyU0VRXzJfU0VRX0xNJTIyJTJDJTBBJTIwJTIwJTIwJTIwaW5pdF9yJTNEMTIlMkMlMEElMjAlMjAlMjAlMjBsb3JhX2FscGhhJTNEMzIlMkMlMEElMjAlMjAlMjAlMjB0YXJnZXRfbW9kdWxlcyUzRCU1QiUyMnElMjIlMkMlMjAlMjJ2JTIyJTVEJTJDJTBBJTIwJTIwJTIwJTIwbG9yYV9kcm9wb3V0JTNEMC4wMSUyQyUwQSUyMCUyMCUyMCUyMHRvdGFsX3N0ZXAlM0QxMDAwJTJDJTBBKSUwQW1vZGVsJTIwJTNEJTIwQXV0b01vZGVsRm9yU2VxMlNlcUxNLmZyb21fcHJldHJhaW5lZCglMjJ0NS1iYXNlJTIyKSUwQWFkYWxvcmFfbW9kZWwlMjAlM0QlMjBnZXRfcGVmdF9tb2RlbChtb2RlbCUyQyUyMGNvbmZpZyk=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForSeq2SeqLM
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> peft <span class="hljs-keyword">import</span> AdaLoraConfig, get_peft_model

<span class="hljs-meta">&gt;&gt;&gt; </span>config = AdaLoraConfig(
<span class="hljs-meta">... </span>    peft_type=<span class="hljs-string">&quot;ADALORA&quot;</span>,
<span class="hljs-meta">... </span>    task_type=<span class="hljs-string">&quot;SEQ_2_SEQ_LM&quot;</span>,
<span class="hljs-meta">... </span>    init_r=<span class="hljs-number">12</span>,
<span class="hljs-meta">... </span>    lora_alpha=<span class="hljs-number">32</span>,
<span class="hljs-meta">... </span>    target_modules=[<span class="hljs-string">&quot;q&quot;</span>, <span class="hljs-string">&quot;v&quot;</span>],
<span class="hljs-meta">... </span>    lora_dropout=<span class="hljs-number">0.01</span>,
<span class="hljs-meta">... </span>    total_step=<span class="hljs-number">1000</span>,
<span class="hljs-meta">... </span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>model = AutoModelForSeq2SeqLM.from_pretrained(<span class="hljs-string">&quot;t5-base&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>adalora_model = get_peft_model(model, config)`,lang:"py",wrap:!1}),i(e,n)},$$slots:{default:!0}});var h=a(x,6),q=o(h);l(q,{name:"add_weighted_adapter",anchor:"peft.AdaLoraModel.add_weighted_adapter",source:"https://github.com/huggingface/peft/blob/vr_3111/src/peft/tuners/adalora/model.py#L370",parameters:[{name:"*args",val:""},{name:"**kwargs",val:""}]}),d(2),s(h);var R=a(h,2),N=o(R);l(N,{name:"update_and_allocate",anchor:"peft.AdaLoraModel.update_and_allocate",source:"https://github.com/huggingface/peft/blob/vr_3111/src/peft/tuners/adalora/model.py#L328",parameters:[{name:"global_step",val:""}],parametersDescription:[{anchor:"peft.AdaLoraModel.update_and_allocate.global_step",description:"<strong>global_step</strong> (<code>int</code>) &#x2014; The current training step, it is used to calculate adalora budget.",name:"global_step"}]});var G=a(N,8);I(G,{anchor:"peft.AdaLoraModel.update_and_allocate.example",children:(e,r)=>{var n=Z(),g=a(b(n),2);u(g,{code:"bG9zcyUyMCUzRCUyMG1vZGVsKCoqaW5wdXQpLmxvc3MlMEFsb3NzLmJhY2t3YXJkKCklMEFvcHRpbWl6ZXIuc3RlcCgpJTBBbW9kZWwuYmFzZV9tb2RlbC51cGRhdGVfYW5kX2FsbG9jYXRlKGlfc3RlcCklMEFvcHRpbWl6ZXIuemVyb19ncmFkKCk=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>loss = model(**<span class="hljs-built_in">input</span>).loss
<span class="hljs-meta">&gt;&gt;&gt; </span>loss.backward()
<span class="hljs-meta">&gt;&gt;&gt; </span>optimizer.step()
<span class="hljs-meta">&gt;&gt;&gt; </span>model.base_model.update_and_allocate(i_step)
<span class="hljs-meta">&gt;&gt;&gt; </span>optimizer.zero_grad()`,lang:"python",wrap:!1}),i(e,n)},$$slots:{default:!0}}),s(R),s(m);var Q=a(m,2);V(Q,{source:"https://github.com/huggingface/peft/blob/main/docs/source/package_reference/adalora.md"}),d(2),i(W,v),H()}export{oa as component};