peft/pr_3271/en/_app/immutable/nodes/68.ebeffc19.js

import{s as fe,n as de,o as ce}from"../chunks/scheduler.78382b47.js";import{S as pe,i as me,e as r,s as o,c as w,h as ue,a as s,d as a,b as i,f as K,g as F,j as D,k as Q,l as Y,m as n,n as W,t as $,o as C,p as L}from"../chunks/index.6dd35eb6.js";import{C as ge,H as Z,E as he}from"../chunks/MermaidChart.svelte_svelte_type_style_lang.bf5c41ed.js";import{D as le}from"../chunks/Docstring.73dbdfab.js";function ve(ee){let l,M,k,P,d,O,c,E,p,te='<a href="https://huggingface.co/papers/2505.12532" rel="nofollow">WaveFT</a> is a novel parameter-efficient fine-tuning (PEFT) method that introduces sparse updates in the <strong>wavelet domain</strong> of residual matrices. Unlike LoRA, which is constrained by discrete low-rank choices, WaveFT enables fine-grained control over the number of trainable parameters by directly learning a sparse set of coefficients in the transformed space. These coefficients are then mapped back to the weight domain via the Inverse Discrete Wavelet Transform (IDWT), producing high-rank updates without incurring inference overhead.',U,m,ae="WaveFT currently has the following constraint:",N,u,ne="<li>Only <code>nn.Linear</code> layers are supported.</li>",S,g,oe="The abstract from the paper is:",H,h,ie="<p>Efficiently adapting large foundation models is critical, especially with tight compute and memory budgets. Parameter-Efficient Fine-Tuning (PEFT) methods such as LoRA offer limited granularity and effectiveness in few-parameter regimes. We propose Wavelet Fine-Tuning (WaveFT), a novel PEFT method that learns highly sparse updates in the wavelet domain of residual matrices. WaveFT allows precise control of trainable parameters, offering fine-grained capacity adjustment and excelling with remarkably low parameter count, potentially far fewer than LoRA’s minimum—ideal for extreme parameter-efficient scenarios. Evaluated on personalized text-to-image generation using Stable Diffusion XL as baseline, WaveFT significantly outperforms LoRA and other PEFT methods, especially at low parameter counts; achieving superior subject fidelity, prompt alignment, and image diversity.</p>",z,v,A,f,_,J,q,re=`This is the configuration class to store the configuration of a <a href="/docs/peft/pr_3271/en/package_reference/waveft#peft.WaveFTModel">WaveFTModel</a>. It is used to define the
parameters for Wavelet-based Fine-Tuning (WaveFT), an approach that leverages the sparsity of wavelet transforms
for parameter-efficient fine-tuning of pretrained models.`,j,y,R,T,x,X,b,B,I,G;return d=new ge({props:{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"}}),c=new Z({props:{title:"WaveFT: Wavelet Fine-Tuning",local:"waveft-wavelet-fine-tuning",headingTag:"h1"}}),v=new Z({props:{title:"WaveFTConfig",local:"peft.WaveFTConfig",headingTag:"h2"}}),_=new le({props:{name:"class peft.WaveFTConfig",anchor:"peft.WaveFTConfig",parameters:[{name:"task_type",val:": Optional[Union[str, TaskType]] = None"},{name:"peft_type",val:": Optional[Union[str, PeftType]] = None"},{name:"auto_mapping",val:": Optional[dict] = None"},{name:"peft_version",val:": Optional[str] = None"},{name:"base_model_name_or_path",val:": Optional[str] = None"},{name:"revision",val:": Optional[str] = None"},{name:"inference_mode",val:": bool = False"},{name:"n_frequency",val:": int = 2592"},{name:"scaling",val:": float = 25.0"},{name:"wavelet_family",val:": str = 'db1'"},{name:"use_idwt",val:": bool = True"},{name:"random_loc_seed",val:": int = 777"},{name:"fan_in_fan_out",val:": bool = False"},{name:"target_modules",val:": Optional[Union[list[str], str]] = None"},{name:"exclude_modules",val:": Optional[Union[list[str], str]] = None"},{name:"bias",val:": str = 'none'"},{name:"modules_to_save",val:": Optional[list[str]] = None"},{name:"layers_to_transform",val:": Optional[Union[list[int], int]] = None"},{name:"layers_pattern",val:": Optional[Union[list[str], str]] = None"},{name:"n_frequency_pattern",val:": Optional[dict] = <factory>"},{name:"proportional_parameters",val:": bool = False"},{name:"init_weights",val:": bool = True"}],parametersDescription:[{anchor:"peft.WaveFTConfig.n_frequency",description:`<strong>n_frequency</strong> (<code>int</code>) &#x2014;
Number of learnable wavelet coefficients for the Discrete Wavelet Transform (DWT). &#x2018;n_frequency&#x2019; is an
integer that is greater than 0 and less than or equal to the total number of elements in the original
weight matrix (d_out * d_in). This parameter directly controls the number of trainable parameters for each
adapted layer. A higher &#x2018;n_frequency&#x2019; generally leads to better performance but also increases GPU memory
usage, with a minor impact on training speed.`,name:"n_frequency"},{anchor:"peft.WaveFTConfig.scaling",description:`<strong>scaling</strong> (<code>float</code>) &#x2014;
The scaling factor applied to the reconstructed delta W matrix. This is a crucial hyperparameter, analogous
to <code>lora_alpha</code> in LoRA. It can be tuned during hyperparameter search. Our default value for SDXL
personalization is 25.`,name:"scaling"},{anchor:"peft.WaveFTConfig.wavelet_family",description:`<strong>wavelet_family</strong> (<code>str</code>) &#x2014;
The wavelet family (e.g., &#x2018;db1&#x2019;, &#x2018;sym2&#x2019;, &#x2018;coif1&#x2019;) to use for the DWT and Inverse DWT (IDWT). Defaults to
&#x2018;db1&#x2019; (Haar wavelet). Different wavelet families have varying filter lengths which affect the training time
substantially`,name:"wavelet_family"},{anchor:"peft.WaveFTConfig.use_idwt",description:`<strong>use_idwt</strong> (<code>bool</code>) &#x2014;
Set to False for efficient adaptation. Whether to use the Inverse Discrete Wavelet Transform (IDWT) to
reconstruct the delta weights from the learned wavelet coefficients. If <code>True</code> (default), the IDWT is
applied. If <code>False</code>, the learned coefficients are directly used to form a sparse delta weight matrix, which
is faster but performs worse for the SDXL personalization task.`,name:"use_idwt"},{anchor:"peft.WaveFTConfig.random_loc_seed",description:`<strong>random_loc_seed</strong> (<code>int</code>) &#x2014;
Seed for determining the random locations of the <code>n_frequency</code> learnable wavelet coefficients within the
full wavelet coefficient matrix.`,name:"random_loc_seed"},{anchor:"peft.WaveFTConfig.target_modules",description:`<strong>target_modules</strong> (<code>Union[list[str],str]</code>) &#x2014;
List of module names or a regex expression identifying the modules to be adapted with WaveFT. For example,
<code>[&apos;q_proj&apos;, &apos;v_proj&apos;]</code> or <code>&apos;.*decoder.*(SelfAttention|EncDecAttention).*(q|v)$&apos;</code>. Currently, only linear
layers (<code>torch.nn.Linear</code>) are supported.`,name:"target_modules"},{anchor:"peft.WaveFTConfig.exclude_modules",description:`<strong>exclude_modules</strong> (<code>Optional[Union[List[str], str]]</code>) &#x2014;
List of module names or a regex expression for modules to exclude from WaveFT adaptation.`,name:"exclude_modules"},{anchor:"peft.WaveFTConfig.fan_in_fan_out",description:`<strong>fan_in_fan_out</strong> (<code>bool</code>) &#x2014;
Set to <code>True</code> if the weights of the layer to be replaced are stored in <code>(fan_in, fan_out)</code> format. Default
is <code>False</code>.`,name:"fan_in_fan_out"},{anchor:"peft.WaveFTConfig.bias",description:`<strong>bias</strong> (<code>str</code>) &#x2014;
Bias type for WaveFT. Can be &#x2018;none&#x2019;, &#x2018;all&#x2019;, or &#x2018;waveft_only&#x2019;. (&#x2018;fourier_only&#x2019; was likely a typo and has
been corrected to &#x2018;waveft_only&#x2019; if it implies bias only on adapted parameters) If &#x2018;waveft_only&#x2019;, biases are
added only to the WaveFT components. If &#x2018;all&#x2019;, biases are added to both base and WaveFT components. If
&#x2018;none&#x2019;, no new biases are added.`,name:"bias"},{anchor:"peft.WaveFTConfig.modules_to_save",description:`<strong>modules_to_save</strong> (<code>list[str]</code>) &#x2014;
List of modules, in addition to WaveFT layers, that should be marked as trainable and saved in the final
checkpoint. Useful for layers like classifiers in sequence or token classification tasks that are randomly
initialized and need training.`,name:"modules_to_save"},{anchor:"peft.WaveFTConfig.layers_to_transform",description:`<strong>layers_to_transform</strong> (<code>Union[list[int],int]</code>) &#x2014;
Specific layer indices to transform. If provided, PEFT will only adapt layers at these indices. If a single
integer is given, only that layer is transformed.`,name:"layers_to_transform"},{anchor:"peft.WaveFTConfig.layers_pattern",description:`<strong>layers_pattern</strong> (<code>Optional[Union[List[str], str]]</code>) &#x2014;
Pattern for layer names, used if <code>layers_to_transform</code> is specified and the layer pattern is not standard
(e.g., not &#x2018;layers&#x2019; or &#x2018;h&#x2019;). This should target the <code>nn.ModuleList</code> attribute in the model.`,name:"layers_pattern"},{anchor:"peft.WaveFTConfig.n_frequency_pattern",description:`<strong>n_frequency_pattern</strong> (<code>dict</code>) &#x2014;
A dictionary mapping layer names (or regex) to specific <code>n_frequency</code> values, overriding the global
<code>n_frequency</code>. Example: <code>{&quot;model.decoder.layers.0.encoder_attn.k_proj&quot;: 1000}</code>.`,name:"n_frequency_pattern"},{anchor:"peft.WaveFTConfig.init_weights",description:`<strong>init_weights</strong> (<code>bool</code>) &#x2014;
Initialization strategy for the learnable wavelet coefficients (spectrum). If <code>True</code> (default),
coefficients are initialized to zeros. If <code>False</code>, coefficients are initialized from a standard normal
distribution scaled by a small factor.`,name:"init_weights"},{anchor:"peft.WaveFTConfig.proportional_parameters",description:`<strong>proportional_parameters</strong> (<code>bool</code>) &#x2014;
If <code>True</code>, <code>n_frequency</code> is allocated proportionally to each layer&#x2019;s <code>input_dim * output_dim</code>. Default is
<code>False</code>. Note: This option is included for experimental thoroughness to allow researchers to reproduce
paper results, rather than for practical utility, as no beneficial scenarios have been identified.`,name:"proportional_parameters"}],source:"https://github.com/huggingface/peft/blob/vr_3271/src/peft/tuners/waveft/config.py#L27"}}),y=new Z({props:{title:"WaveFTModel",local:"peft.WaveFTModel",headingTag:"h2"}}),x=new le({props:{name:"class peft.WaveFTModel",anchor:"peft.WaveFTModel",parameters:[{name:"model",val:""},{name:"peft_config",val:": Union[PeftConfig, dict[str, PeftConfig]]"},{name:"adapter_name",val:": str"},{name:"low_cpu_mem_usage",val:": bool = False"},{name:"state_dict",val:": Optional[dict[str, torch.Tensor]] = None"}],source:"https://github.com/huggingface/peft/blob/vr_3271/src/peft/tuners/waveft/model.py#L30"}}),b=new he({props:{source:"https://github.com/huggingface/peft/blob/main/docs/source/package_reference/waveft.md"}}),{c(){l=r("meta"),M=o(),k=r("p"),P=o(),w(d.$$.fragment),O=o(),w(c.$$.fragment),E=o(),p=r("p"),p.innerHTML=te,U=o(),m=r("p"),m.textContent=ae,N=o(),u=r("ul"),u.innerHTML=ne,S=o(),g=r("p"),g.textContent=oe,H=o(),h=r("blockquote"),h.innerHTML=ie,z=o(),w(v.$$.fragment),A=o(),f=r("div"),w(_.$$.fragment),J=o(),q=r("p"),q.innerHTML=re,j=o(),w(y.$$.fragment),R=o(),T=r("div"),w(x.$$.fragment),X=o(),w(b.$$.fragment),B=o(),I=r("p"),this.h()},l(e){const t=ue("svelte-u9bgzb",document.head);l=s(t,"META",{name:!0,content:!0}),t.forEach(a),M=i(e),k=s(e,"P",{}),K(k).forEach(a),P=i(e),F(d.$$.fragment,e),O=i(e),F(c.$$.fragment,e),E=i(e),p=s(e,"P",{"data-svelte-h":!0}),D(p)!=="svelte-10i400t"&&(p.innerHTML=te),U=i(e),m=s(e,"P",{"data-svelte-h":!0}),D(m)!=="svelte-1vqytii"&&(m.textContent=ae),N=i(e),u=s(e,"UL",{"data-svelte-h":!0}),D(u)!=="svelte-18w7elh"&&(u.innerHTML=ne),S=i(e),g=s(e,"P",{"data-svelte-h":!0}),D(g)!=="svelte-1cwsb16"&&(g.textContent=oe),H=i(e),h=s(e,"BLOCKQUOTE",{"data-svelte-h":!0}),D(h)!=="svelte-17ky0yq"&&(h.innerHTML=ie),z=i(e),F(v.$$.fragment,e),A=i(e),f=s(e,"DIV",{class:!0});var V=K(f);F(_.$$.fragment,V),J=i(V),q=s(V,"P",{"data-svelte-h":!0}),D(q)!=="svelte-1dtik3c"&&(q.innerHTML=re),V.forEach(a),j=i(e),F(y.$$.fragment,e),R=i(e),T=s(e,"DIV",{class:!0});var se=K(T);F(x.$$.fragment,se),se.forEach(a),X=i(e),F(b.$$.fragment,e),B=i(e),I=s(e,"P",{}),K(I).forEach(a),this.h()},h(){Q(l,"name","hf:doc:metadata"),Q(l,"content",_e),Q(f,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),Q(T,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8")},m(e,t){Y(document.head,l),n(e,M,t),n(e,k,t),n(e,P,t),W(d,e,t),n(e,O,t),W(c,e,t),n(e,E,t),n(e,p,t),n(e,U,t),n(e,m,t),n(e,N,t),n(e,u,t),n(e,S,t),n(e,g,t),n(e,H,t),n(e,h,t),n(e,z,t),W(v,e,t),n(e,A,t),n(e,f,t),W(_,f,null),Y(f,J),Y(f,q),n(e,j,t),W(y,e,t),n(e,R,t),n(e,T,t),W(x,T,null),n(e,X,t),W(b,e,t),n(e,B,t),n(e,I,t),G=!0},p:de,i(e){G||($(d.$$.fragment,e),$(c.$$.fragment,e),$(v.$$.fragment,e),$(_.$$.fragment,e),$(y.$$.fragment,e),$(x.$$.fragment,e),$(b.$$.fragment,e),G=!0)},o(e){C(d.$$.fragment,e),C(c.$$.fragment,e),C(v.$$.fragment,e),C(_.$$.fragment,e),C(y.$$.fragment,e),C(x.$$.fragment,e),C(b.$$.fragment,e),G=!1},d(e){e&&(a(M),a(k),a(P),a(O),a(E),a(p),a(U),a(m),a(N),a(u),a(S),a(g),a(H),a(h),a(z),a(A),a(f),a(j),a(R),a(T),a(X),a(B),a(I)),a(l),L(d,e),L(c,e),L(v,e),L(_),L(y,e),L(x),L(b,e)}}}const _e='{"title":"WaveFT: Wavelet Fine-Tuning","local":"waveft-wavelet-fine-tuning","sections":[{"title":"WaveFTConfig","local":"peft.WaveFTConfig","sections":[],"depth":2},{"title":"WaveFTModel","local":"peft.WaveFTModel","sections":[],"depth":2}],"depth":1}';function ye(ee){return ce(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Fe extends pe{constructor(l){super(),me(this,l,ye,ve,fe,{})}}export{Fe as component};