Add files using upload-large-folder tool

lib/python3.12/site-packages/peft/tuners/adalora/__init__.py

@@ -0,0 +1,43 @@
+# Copyright 2023-present the HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from peft.import_utils import is_bnb_4bit_available, is_bnb_available
+from peft.utils import register_peft_method
+
+from .config import AdaLoraConfig
+from .gptq import SVDQuantLinear
+from .layer import AdaLoraLayer, RankAllocator, SVDLinear
+from .model import AdaLoraModel
+
+
+__all__ = ["AdaLoraConfig", "AdaLoraLayer", "AdaLoraModel", "RankAllocator", "SVDLinear", "SVDQuantLinear"]
+
+
+register_peft_method(
+    name="adalora", config_cls=AdaLoraConfig, model_cls=AdaLoraModel, prefix="lora_", is_mixed_compatible=True
+)
+
+
+def __getattr__(name):
+    if (name == "SVDLinear8bitLt") and is_bnb_available():
+        from .bnb import SVDLinear8bitLt
+
+        return SVDLinear8bitLt
+
+    if (name == "SVDLinear4bit") and is_bnb_4bit_available():
+        from .bnb import SVDLinear4bit
+
+        return SVDLinear4bit
+
+    raise AttributeError(f"module {__name__} has no attribute {name}")

lib/python3.12/site-packages/peft/tuners/adalora/bnb.py

@@ -0,0 +1,143 @@
+# Copyright 2023-present the HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import Any
+
+import torch
+
+from peft.import_utils import is_bnb_4bit_available, is_bnb_available
+
+from .layer import AdaLoraLayer
+
+
+if is_bnb_available():
+
+    class SVDLinear8bitLt(torch.nn.Module, AdaLoraLayer):
+        # Low-rank matrix for SVD-based adaptation
+        def __init__(
+            self,
+            base_layer: torch.nn.Module,
+            adapter_name: str,
+            r: int = 0,
+            lora_alpha: int = 1,
+            lora_dropout: float = 0.0,
+            init_lora_weights: bool = True,
+            **kwargs,
+        ) -> None:
+            super().__init__()
+            AdaLoraLayer.__init__(self, base_layer)
+            # Freezing the pre-trained weight matrix
+            self.get_base_layer().weight.requires_grad = False
+
+            self._active_adapter = adapter_name
+            self.update_layer(adapter_name, r, lora_alpha, lora_dropout, init_lora_weights)
+
+        def forward(self, x: torch.Tensor) -> torch.Tensor:
+            # note: no check for self.merged because merging is not supported (yet)
+            result = self.base_layer(x)
+
+            if self.disable_adapters:
+                return result
+
+            for active_adapter in self.active_adapters:
+                if active_adapter not in self.lora_A.keys():
+                    continue
+                requires_conversion = not torch.is_autocast_enabled()
+                if requires_conversion:
+                    expected_dtype = result.dtype
+                    if x.dtype != torch.float32:
+                        x = x.float()
+
+                lora_A = self.lora_A[active_adapter]
+                lora_B = self.lora_B[active_adapter]
+                lora_E = self.lora_E[active_adapter]
+                dropout = self.lora_dropout[active_adapter]
+                scaling = self.scaling[active_adapter]
+                ranknum = self.ranknum[active_adapter] + 1e-5
+
+                output = dropout(x) @ (lora_A * lora_E).T @ lora_B.T
+                if requires_conversion:
+                    output = output.to(expected_dtype)
+                output = output * scaling / ranknum
+                # inplace operation on view is forbidden for MatMul8bitLtBackward, so avoid it
+                result = result + output
+            return result
+
+        def __repr__(self) -> str:
+            rep = super().__repr__()
+            return "adalora." + rep
+
+
+if is_bnb_4bit_available():
+
+    class SVDLinear4bit(torch.nn.Module, AdaLoraLayer):
+        # Low-rank matrix for SVD-based adaptation
+        def __init__(
+            self,
+            base_layer: torch.nn.Module,
+            adapter_name: str,
+            r: int = 0,
+            lora_alpha: int = 1,
+            lora_dropout: float = 0.0,
+            init_lora_weights: bool = True,
+            **kwargs,
+        ) -> None:
+            super().__init__()
+            AdaLoraLayer.__init__(self, base_layer)
+            # Freezing the pre-trained weight matrix
+            self.get_base_layer().weight.requires_grad = False
+
+            self._active_adapter = adapter_name
+            self.update_layer(adapter_name, r, lora_alpha, lora_dropout, init_lora_weights)
+
+        def forward(self, x: torch.Tensor, *args: Any, **kwargs: Any) -> torch.Tensor:
+            # note: no check for self.merged because merging is not supported (yet)
+            result = self.base_layer(x, *args, **kwargs)
+
+            if self.disable_adapters:
+                return result
+
+            # As per Tim Dettmers, for 4bit, we need to defensively clone here.
+            # The reason is that in some cases, an error can occur that backprop
+            # does not work on a manipulated view. This issue may be solved with
+            # newer PyTorch versions but this would need extensive testing to be
+            # sure.
+            result = result.clone()
+
+            for active_adapter in self.active_adapters:
+                if active_adapter not in self.lora_A.keys():
+                    continue
+
+                lora_A = self.lora_A[active_adapter]
+                lora_B = self.lora_B[active_adapter]
+                lora_E = self.lora_E[active_adapter]
+                dropout = self.lora_dropout[active_adapter]
+                scaling = self.scaling[active_adapter]
+                ranknum = self.ranknum[active_adapter] + 1e-5
+
+                requires_conversion = not torch.is_autocast_enabled()
+                if requires_conversion:
+                    expected_dtype = result.dtype
+                    x = self._cast_input_dtype(x, lora_A.dtype)
+
+                output = dropout(x) @ (lora_A * lora_E).T @ lora_B.T
+                if requires_conversion:
+                    output = output.to(expected_dtype)
+                output = output * scaling / ranknum
+                result += output
+            return result
+
+        def __repr__(self) -> str:
+            rep = super().__repr__()
+            return "adalora." + rep

lib/python3.12/site-packages/peft/tuners/adalora/config.py

@@ -0,0 +1,108 @@
+# Copyright 2023-present the HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import warnings
+from dataclasses import dataclass, field
+from typing import Optional
+
+from peft.tuners.lora import LoraConfig
+from peft.utils import PeftType
+
+
+@dataclass
+class AdaLoraConfig(LoraConfig):
+    """
+    This is the configuration class to store the configuration of a [`~peft.AdaLora`].
+
+    AdaLoRA has three phases defined by `tinit`, `tfinal` and `total_step`.
+
+    The initial phase can be understood as a step for pre-training the adapters so that when reducing their rank, there
+    is already some information encoded that can be reduced instead of random matrices. This phase is defined by
+    supplying `tinit`.
+
+    After the initial phase is over (`tinit` steps have passed) and the final phase has not begun, AdaLoRA reduces the
+    budget of how much rank each layer is allowed to have with each step. This is where the reduction of rank is
+    happening. This goes on until `total_step - tfinal` steps are reached.
+
+    The last phase, beginning once `total_step - tfinal` steps are reached, does not change the layer ranks anymore but
+    fine-tunes the reduced-rank layers that resulted from the previous phase.
+
+    A practical example: `tinit` is 10, `tfinal` is 20, `total_step` is 100. We spend 10 steps doing pre-training
+    without rank reduction because our budget is constant (init phase), then we spend 80 (100-20) steps in the
+    reduction phase where our budget decreases step-wise and, finally, 20 steps in the final fine-tuning stage without
+    reduction.
+
+    Args:
+        target_r (`int`): The target average rank of incremental matrix.
+        init_r (`int`): The initial rank for each incremental matrix.
+        tinit (`int`): The steps of initial fine-tuning warmup.
+        tfinal (`int`): The number of steps of final fine-tuning.
+        deltaT (`int`): The time internval between two budget allocations.
+        beta1 (`float`): The hyperparameter of EMA for sensitivity smoothing.
+        beta2 (`float`): The hyperparameter of EMA for undertainty quantification.
+        orth_reg_weight (`float`): The coefficient of orthogonal regularization.
+        total_step (`int`): The total training steps that should be specified before training.
+        rank_pattern (`list`): The allocated rank for each weight matrix by RankAllocator.
+    """
+
+    target_r: int = field(default=8, metadata={"help": "Target Lora matrix dimension."})
+    init_r: int = field(default=12, metadata={"help": "Initial Lora matrix dimension."})
+    tinit: int = field(default=0, metadata={"help": "The steps of initial warmup."})
+    tfinal: int = field(default=0, metadata={"help": "The steps of final warmup."})
+    deltaT: int = field(default=1, metadata={"help": "Step interval of rank allocation."})
+    beta1: float = field(default=0.85, metadata={"help": "Hyperparameter of EMA."})
+    beta2: float = field(default=0.85, metadata={"help": "Hyperparameter of EMA."})
+    orth_reg_weight: float = field(default=0.5, metadata={"help": "The orthogonal regularization coefficient."})
+    total_step: Optional[int] = field(default=None, metadata={"help": "The total training steps."})
+    rank_pattern: Optional[dict] = field(default=None, metadata={"help": "The saved rank pattern."})
+
+    def __post_init__(self):
+        super().__post_init__()
+        self.peft_type = PeftType.ADALORA
+
+        if self.use_dora:
+            raise ValueError(f"{self.peft_type} does not support DoRA.")
+
+        if self.loftq_config:
+            raise ValueError(f"{self.peft_type} does not support LOFTQ.")
+
+        self.target_modules = (
+            set(self.target_modules) if isinstance(self.target_modules, list) else self.target_modules
+        )
+        self.exclude_modules = (
+            set(self.exclude_modules) if isinstance(self.exclude_modules, list) else self.exclude_modules
+        )
+        # if target_modules is a regex expression, then layers_to_transform should be None
+        if isinstance(self.target_modules, str) and self.layers_to_transform is not None:
+            raise ValueError("`layers_to_transform` cannot be used when `target_modules` is a str.")
+
+        # check for layers_to_transform and layers_pattern
+        if self.layers_pattern and not self.layers_to_transform:
+            raise ValueError("When `layers_pattern` is specified, `layers_to_transform` must also be specified. ")
+
+        # Check if 'r' has been set to a non-default value
+        if self.r != 8:  # 8 is the default value for 'r' in LoraConfig
+            warnings.warn(
+                "Note that `r` is not used in AdaLora and will be ignored."
+                "If you intended to set the initial rank, use `init_r` instead."
+            )
+
+        if self.total_step is None or self.total_step <= 0:
+            raise ValueError("AdaLoRA does not work when `total_step` is None, supply a value > 0.")
+
+        if self.tinit >= (self.total_step - self.tfinal):
+            raise ValueError(
+                "The supplied schedule values don't allow for a budgeting phase. Decrease `tfinal`/`tinit` or "
+                "increase `total_step`."
+            )

lib/python3.12/site-packages/peft/tuners/adalora/gptq.py

@@ -0,0 +1,71 @@
+# Copyright 2023-present the HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import torch
+
+from .layer import AdaLoraLayer
+
+
+class SVDQuantLinear(torch.nn.Module, AdaLoraLayer):
+    def __init__(
+        self,
+        base_layer,
+        adapter_name,
+        r: int = 0,
+        lora_alpha: int = 1,
+        lora_dropout: float = 0.0,
+        init_lora_weights: bool = True,
+        **kwargs,
+    ) -> None:
+        super().__init__()
+        AdaLoraLayer.__init__(self, base_layer)
+
+        # self.base_layer and self.quant_linear_module are the same; we need the former for consistency and the latter
+        # for backwards compatibility
+        self.quant_linear_module = base_layer
+        self._active_adapter = adapter_name
+        self.update_layer(adapter_name, r, lora_alpha, lora_dropout, init_lora_weights)
+
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        result = self.quant_linear_module(x)
+
+        if self.disable_adapters:
+            return result
+
+        for active_adapter in self.active_adapters:
+            if active_adapter not in self.lora_A.keys():
+                continue
+            lora_A = self.lora_A[active_adapter]
+            lora_B = self.lora_B[active_adapter]
+            lora_E = self.lora_E[active_adapter]
+            dropout = self.lora_dropout[active_adapter]
+            scaling = self.scaling[active_adapter]
+            ranknum = self.ranknum[active_adapter] + 1e-5
+
+            requires_conversion = not torch.is_autocast_enabled()
+            if requires_conversion:
+                expected_dtype = result.dtype
+                x = self._cast_input_dtype(x, torch.float32)
+
+            output = (dropout(x) @ (lora_A * lora_E).T @ lora_B.T) * scaling / ranknum
+            # TODO: here, the dtype conversion is applied on the *whole expression*,
+            # not the intermediate result, unlike for SVDLinear8bitLT and
+            # SVDLinear4bit, is that correct?
+            if requires_conversion:
+                output = output.to(expected_dtype)
+            result += output
+        return result
+
+        def __repr__(self) -> str:
+            rep = super().__repr__()
+            return "adalora." + rep

lib/python3.12/site-packages/peft/tuners/adalora/layer.py

@@ -0,0 +1,358 @@
+# Copyright 2023-present the HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import warnings
+from typing import Any, List, Optional
+
+import packaging
+import torch
+import transformers
+from torch import nn
+
+from peft.tuners.lora import LoraLayer
+from peft.tuners.tuners_utils import check_adapters_to_merge
+from peft.utils import transpose
+
+
+if packaging.version.parse(transformers.__version__) >= packaging.version.parse("4.33.0"):
+    from transformers.integrations import deepspeed_config
+else:
+    from transformers.deepspeed import deepspeed_config
+
+
+class AdaLoraLayer(LoraLayer):
+    # List all names of layers that may contain adapter weights
+    # Note: ranknum doesn't need to be included as it is not an nn.Module
+    adapter_layer_names = ("lora_A", "lora_B", "lora_E", "lora_embedding_A", "lora_embedding_B")
+    # All names of other parameters that may contain adapter-related parameters
+    other_param_names = ("r", "lora_alpha", "scaling", "lora_dropout", "ranknum")
+
+    def __init__(self, base_layer: nn.Module) -> None:
+        super().__init__(base_layer)
+        self.lora_E = nn.ParameterDict({})
+        self.lora_A = nn.ParameterDict({})
+        self.lora_B = nn.ParameterDict({})
+        self.ranknum = nn.ParameterDict({})
+
+    def update_layer(self, adapter_name, r, lora_alpha, lora_dropout, init_lora_weights):
+        if r < 0:
+            # note: r == 0 is allowed for AdaLora, see #1539
+            raise ValueError(f"`r` should be a positive integer or 0, but the value passed is {r}")
+
+        self.r[adapter_name] = r
+        self.lora_alpha[adapter_name] = lora_alpha
+        if lora_dropout > 0.0:
+            lora_dropout_layer = nn.Dropout(p=lora_dropout)
+        else:
+            lora_dropout_layer = nn.Identity()
+
+        self.lora_dropout[adapter_name] = lora_dropout_layer
+        # Actual trainable parameters
+        # Right singular vectors
+        self.lora_A[adapter_name] = nn.Parameter(torch.randn(r, self.in_features))
+        # Singular values
+        self.lora_E[adapter_name] = nn.Parameter(torch.randn(r, 1))
+        # Left singular vectors
+        self.lora_B[adapter_name] = nn.Parameter(torch.randn(self.out_features, r))
+        # The current rank
+        self.ranknum[adapter_name] = nn.Parameter(torch.randn(1), requires_grad=False)
+        self.ranknum[adapter_name].data.fill_(float(r))
+        self.ranknum[adapter_name].requires_grad = False
+        self.scaling[adapter_name] = lora_alpha if lora_alpha > 0 else float(r)
+        if init_lora_weights:
+            self.reset_lora_parameters(adapter_name)
+
+        self._move_adapter_to_device_of_base_layer(adapter_name)
+        self.set_adapter(self.active_adapters)
+
+    def reset_lora_parameters(self, adapter_name):
+        if adapter_name in self.lora_A.keys():
+            nn.init.zeros_(self.lora_E[adapter_name])
+            nn.init.normal_(self.lora_A[adapter_name], mean=0.0, std=0.02)
+            nn.init.normal_(self.lora_B[adapter_name], mean=0.0, std=0.02)
+
+
+class SVDLinear(nn.Module, AdaLoraLayer):
+    # SVD-based adaptation by a dense layer
+    def __init__(
+        self,
+        base_layer: nn.Module,
+        adapter_name: str,
+        r: int = 0,
+        lora_alpha: int = 1,
+        lora_dropout: float = 0.0,
+        fan_in_fan_out: bool = False,
+        init_lora_weights: bool = True,
+        **kwargs,
+    ) -> None:
+        super().__init__()
+        AdaLoraLayer.__init__(self, base_layer)
+        # Freezing the pre-trained weight matrix
+        self.get_base_layer().weight.requires_grad = False
+
+        self.fan_in_fan_out = fan_in_fan_out
+        self._active_adapter = adapter_name
+        self.update_layer(adapter_name, r, lora_alpha, lora_dropout, init_lora_weights)
+
+    def merge(self, safe_merge: bool = False, adapter_names: Optional[List[str]] = None) -> None:
+        """
+        Merge the active adapter weights into the base weights
+
+        Args:
+            safe_merge (`bool`, *optional*):
+                If True, the merge operation will be performed in a copy of the original weights and check for NaNs
+                before merging the weights. This is useful if you want to check if the merge operation will produce
+                NaNs. Defaults to `False`.
+            adapter_names (`List[str]`, *optional*):
+                The list of adapter names that should be merged. If None, all active adapters will be merged. Defaults
+                to `None`.
+        """
+        adapter_names = check_adapters_to_merge(self, adapter_names)
+        if not adapter_names:
+            # no adapter to merge
+            return
+
+        for active_adapter in adapter_names:
+            base_layer = self.get_base_layer()
+            if active_adapter in self.lora_A.keys():
+                if safe_merge:
+                    # Note that safe_merge will be slower than the normal merge
+                    # because of the copy operation.
+                    orig_weights = base_layer.weight.data.clone()
+                    orig_weights += self.get_delta_weight(active_adapter)
+
+                    if not torch.isfinite(orig_weights).all():
+                        raise ValueError(
+                            f"NaNs detected in the merged weights. The adapter {active_adapter} seems to be broken"
+                        )
+
+                    base_layer.weight.data = orig_weights
+                else:
+                    base_layer.weight.data += self.get_delta_weight(active_adapter)
+                self.merged_adapters.append(active_adapter)
+
+    def unmerge(self) -> None:
+        """
+        This method unmerges all merged adapter layers from the base weights.
+        """
+        if not self.merged:
+            warnings.warn("Already unmerged. Nothing to do.")
+            return
+        while len(self.merged_adapters) > 0:
+            active_adapter = self.merged_adapters.pop()
+            if active_adapter in self.lora_A.keys():
+                self.get_base_layer().weight.data -= self.get_delta_weight(active_adapter)
+
+    def get_delta_weight(self, adapter) -> torch.Tensor:
+        return (
+            transpose(self.lora_B[adapter] @ (self.lora_A[adapter] * self.lora_E[adapter]), self.fan_in_fan_out)
+            * self.scaling[adapter]
+            / (self.ranknum[adapter] + 1e-5)
+        )
+
+    def forward(self, x: torch.Tensor, *args: Any, **kwargs: Any) -> torch.Tensor:
+        if self.disable_adapters:
+            if self.merged:
+                self.unmerge()
+            result = self.base_layer(x, *args, **kwargs)
+        elif self.merged:
+            result = self.base_layer(x, *args, **kwargs)
+        else:
+            result = self.base_layer(x, *args, **kwargs)
+            for active_adapter in self.active_adapters:
+                if active_adapter not in self.lora_A.keys():
+                    continue
+                lora_A = self.lora_A[active_adapter]
+                lora_B = self.lora_B[active_adapter]
+                lora_E = self.lora_E[active_adapter]
+                dropout = self.lora_dropout[active_adapter]
+                scaling = self.scaling[active_adapter]
+                ranknum = self.ranknum[active_adapter] + 1e-5
+
+                x = self._cast_input_dtype(x, lora_A.dtype)
+                result += (dropout(x) @ (lora_A * lora_E).T @ lora_B.T) * scaling / ranknum
+
+        return result
+
+    def __repr__(self) -> str:
+        rep = super().__repr__()
+        return "adalora." + rep
+
+
+class RankAllocator:
+    """
+    The RankAllocator for AdaLoraModel. Paper: https://openreview.net/pdf?id=lq62uWRJjiY
+
+    Args:
+        config ([`AdaLoraConfig`]): The configuration of the AdaLora model.
+        model: the model that we apply AdaLoRA to.
+
+    """
+
+    def __init__(self, model, peft_config, adapter_name):
+        self.peft_config = peft_config
+        self.adapter_name = adapter_name
+        self.beta1 = peft_config.beta1
+        self.beta2 = peft_config.beta2
+        assert self.beta1 > 0 and self.beta1 < 1
+        assert self.beta2 > 0 and self.beta2 < 1
+
+        self.reset_ipt()
+        self._set_budget_scheduler(model)
+
+    def set_total_step(self, total_step):
+        self.peft_config.total_step = total_step
+
+    def reset_ipt(self):
+        self.ipt = {}
+        self.exp_avg_ipt = {}
+        self.exp_avg_unc = {}
+
+    def _set_budget_scheduler(self, model):
+        self.init_bgt = 0
+        self.name_set = set()
+        for n, p in model.named_parameters():
+            if f"lora_A.{self.adapter_name}" in n:
+                self.init_bgt += p.size(0)
+                self.name_set.add(n.replace("lora_A", "%s"))
+        self.name_set = sorted(self.name_set)
+        # The total final rank budget
+        self.target_bgt = self.peft_config.target_r * len(self.name_set)
+
+    def budget_schedule(self, step: int):
+        tinit = self.peft_config.tinit
+        tfinal = self.peft_config.tfinal
+        total_step = self.peft_config.total_step
+        # Initial warmup
+        if step <= tinit:
+            budget = self.init_bgt
+            mask_ind = False
+        # Final fine-tuning
+        elif step > total_step - tfinal:
+            budget = self.target_bgt
+            mask_ind = True
+        else:
+            # Budget decreasing with a cubic scheduler
+            mul_coeff = 1 - (step - tinit) / (total_step - tfinal - tinit)
+            budget = int((self.init_bgt - self.target_bgt) * (mul_coeff**3) + self.target_bgt)
+            mask_ind = True if step % self.peft_config.deltaT == 0 else False
+        return budget, mask_ind
+
+    def update_ipt(self, model):
+        # Update the sensitivity and uncertainty for every weight
+        for n, p in model.named_parameters():
+            if "lora_" in n and self.adapter_name in n:
+                if n not in self.ipt:
+                    self.ipt[n] = torch.zeros_like(p)
+                    self.exp_avg_ipt[n] = torch.zeros_like(p)
+                    self.exp_avg_unc[n] = torch.zeros_like(p)
+                with torch.no_grad():
+                    if deepspeed_config() is not None:
+                        import deepspeed
+
+                        grad = deepspeed.utils.safe_get_full_grad(p)
+                        self.ipt[n] = (p * grad).abs().detach()
+                    else:
+                        self.ipt[n] = (p * p.grad).abs().detach()
+                    # Sensitivity smoothing
+                    self.exp_avg_ipt[n] = self.beta1 * self.exp_avg_ipt[n] + (1 - self.beta1) * self.ipt[n]
+                    # Uncertainty quantification
+                    self.exp_avg_unc[n] = (
+                        self.beta2 * self.exp_avg_unc[n] + (1 - self.beta2) * (self.ipt[n] - self.exp_avg_ipt[n]).abs()
+                    )
+
+    def _element_score(self, n):
+        return self.exp_avg_ipt[n] * self.exp_avg_unc[n]
+
+    def _combine_ipt(self, ipt_E, ipt_AB):
+        ipt_AB = ipt_AB.sum(dim=1, keepdim=False)
+        sum_ipt = ipt_E.view(-1) + ipt_AB.view(-1)
+        return sum_ipt
+
+    def mask_to_budget(self, model, budget):
+        value_ipt = {}
+        vector_ipt = {}
+        triplet_ipt = {}
+        # Get the importance score for A, E, B
+        for n, p in model.named_parameters():
+            if f"lora_A.{self.adapter_name}" in n:
+                entry_ipt = self._element_score(n)
+                comb_ipt = torch.mean(entry_ipt, dim=1, keepdim=True)
+                name_m = n.replace("lora_A", "%s")
+                if name_m not in vector_ipt:
+                    vector_ipt[name_m] = [comb_ipt]
+                else:
+                    vector_ipt[name_m].append(comb_ipt)
+            if f"lora_B.{self.adapter_name}" in n:
+                entry_ipt = self._element_score(n)
+                comb_ipt = torch.mean(entry_ipt, dim=0, keepdim=False).view(-1, 1)
+                name_m = n.replace("lora_B", "%s")
+                if name_m not in vector_ipt:
+                    vector_ipt[name_m] = [comb_ipt]
+                else:
+                    vector_ipt[name_m].append(comb_ipt)
+            if f"lora_E.{self.adapter_name}" in n:
+                entry_ipt = self._element_score(n)
+                name_m = n.replace("lora_E", "%s")
+                value_ipt[name_m] = entry_ipt
+
+        all_score = []
+        # Calculate the score for each triplet
+        for name_m in vector_ipt:
+            ipt_E = value_ipt[name_m]
+            ipt_AB = torch.cat(vector_ipt[name_m], dim=1)
+            sum_ipt = self._combine_ipt(ipt_E, ipt_AB)
+            name_E = name_m % "lora_E"
+            triplet_ipt[name_E] = sum_ipt.view(-1, 1)
+            all_score.append(sum_ipt.view(-1))
+
+        # Get the threshold by ranking ipt
+        mask_threshold = torch.kthvalue(
+            torch.cat(all_score),
+            k=self.init_bgt - budget,
+        )[0].item()
+
+        rank_pattern = {}
+        # Mask the unimportant triplets
+        with torch.no_grad():
+            for n, p in model.named_parameters():
+                if f"lora_E.{self.adapter_name}" in n:
+                    p.masked_fill_(triplet_ipt[n] <= mask_threshold, 0.0)
+                    rank_pattern[n] = (~(triplet_ipt[n] <= mask_threshold)).view(-1).tolist()
+        return rank_pattern
+
+    def update_and_allocate(self, model, global_step, force_mask=False):
+        # # Update the importance score and allocate the budget
+        if global_step < self.peft_config.total_step - self.peft_config.tfinal:
+            self.update_ipt(model)
+        budget, mask_ind = self.budget_schedule(global_step)
+        # Allocate the budget according to importance scores
+        if mask_ind or force_mask:
+            rank_pattern = self.mask_to_budget(model, budget)
+        else:
+            rank_pattern = None
+        return budget, rank_pattern
+
+    def mask_using_rank_pattern(self, model, rank_pattern):
+        # Mask the unimportant triplets
+        is_adapter_name_truncated = False
+        if self.adapter_name not in next(iter(rank_pattern.keys())):
+            is_adapter_name_truncated = True
+
+        with torch.no_grad():
+            for n, p in model.named_parameters():
+                if f"lora_E.{self.adapter_name}" in n:
+                    key = n if not is_adapter_name_truncated else n.replace(f".{self.adapter_name}", "")
+                    mask = torch.Tensor(rank_pattern[key]).unsqueeze(-1).to(p.device)
+                    p.masked_fill_(~mask.bool(), 0.0)

lib/python3.12/site-packages/peft/tuners/adalora/model.py

@@ -0,0 +1,364 @@
+# Copyright 2023-present the HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import warnings
+
+import torch
+from transformers.pytorch_utils import Conv1D
+
+from peft.import_utils import is_bnb_4bit_available, is_bnb_available, is_gptqmodel_available
+from peft.tuners.lora import LoraConfig, LoraModel
+from peft.tuners.tuners_utils import BaseTunerLayer
+from peft.utils import (
+    TRANSFORMERS_MODELS_TO_ADALORA_TARGET_MODULES_MAPPING,
+    _freeze_adapter,
+    _get_submodules,
+    get_auto_gptq_quant_linear,
+    get_gptqmodel_quant_linear,
+    get_quantization_config,
+)
+from peft.utils.integrations import gather_params_ctx
+
+from .gptq import SVDQuantLinear
+from .layer import AdaLoraLayer, RankAllocator, SVDLinear
+
+
+class AdaLoraModel(LoraModel):
+    """
+    Creates AdaLoRA (Adaptive LoRA) model from a pretrained transformers model. Paper:
+    https://openreview.net/forum?id=lq62uWRJjiY
+
+    Args:
+        model ([`transformers.PreTrainedModel`]): The model to be adapted.
+        config ([`AdaLoraConfig`]): The configuration of the AdaLora model.
+        adapter_name (`str`): The name of the adapter, defaults to `"default"`.
+        low_cpu_mem_usage (`bool`, `optional`, defaults to `False`):
+            Create empty adapter weights on meta device. Useful to speed up the loading process.
+
+    Returns:
+        `torch.nn.Module`: The AdaLora model.
+
+    Example::
+
+        >>> from transformers import AutoModelForSeq2SeqLM >>> from peft import LoraConfig, AdaLoraModel, AdaLoraConfig
+        >>> config = AdaLoraConfig(
+                peft_type="ADALORA", task_type="SEQ_2_SEQ_LM", init_r=12, lora_alpha=32, target_modules=["q", "v"],
+                lora_dropout=0.01,
+            )
+        >>> model = AutoModelForSeq2SeqLM.from_pretrained("t5-base") >>> model = AdaLoraModel(model, config, "default")
+
+    **Attributes**:
+        - **model** ([`transformers.PreTrainedModel`]) -- The model to be adapted.
+        - **peft_config** ([`AdaLoraConfig`]): The configuration of the AdaLora model.
+    """
+
+    # Note: don't redefine prefix here, it should be inherited from LoraModel
+
+    def __init__(self, model, config, adapter_name):
+        super().__init__(model, config, adapter_name)
+
+        traininable_mode_counter = 0
+        for config in self.peft_config.values():
+            if not config.inference_mode:
+                traininable_mode_counter += 1
+
+        if traininable_mode_counter > 1:
+            raise ValueError(
+                "AdaLoraModel supports only 1 trainable adapter. "
+                "When using multiple adapters, set inference_mode to True for all adapters except the one you want to train."
+            )
+
+        if self.peft_config[adapter_name].inference_mode:
+            _freeze_adapter(self.model, adapter_name)
+        else:
+            self.trainable_adapter_name = adapter_name
+            self.rankallocator = RankAllocator(self.model, self.peft_config[adapter_name], self.trainable_adapter_name)
+
+    def _check_new_adapter_config(self, config: LoraConfig) -> None:
+        """
+        A helper method to check the config when a new adapter is being added.
+
+        Raise a ValueError if there is something wrong with the config or if it conflicts with existing adapters.
+
+        """
+        super()._check_new_adapter_config(config)
+
+        traininable_mode_counter = 0
+        for config_ in self.peft_config.values():
+            if not config_.inference_mode:
+                traininable_mode_counter += 1
+
+        if traininable_mode_counter > 1:
+            raise ValueError(
+                f"{self.__class__.__name__} supports only 1 trainable adapter. "
+                "When using multiple adapters, set inference_mode to True for all adapters except the one "
+                "you want to train."
+            )
+
+    def _create_and_replace(
+        self,
+        lora_config,
+        adapter_name,
+        target,
+        target_name,
+        parent,
+        current_key,
+    ):
+        kwargs = {
+            "r": lora_config.init_r,
+            "lora_alpha": lora_config.lora_alpha,
+            "lora_dropout": lora_config.lora_dropout,
+            "fan_in_fan_out": lora_config.fan_in_fan_out,
+            "init_lora_weights": lora_config.init_lora_weights,
+            "loaded_in_8bit": getattr(self.model, "is_loaded_in_8bit", False),
+            "loaded_in_4bit": getattr(self.model, "is_loaded_in_4bit", False),
+        }
+        if (kwargs["loaded_in_8bit"] or kwargs["loaded_in_4bit"]) and not is_bnb_available():
+            raise ImportError(
+                "To use AdaLora with 8-bit quantization, please install the `bitsandbytes` package. "
+                "You can install it with `pip install bitsandbytes`."
+            )
+
+        quantization_config = get_quantization_config(self.model, method="gptq")
+        if quantization_config is not None:
+            kwargs["gptq_quantization_config"] = quantization_config
+
+        # If it is not an AdaLoraLayer, create a new module, else update it with new adapters
+        if not isinstance(target, AdaLoraLayer):
+            device_map = self.model.hf_device_map if hasattr(self.model, "hf_device_map") else None
+            new_module = self._create_new_module(lora_config, adapter_name, target, device_map=device_map, **kwargs)
+            if adapter_name not in self.active_adapters:
+                # adding an additional adapter: it is not automatically trainable
+                new_module.requires_grad_(False)
+            self._replace_module(parent, target_name, new_module, target)
+        else:
+            target.update_layer(
+                adapter_name,
+                lora_config.init_r,
+                lora_config.lora_alpha,
+                lora_config.lora_dropout,
+                lora_config.init_lora_weights,
+            )
+
+    @staticmethod
+    def _create_new_module(lora_config, adapter_name, target, device_map=None, **kwargs):
+        # avoid eager bnb import
+        if is_bnb_available():
+            import bitsandbytes as bnb
+
+            from .bnb import SVDLinear8bitLt
+        if is_bnb_4bit_available():
+            from .bnb import SVDLinear4bit
+
+        gptq_quantization_config = kwargs.get("gptq_quantization_config", None)
+
+        if is_gptqmodel_available():
+            QuantLinear = get_gptqmodel_quant_linear(gptq_quantization_config, device_map=device_map)
+        else:
+            QuantLinear = get_auto_gptq_quant_linear(gptq_quantization_config)
+
+        loaded_in_8bit = kwargs.pop("loaded_in_8bit", False)
+        loaded_in_4bit = kwargs.pop("loaded_in_4bit", False)
+
+        if isinstance(target, BaseTunerLayer):
+            target_base_layer = target.get_base_layer()
+        else:
+            target_base_layer = target
+
+        if loaded_in_8bit and isinstance(target_base_layer, bnb.nn.Linear8bitLt):
+            kwargs.update(
+                {
+                    "has_fp16_weights": target_base_layer.state.has_fp16_weights,
+                    "threshold": target_base_layer.state.threshold,
+                    "index": target_base_layer.index,
+                }
+            )
+            new_module = SVDLinear8bitLt(target, adapter_name, **kwargs)
+        elif loaded_in_4bit and is_bnb_4bit_available() and isinstance(target_base_layer, bnb.nn.Linear4bit):
+            fourbit_kwargs = kwargs.copy()
+            fourbit_kwargs.update(
+                {
+                    "compute_dtype": target_base_layer.compute_dtype,
+                    "compress_statistics": target_base_layer.weight.compress_statistics,
+                    "quant_type": target_base_layer.weight.quant_type,
+                }
+            )
+            new_module = SVDLinear4bit(target, adapter_name, **fourbit_kwargs)
+        elif QuantLinear is not None and isinstance(target, QuantLinear):
+            new_module = SVDQuantLinear(target, adapter_name, **kwargs)
+        else:
+            if isinstance(target_base_layer, torch.nn.Linear):
+                if kwargs["fan_in_fan_out"]:
+                    warnings.warn(
+                        "fan_in_fan_out is set to True but the target module is `torch.nn.Linear`. "
+                        "Setting fan_in_fan_out to False."
+                    )
+                    kwargs["fan_in_fan_out"] = lora_config.fan_in_fan_out = False
+            elif isinstance(target_base_layer, Conv1D):
+                if not kwargs["fan_in_fan_out"]:
+                    warnings.warn(
+                        "fan_in_fan_out is set to False but the target module is `Conv1D`. "
+                        "Setting fan_in_fan_out to True."
+                    )
+                    kwargs["fan_in_fan_out"] = lora_config.fan_in_fan_out = True
+            else:
+                raise ValueError(
+                    f"Target module {target} is not supported. "
+                    f"Currently, only `torch.nn.Linear` and `Conv1D` are supported."
+                )
+            new_module = SVDLinear(target, adapter_name, **kwargs)
+
+        return new_module
+
+    @staticmethod
+    def _prepare_adapter_config(peft_config, model_config):
+        if peft_config.target_modules is None:
+            if model_config["model_type"] not in TRANSFORMERS_MODELS_TO_ADALORA_TARGET_MODULES_MAPPING:
+                raise ValueError("Please specify `target_modules` in `peft_config`")
+            peft_config.target_modules = TRANSFORMERS_MODELS_TO_ADALORA_TARGET_MODULES_MAPPING[
+                model_config["model_type"]
+            ]
+        return peft_config
+
+    def __getattr__(self, name: str):
+        """Forward missing attributes to the wrapped module."""
+        try:
+            return super().__getattr__(name)  # defer to nn.Module's logic
+        except AttributeError:
+            if name == "model":  # see #1892: prevent infinite recursion if class is not initialized
+                raise
+            return getattr(self.model, name)
+
+    def forward(self, *args, **kwargs):
+        outputs = self.model.forward(*args, **kwargs)
+
+        if (getattr(outputs, "loss", None) is not None) and isinstance(outputs.loss, torch.Tensor):
+            # Calculate the orthogonal regularization
+            orth_reg_weight = self.peft_config[self.trainable_adapter_name].orth_reg_weight
+
+            if orth_reg_weight <= 0:
+                raise ValueError("orth_reg_weight should be greater than 0. ")
+
+            regu_loss = 0
+            num_param = 0
+            for n, p in self.model.named_parameters():
+                if ("lora_A" in n or "lora_B" in n) and self.trainable_adapter_name in n:
+                    if p.shape == torch.Size([0]):
+                        with gather_params_ctx(p, fwd_module=self):
+                            para_cov = p @ p.T if "lora_A" in n else p.T @ p
+                    else:
+                        para_cov = p @ p.T if "lora_A" in n else p.T @ p
+                    I = torch.eye(*para_cov.size(), out=torch.empty_like(para_cov))  # noqa: E741
+                    I.requires_grad = False
+                    num_param += 1
+                    regu_loss += torch.norm(para_cov - I, p="fro")
+            if num_param > 0:
+                regu_loss = regu_loss / num_param
+            else:
+                regu_loss = 0
+            outputs.loss += orth_reg_weight * regu_loss
+        return outputs
+
+    def resize_modules_by_rank_pattern(self, rank_pattern, adapter_name):
+        lora_config = self.peft_config[adapter_name]
+        for name, rank_idx in rank_pattern.items():
+            if isinstance(rank_idx, list):
+                rank = sum(rank_idx)
+            elif isinstance(rank_idx, torch.Tensor):
+                rank_idx = rank_idx.view(-1)
+                rank = rank_idx.sum().item()
+            else:
+                raise ValueError("Unexpected type of rank_idx")
+            key = ".".join(name.split(".")[0:-2]) if adapter_name in name else ".".join(name.split(".")[0:-1])
+            _, target, _ = _get_submodules(self.model, key)
+            lora_E_weights = target.lora_E[adapter_name][rank_idx]
+            lora_A_weights = target.lora_A[adapter_name][rank_idx]
+            lora_B_weights = target.lora_B[adapter_name][:, rank_idx]
+            ranknum = target.ranknum[adapter_name]
+            target.update_layer(
+                adapter_name,
+                rank,
+                lora_config.lora_alpha,
+                lora_config.lora_dropout,
+                lora_config.init_lora_weights,
+            )
+            with torch.no_grad():
+                if rank > 0:
+                    target.lora_E[adapter_name].copy_(lora_E_weights)
+                    target.lora_A[adapter_name].copy_(lora_A_weights)
+                    target.lora_B[adapter_name].copy_(lora_B_weights)
+                    # The scaling is exactly as the previous
+                    target.ranknum[adapter_name].copy_(ranknum)
+
+    def resize_state_dict_by_rank_pattern(self, rank_pattern, state_dict, adapter_name):
+        for name, rank_idx in rank_pattern.items():
+            rank = sum(rank_idx)
+            prefix = ".".join(name.split(".")[0:-2]) if adapter_name in name else ".".join(name.split(".")[0:-1])
+            for layer in ["lora_E", "lora_A", "lora_B"]:
+                key = f"base_model.model.{prefix}.{layer}.{adapter_name}"
+                if layer != "lora_B":
+                    state_dict[key] = (
+                        state_dict[key][rank_idx] if rank != state_dict[key].shape[0] else state_dict[key]
+                    )
+                else:
+                    state_dict[key] = (
+                        state_dict[key][:, rank_idx] if rank != state_dict[key].shape[1] else state_dict[key]
+                    )
+        return state_dict
+
+    def update_and_allocate(self, global_step):
+        """
+        This method updates Adalora budget and mask.
+
+        This should be called in every training step after `loss.backward()` and before `zero_grad()`.
+
+        `tinit`, `tfinal` and `deltaT` are handled with in the method.
+
+        Args:
+            global_step (`int`): The current training step, it is used to calculate adalora budget.
+
+        Example:
+
+        ```python
+        >>> loss = model(**input).loss
+        >>> loss.backward()
+        >>> optimizer.step()
+        >>> model.base_model.update_and_allocate(i_step)
+        >>> optimizer.zero_grad()
+        ```
+        """
+        lora_config = self.peft_config[self.trainable_adapter_name]
+        # Update the importance score and allocate the budget
+        if global_step < lora_config.total_step - lora_config.tfinal:
+            _, rank_pattern = self.rankallocator.update_and_allocate(self.model, global_step)
+            if rank_pattern:
+                lora_config.rank_pattern = rank_pattern
+        # Finalize the budget allocation
+        elif global_step == lora_config.total_step - lora_config.tfinal:
+            _, rank_pattern = self.rankallocator.update_and_allocate(self.model, global_step, force_mask=True)
+            # for some reason, this freezes the trainable parameters and nothing gets updates
+            # self.resize_modules_by_rank_pattern(rank_pattern, self.trainable_adapter_name)
+            lora_config.rank_pattern = rank_pattern
+            self.rankallocator.reset_ipt()
+        # Currently using inefficient way to mask the unimportant weights using the rank pattern
+        #  due to problem mentioned above
+        elif global_step > lora_config.total_step - lora_config.tfinal:
+            self.rankallocator.mask_using_rank_pattern(self.model, lora_config.rank_pattern)
+        # Pass the function and do forward propagation
+        else:
+            return None
+
+    def add_weighted_adapter(self, *args, **kwargs):
+        """This method is not supported for AdaLoRA, use LoRA instead."""
+        raise TypeError(f"{self.__class__.__name__} does not support add_weighted_adapter method.")

lib/python3.12/site-packages/peft/tuners/adaption_prompt/__init__.py

@@ -0,0 +1,23 @@
+# Copyright 2023-present the HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from peft.utils import register_peft_method
+
+from .config import AdaptionPromptConfig
+from .layer import AdaptedAttention
+from .model import AdaptionPromptModel
+
+
+__all__ = ["AdaptedAttention", "AdaptionPromptConfig", "AdaptionPromptModel"]
+
+register_peft_method(name="adaption_prompt", config_cls=AdaptionPromptConfig, model_cls=AdaptionPromptModel)

lib/python3.12/site-packages/peft/tuners/adaption_prompt/config.py

@@ -0,0 +1,81 @@
+# Copyright 2023-present the HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from collections import namedtuple
+from dataclasses import dataclass, field
+
+from peft.config import PeftConfig
+from peft.utils import PeftType
+
+from .utils import llama_compute_query_states
+
+
+@dataclass
+class AdaptionPromptConfig(PeftConfig):
+    """Stores the configuration of an [`AdaptionPromptModel`]."""
+
+    target_modules: str = field(
+        default=None, metadata={"help": "Name of the attention submodules to insert adaption prompts into."}
+    )
+    adapter_len: int = field(default=None, metadata={"help": "Number of adapter tokens to insert"})
+    adapter_layers: int = field(default=None, metadata={"help": "Number of adapter layers (from the top)"})
+
+    def __post_init__(self):
+        super().__post_init__()
+        self.peft_type = PeftType.ADAPTION_PROMPT
+
+    @property
+    def is_adaption_prompt(self) -> bool:
+        """Return True if this is an adaption prompt config."""
+        return True
+
+
+# Contains the config that is specific to a transformers model type.
+ModelTypeConfig = namedtuple(
+    "ModelTypeConfig", ["compute_query_states", "target_modules", "k_proj_layer", "v_proj_layer", "o_proj_layer"]
+)
+
+# Mapping of transformers model types to their specific configuration.
+TRANSFORMERS_MODEL_CONFIG = {
+    "llama": ModelTypeConfig(
+        compute_query_states=llama_compute_query_states,
+        target_modules="self_attn",
+        k_proj_layer="k_proj",
+        v_proj_layer="v_proj",
+        o_proj_layer="o_proj",
+    ),
+    "mistral": ModelTypeConfig(  # same as llama,
+        compute_query_states=llama_compute_query_states,
+        target_modules="self_attn",
+        k_proj_layer="k_proj",
+        v_proj_layer="v_proj",
+        o_proj_layer="o_proj",
+    ),
+}
+
+
+def prepare_config(
+    peft_config: AdaptionPromptConfig,
+    model,
+) -> AdaptionPromptConfig:
+    """Prepare the config based on the llama model type."""
+    if model.config.model_type not in TRANSFORMERS_MODEL_CONFIG:
+        raise ValueError("Unsupported model type for adaption prompt: '{model.config.model_type}'.")
+
+    model_config = TRANSFORMERS_MODEL_CONFIG[model.config.model_type]
+
+    if peft_config.target_modules is None:
+        peft_config.target_modules = model_config.target_modules
+
+    return peft_config

lib/python3.12/site-packages/peft/tuners/adaption_prompt/layer.py

@@ -0,0 +1,138 @@
+# Copyright 2023-present the HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import math
+
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+from .config import TRANSFORMERS_MODEL_CONFIG
+
+
+class AdaptedAttention(nn.Module):
+    """This module wraps a LLamaAttention module and injects adaption prompts."""
+
+    def __init__(self, model_type: str, adapter_len: int, model):
+        """
+        Initialize object.
+
+        Args:
+            model_type: The transformer model type. This is used to retrieve the right method to
+                compute query states.
+            adapter_len: The length of the adaption prompt to insert.
+            model: The original transformer attention module that is being wrapped.
+        """
+        assert not isinstance(model, AdaptedAttention)
+        super().__init__()
+        self.model_type = model_type
+        self.model = model
+        self.adapter_len = adapter_len
+        # Assume all parameters of the attention model we are wrapping are on the same device.
+        device = next(model.parameters()).device
+        # Don't think this was specified in the paper, but we follow the official repo which used an Embedding
+        # which initializes the tokens with standard normal values.
+        # https://github.com/ZrrSkywalker/LLaMA-Adapter/blob/41c3546fe1997ab8a65809dc8d8f9252b19d9faf/llama/model.py#L234
+        # (bsz, adapter_len, hidden_size)
+        target_dtype = (
+            model.q_proj.weight.dtype if model.q_proj.weight.dtype not in [torch.int8, torch.uint8] else torch.float32
+        )
+        if hasattr(self.model, "hidden_size"):
+            # TODO: remove this clause after 2026-01-01
+            hidden_size = self.model.hidden_size
+        else:  # changed in https://github.com/huggingface/transformers/pull/35235
+            hidden_size = self.model.config.hidden_size
+        self.adaption_prompt = nn.Parameter(
+            torch.empty(1, adapter_len, hidden_size, device=device, dtype=target_dtype).normal_()
+        )
+        # Initialize the gate to 0 as this is "zero-init".
+        self.adaption_gate = nn.Parameter(torch.zeros(1, device=device, dtype=target_dtype))
+
+    def forward(self, **kwargs):
+        """
+        Forward pass for the adapter which wraps the original LlamaAttention module.
+
+        "Official" paper implementation:
+        https://github.com/ZrrSkywalker/LLaMA-Adapter/blob/41c3546fe1997ab8a65809dc8d8f9252b19d9faf/llama/model.py#L141
+
+        Args:
+            kwargs: See the original LlamaAttention module.
+        """
+        if kwargs.get("output_attention", False):
+            raise NotImplementedError("output_attention is not currently supported.")
+
+        output, *_ = self.model(**kwargs)
+        bsz = output.shape[0]
+        q_len = output.shape[1]
+        embed_dim = output.shape[2]
+        k_proj_layer = TRANSFORMERS_MODEL_CONFIG[self.model_type].k_proj_layer
+        v_proj_layer = TRANSFORMERS_MODEL_CONFIG[self.model_type].v_proj_layer
+        o_proj_layer = TRANSFORMERS_MODEL_CONFIG[self.model_type].o_proj_layer
+        factor = (
+            self.model.k_proj.in_features // self.model.k_proj.out_features
+        )  # Mistral has different input and output dimension for k_proj and v_proj layers
+
+        if k_proj_layer == v_proj_layer:
+            _, key, value = getattr(self.model, k_proj_layer)(self.adaption_prompt).split(embed_dim, dim=2)
+        else:
+            key = getattr(self.model, k_proj_layer)(self.adaption_prompt)
+            value = getattr(self.model, v_proj_layer)(self.adaption_prompt)
+
+        if hasattr(self.model, "num_heads"):
+            # TODO: remove this clause after 2026-01-01
+            num_heads = self.model.num_heads
+        else:  # changed in https://github.com/huggingface/transformers/pull/35235
+            num_heads = self.model.config.num_attention_heads
+        # (bsz, num_key_value_heads, adapter_len, head_dim)
+        adapter_k = (
+            key.view(1, self.adapter_len, (num_heads // factor), self.model.head_dim)
+            .repeat(bsz, 1, 1, 1)
+            .transpose(1, 2)
+        )
+        adapter_v = (
+            value.view(1, self.adapter_len, (num_heads // factor), self.model.head_dim)
+            .repeat(bsz, 1, 1, 1)
+            .transpose(1, 2)
+        )
+        # Below is taken from https://github.com/huggingface/transformers/blob/e547458c43dfdbbb8f6a7757237e234c44e20a8f/src/transformers/models/mistral/modeling_mistral.py#L181
+        # (bsz, num_heads, adapter_len, head_dim)
+        adapter_k = torch.repeat_interleave(adapter_k, repeats=factor, dim=1)
+        adapter_v = torch.repeat_interleave(adapter_v, repeats=factor, dim=1)
+        # Recompute query states.
+        compute_query_states = TRANSFORMERS_MODEL_CONFIG[self.model_type].compute_query_states
+        # (bsz, num_heads, q_len, head_dim)
+        query_states = compute_query_states(model=self.model, **kwargs)
+
+        previous_dtype = query_states.dtype
+
+        # (bsz, num_heads, q_len, adapter_len)
+        scores = torch.matmul(query_states, adapter_k.transpose(2, 3).to(previous_dtype)) / math.sqrt(
+            self.model.head_dim
+        )
+        # Upcast attention to fp32
+        # (bsz, num_heads, q_len, adapter_len)
+        scores = self.adaption_gate * F.softmax(scores, dim=-1, dtype=torch.float32).to(previous_dtype)
+        # (bsz, q_len, num_heads * head_dim)
+        adapter_output = torch.matmul(scores, adapter_v).transpose(1, 2).reshape(bsz, q_len, -1)
+
+        # (bsz, q_len, hidden_size)
+        if o_proj_layer is not None:
+            adapter_output = getattr(self.model, o_proj_layer)(adapter_output)
+
+        # Add adaption prompt output to original output.
+        output = output + adapter_output
+
+        # Restore original dtype.
+        output = output.to(previous_dtype)
+        return output, *_

lib/python3.12/site-packages/peft/tuners/adaption_prompt/model.py

@@ -0,0 +1,162 @@
+# Copyright 2023-present the HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import Dict, List
+
+import torch.nn as nn
+
+from peft.utils import _freeze_adapter, _get_submodules
+
+from .config import AdaptionPromptConfig, prepare_config
+from .layer import AdaptedAttention
+from .utils import is_adaption_prompt_trainable
+
+
+class AdaptionPromptModel(nn.Module):
+    """
+    Implements adaption prompts as described in https://arxiv.org/pdf/2303.16199.pdf.
+
+    The top L attention modules are replaced with AdaptedAttention modules that wrap the original ones, but insert
+    trainable prompts with gates (for zero init).
+
+    Notes on the multi-adapter pattern:
+    - We store the states of different adapters by keeping a dictionary of AdaptedAttention modules indexed by adapter
+      name.
+    - Every time we switch adapters, we remove the modules of the currently active adapter from the model, store them
+      in the dictionary, and replace them with the modules of the new adapter.
+    - To avoid duplicated and potentially inconsistent state, the currently active adapter is always removed from the
+      dictionary.
+    - Disabling the adapter would also result in the modules being removed from the model.
+    """
+
+    def __init__(self, model, configs: Dict, adapter_name: str):
+        super().__init__()
+        self.model = model
+        # Store adapter configs by name.
+        self.peft_config: Dict[str, AdaptionPromptConfig] = {}
+        # Store lists of the parents of the affected attention modules by adapter name.
+        # We keep references to the parents so we can swap the adapters in-and-out of the model.
+        self._parents: Dict[str, List[nn.Module]] = {}
+        # Store lists of cached AdaptedAttention modules by name.
+        self._cached_adapters: Dict[str, List] = {}
+        # The name of the currently active adapter.
+        self._active_adapter = None
+        # Whether the adapter is enabled.
+        self._enabled = True
+        self.forward = self.model.forward
+        self.add_adapter(adapter_name, configs[adapter_name])
+        self._mark_only_adaption_prompts_as_trainable(self.model)
+
+    def add_adapter(self, adapter_name: str, config: AdaptionPromptConfig) -> None:
+        """Add an adapter with the given name and config."""
+        config = prepare_config(config, self.model)
+        if adapter_name in self.peft_config:
+            raise ValueError(f"Adapter with name '{adapter_name}' already exists.")
+
+        parents = []
+        for name, _ in self.model.named_modules():
+            if name.endswith(config.target_modules):
+                par, _, _ = _get_submodules(self.model, name)
+                parents.append(par)
+        if len(parents) < config.adapter_layers:
+            raise ValueError(
+                f"Config specifies more adapter layers '{config.adapter_layers}' than the model has '{len(parents)}'."
+            )
+        # Note that if the target modules are not in Sequential, ModuleList, or
+        # some other PyTorch ordered container, the behavior is undefined as we
+        # assume here that the order of the modules is the same as the order of
+        # the transformer decoder layers.
+        parents = parents[-config.adapter_layers :]
+        self._parents[adapter_name] = parents
+
+        # It is only None during initialization.
+        # If it is disabled, we don't have to remove the modules.
+        if self._active_adapter is not None and self._enabled:
+            self._remove_adapted_attentions(self._active_adapter)
+        self._active_adapter = adapter_name
+        self.peft_config[adapter_name] = config
+        self._create_adapted_attentions(config, parents)
+        if not self._enabled:
+            self._remove_adapted_attentions(self._active_adapter)
+
+        if config.inference_mode:
+            _freeze_adapter(self.model, adapter_name)
+
+    def set_adapter(self, adapter_name: str) -> None:
+        """Set the model to use the adapter with the given name."""
+        if self._active_adapter == adapter_name:
+            return
+        if adapter_name not in self.peft_config:
+            raise ValueError(f"Adapter with name '{adapter_name}' does not exist.")
+
+        if self._enabled:
+            self._remove_adapted_attentions(self._active_adapter)
+            self._set_adapted_attentions(adapter_name)
+
+        self._active_adapter = adapter_name
+
+    def enable_adapter_layers(self):
+        """Enable adapter layers by swapping in cached AdaptedAttention modules."""
+        self._enabled = True
+        self._set_adapted_attentions(self._active_adapter)
+
+    def disable_adapter_layers(self):
+        """Disable adapter layers by swapping out AdaptedAttention modules."""
+        self._enabled = False
+        self._remove_adapted_attentions(self._active_adapter)
+
+    def _create_adapted_attentions(self, config: AdaptionPromptConfig, parents: List[nn.Module]) -> None:
+        """Wrap LlamaAttention modules with newly created AdaptedAttention modules."""
+        for par in parents:
+            attn = AdaptedAttention(
+                model_type=self.model.config.model_type,
+                adapter_len=config.adapter_len,
+                model=getattr(par, config.target_modules),
+            )
+            setattr(par, config.target_modules, attn)
+
+    def _set_adapted_attentions(self, adapter_name: str) -> None:
+        """Replace LlamaAttention modules with cached AdaptedAttention modules."""
+        cached = self._cached_adapters[adapter_name]
+        del self._cached_adapters[adapter_name]
+        config = self.peft_config[adapter_name]
+        for i, par in enumerate(self._parents[adapter_name]):
+            setattr(par, config.target_modules, cached[i])
+
+    def _remove_adapted_attentions(self, adapter_name: str) -> None:
+        """Remove AdaptedAttention modules from the model and store them in the cache."""
+        config = self.peft_config[adapter_name]
+        adapted_attentions = []
+        for par in self._parents[adapter_name]:
+            attn = getattr(par, config.target_modules)
+            adapted_attentions.append(attn)
+            setattr(par, config.target_modules, attn.model)
+        self._cached_adapters[adapter_name] = adapted_attentions
+
+    def _mark_only_adaption_prompts_as_trainable(self, model: nn.Module) -> None:
+        """Freeze all parameters of the model except the adaption prompts."""
+        for n, p in model.named_parameters():
+            if not is_adaption_prompt_trainable(n):
+                p.requires_grad = False
+
+    def __getattr__(self, name: str):
+        """Forward missing attributes to the wrapped module."""
+        try:
+            return super().__getattr__(name)  # defer to nn.Module's logic
+        except AttributeError:
+            # This is necessary as e.g. causal models have various methods that we
+            # don't want to re-implement here.
+            if name == "model":  # see #1892: prevent infinite recursion if class is not initialized
+                raise
+            return getattr(self.model, name)

lib/python3.12/site-packages/peft/tuners/adaption_prompt/utils.py

@@ -0,0 +1,132 @@
+# Copyright 2023-present the HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import inspect
+
+import torch
+import torch.nn as nn
+
+
+def llama_rotate_half(x: torch.Tensor) -> torch.Tensor:
+    """
+    Rotate half the hidden dims of the input.
+
+    This function was duplicated verbatim from:
+    https://github.com/huggingface/transformers/blob/1de8ce9ee1191ba761a593ac15d9ccbf5851bfc5/src/transformers/models/llama/modeling_llama.py#L126
+
+    This was done to eliminate the Llama transformers implementation as a dependency of this file. Note that some other
+    functions were also adapted from the transformers implementation but were modified.
+    """
+    x1 = x[..., : x.shape[-1] // 2]
+    x2 = x[..., x.shape[-1] // 2 :]
+    return torch.cat((-x2, x1), dim=-1)
+
+
+def llama_apply_rotary_pos_emb(q, cos, sin, position_ids):
+    """
+    Apply rotary position embedding to query states in the Llama model.
+
+    This function was adapted from:
+    https://github.com/huggingface/transformers/blob/1de8ce9ee1191ba761a593ac15d9ccbf5851bfc5/src/transformers/models/llama/modeling_llama.py#L133
+
+    It was modified to remove unnecessary processing of key states. The method is compatible with transformers <=
+    4.34.2 and also with the latest version (>=4.35).
+    """
+    # In previous transformers version cos/sin cached had a shape of 4D
+    if len(cos.shape) == 4:
+        gather_indices = position_ids[:, None, :, None]  # [bs, 1, seq_len, 1]
+        gather_indices = gather_indices.repeat(1, cos.shape[1], 1, cos.shape[3])
+        cos = torch.gather(cos.repeat(gather_indices.shape[0], 1, 1, 1), 2, gather_indices)
+        sin = torch.gather(sin.repeat(gather_indices.shape[0], 1, 1, 1), 2, gather_indices)
+    # In the new version, it is 2D so we fall back to the new implementation
+    # https://github.com/huggingface/transformers/blame/eef7ea98c31a333bacdc7ae7a2372bde772be8e4/src/transformers/models/llama/modeling_llama.py#L222-L226
+    else:
+        cos = cos[position_ids].unsqueeze(1)
+        sin = sin[position_ids].unsqueeze(1)
+    q_embed = (q * cos) + (llama_rotate_half(q) * sin)
+    return q_embed
+
+
+def llama_compute_query_states(model: nn.Module, **kwargs) -> torch.Tensor:
+    """
+    Compute query states for Llama models specifically. They need to be recomputed as the forward() method of the
+    original LlamaModel in the transformers library does not return them. See the related discussion in the PR:
+    https://github.com/huggingface/peft/pull/268
+    """
+    hidden_states = kwargs.get("hidden_states")
+    position_ids = kwargs.get("position_ids")
+    past_key_value = kwargs.get("past_key_value")
+    bsz, q_len, _ = hidden_states.size()
+    if hasattr(model, "num_heads"):
+        # TODO: remove this clause after 2026-01-01
+        num_heads = model.num_heads
+    else:  # changed in https://github.com/huggingface/transformers/pull/35235
+        num_heads = model.config.num_attention_heads
+    query_states = model.q_proj(hidden_states).view(bsz, q_len, num_heads, model.head_dim).transpose(1, 2)
+
+    factor = model.k_proj.in_features // model.k_proj.out_features
+    value_states = model.v_proj(hidden_states).view(bsz, q_len, (num_heads // factor), model.head_dim).transpose(1, 2)
+
+    seq_len = q_len
+
+    if past_key_value is not None:
+        if isinstance(past_key_value, tuple):
+            # for transformers <= 4.35
+            seq_len += past_key_value[0].shape[-2]
+        else:
+            # since transformers 4.36, this is a DynamicCache instance
+            seq_len += past_key_value.get_seq_length(model.layer_idx)
+
+    # model.rotary_emb is deprecated and will be removed in transformers > 4.47.0. Instead, the position embeddings are
+    # passed via the kwargs
+    if "position_embeddings" in kwargs:
+        cos, sin = kwargs["position_embeddings"]
+        cos = cos.unsqueeze(1)
+        sin = sin.unsqueeze(1)
+        return (query_states * cos) + (llama_rotate_half(query_states) * sin)
+
+    # For transformers > 4.37.2 `position_ids` became a required arguments in the rotary embedding's forward pass.
+    if "position_ids" not in inspect.signature(model.rotary_emb.forward).parameters:
+        # TODO we assume that position_ids is not None here, not sure if that is safe but the old code also did that
+        cos, sin = model.rotary_emb(value_states, seq_len=seq_len)
+        return llama_apply_rotary_pos_emb(query_states, cos, sin, position_ids)
+
+    past_seen_tokens = 0
+    if position_ids is None:
+        # Compute position_ids, since they are required for transformers > 4.37.2
+        if past_key_value is None:
+            new_cache_positions = torch.arange(q_len, q_len + q_len, device=value_states.device)
+        else:
+            past_seen_tokens = past_key_value.get_usable_length(q_len, model.layer_idx)
+            new_cache_positions = torch.arange(past_seen_tokens, past_seen_tokens + q_len, device=value_states.device)
+        position_ids = new_cache_positions.unsqueeze(0)
+
+    rotary_emb_kwargs = {"position_ids": position_ids}
+    # The `seq_len` argument has been officially removed in transformers >= 4.39.0
+    if "seq_len" in inspect.signature(model.rotary_emb.forward).parameters:
+        rotary_emb_kwargs["seq_len"] = q_len + past_seen_tokens
+
+    cos, sin = model.rotary_emb(value_states, **rotary_emb_kwargs)
+
+    # For batched inference unsqueeze it on the correct dim
+    # since: https://github.com/huggingface/transformers/pull/29109
+    if len(cos.shape) == 3:
+        cos = cos.unsqueeze(1)
+        sin = sin.unsqueeze(1)
+
+    return (query_states * cos) + (llama_rotate_half(query_states) * sin)
+
+
+def is_adaption_prompt_trainable(params: str) -> bool:
+    """Return True if module is trainable under adaption prompt fine-tuning."""
+    return params.split(".")[-1].startswith("adaption_")

lib/python3.12/site-packages/peft/tuners/boft/__init__.py

@@ -0,0 +1,24 @@
+# Copyright 2023-present the HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from peft.utils import register_peft_method
+
+from .config import BOFTConfig
+from .layer import BOFTLayer
+from .model import BOFTModel
+
+
+__all__ = ["BOFTConfig", "BOFTLayer", "BOFTModel"]
+
+register_peft_method(name="boft", config_cls=BOFTConfig, model_cls=BOFTModel)

lib/python3.12/site-packages/peft/tuners/boft/config.py

@@ -0,0 +1,160 @@
+# Copyright 2023-present the HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# The implementation is based on "Parameter-Efficient Orthogonal Finetuning
+# via Butterfly Factorization" (https://arxiv.org/abs/2311.06243) in ICLR 2024.
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Optional, Union
+
+from peft.config import PeftConfig
+from peft.utils import PeftType
+
+
+@dataclass
+class BOFTConfig(PeftConfig):
+    """
+    This is the configuration class to store the configuration of a [`BOFTModel`].
+
+    Args:
+        boft_block_size (`int`): BOFT block size across different layers.
+        boft_block_num (`int`): Number of BOFT blocks per injected layer.
+        boft_n_butterfly_factor (`int`): Number of butterfly factors across different layers.
+        target_modules (`Union[List[str],str]`): The names of the modules to apply the adapter to.
+        exclude_modules (`Optional[Union[List[str], str]]`):
+            The names of the modules to not apply the adapter. When passing a string, a regex match will be performed.
+            When passing a list of strings, either an exact match will be performed or it is checked if the name of the
+            module ends with any of the passed strings.
+        boft_dropout (`float`):
+            The multiplicative dropout probability, by setting OFT blocks to identity during training, similar to the
+            dropout layer in LoRA.
+        fan_in_fan_out (`bool`): Set this to True if the layer to replace stores weight like (fan_in, fan_out).
+            For example, gpt-2 uses `Conv1D` which stores weights like (fan_in, fan_out) and hence this should be set
+            to `True`.
+        bias (`str`): Bias type for BOFT. Can be 'none', 'all' or 'boft_only'. If 'all' or 'boft_only', the
+            corresponding biases will be updated during training. Be aware that this means that, even when disabling
+            the adapters, the model will not produce the same output as the base model would have without adaptation.
+        modules_to_save (`List[str]`):List of modules apart from BOFT layers to be set as trainable
+            and saved in the final checkpoint.
+        layers_to_transform (`Union[List[int],int]`):
+            The layer indexes to transform, if this argument is specified, it will apply the BOFT transformations on
+            the layer indexes that are specified in this list. If a single integer is passed, it will apply the BOFT
+            transformations on the layer at this index.
+        layers_pattern (`Optional[Union[List[str], str]]`):
+            The layer pattern name, used only if `layers_to_transform` is different from `None` and if the layer
+            pattern is not in the common layers pattern. This should target the `nn.ModuleList` of the model, which is
+            often called `'layers'` or `'h'`.
+    """
+
+    boft_block_size: int = field(
+        default=4,
+        metadata={
+            "help": "BOFT block size across different layers.",
+            "note": "You can only specify either boft_block_size or boft_block_num, but not both simultaneously, because boft_block_size x boft_block_num = layer dimension.",
+        },
+    )
+    boft_block_num: int = field(
+        default=0,
+        metadata={
+            "help": "Number of BOFT blocks per injected layer.",
+            "note": "You can only specify either boft_block_size or boft_block_num, but not both simultaneously, because boft_block_size x boft_block_num = layer dimension.",
+        },
+    )
+    boft_n_butterfly_factor: int = field(
+        default=1,
+        metadata={
+            "help": "Number of butterfly factors.",
+            "note": (
+                "for example, boft_n_butterfly_factor=2, the effective block size of OFT becomes twice as big and the number of blocks become half.",
+                "note: for boft_n_butterfly_factor=1, BOFT is the same as vanilla OFT.",
+            ),
+        },
+    )
+    target_modules: Optional[Union[list[str], str]] = field(
+        default=None,
+        metadata={
+            "help": "List of module names or regex expression of the module names to replace with BOFT.",
+            "example": "For example, ['q', 'v'] or '.*decoder.*(SelfAttention|EncDecAttention).*(q|v)$' ",
+        },
+    )
+    exclude_modules: Optional[Union[list[str], str]] = field(
+        default=None,
+        metadata={"help": "List of module names or regex expression of the module names to exclude from BOFT."},
+    )
+    boft_dropout: float = field(
+        default=0.0,
+        metadata={
+            "help": "BOFT multiplicative dropout, randomly setting blocks of OFT to be identity matrix, similar to the dropout layer in LoRA."
+        },
+    )
+    fan_in_fan_out: bool = field(
+        default=False,
+        metadata={"help": "Set this to True if the layer to replace stores weight like (fan_in, fan_out)"},
+    )
+    bias: str = field(default="none", metadata={"help": "Bias type for BOFT. Can be 'none', 'all' or 'boft_only'"})
+    modules_to_save: Optional[list[str]] = field(
+        default=None,
+        metadata={
+            "help": "List of modules apart from BOFT layers to be set as trainable and saved in the final checkpoint. ",
+            "note": (
+                "For example, in Sequence Classification or Token Classification tasks, ",
+                "the final layer `classifier/score` are randomly initialized and as such need to be trainable and saved.",
+            ),
+        },
+    )
+    init_weights: bool = field(
+        default=True,
+        metadata={
+            "help": (
+                "Whether to initialize the weights of the BOFT layers with their default initialization. Don't change ",
+                "this setting, except if you know exactly what you're doing.",
+            ),
+        },
+    )
+    layers_to_transform: Optional[Union[list[int], int]] = field(
+        default=None,
+        metadata={
+            "help": "The layer indexes to transform, is this argument is specified, PEFT will transform only the layers indexes that are specified inside this list. If a single integer is passed, PEFT will transform only the layer at this index."
+        },
+    )
+    layers_pattern: Optional[Union[list[str], str]] = field(
+        default=None,
+        metadata={
+            "help": "The layer pattern name, used only if `layers_to_transform` is different to None and if the layer pattern is not in the common layers pattern. "
+            "This should target the `nn.ModuleList` of the model, which is often called `'layers'` or `'h'`."
+        },
+    )
+
+    def __post_init__(self):
+        super().__post_init__()
+        self.peft_type = PeftType.BOFT
+        self.target_modules = (
+            set(self.target_modules) if isinstance(self.target_modules, list) else self.target_modules
+        )
+        self.exclude_modules = (
+            set(self.exclude_modules) if isinstance(self.exclude_modules, list) else self.exclude_modules
+        )
+        # check for layers_to_transform and layers_pattern
+        if self.layers_pattern and not self.layers_to_transform:
+            raise ValueError("When `layers_pattern` is specified, `layers_to_transform` must also be specified. ")
+        if self.boft_block_size == 0 and self.boft_block_num == 0:
+            raise ValueError(
+                f"Either `boft_block_size` or `boft_block_num` must be non-zero. Currently, boft_block_size = {self.boft_block_size} and boft_block_num = {self.boft_block_num}."
+            )
+        if not (self.boft_block_size != 0) ^ (self.boft_block_num != 0):
+            raise ValueError(
+                f"You can only specify either boft_block_size ({self.boft_block_size}) or boft_block_num ({self.boft_block_num}), but not both simultaneously, because boft_block_size x boft_block_num == in_features."
+            )

lib/python3.12/site-packages/peft/tuners/boft/fbd/fbd_cuda.cpp

@@ -0,0 +1,28 @@
+#include <torch/torch.h>
+#include <vector>
+#include <iostream>
+#include <torch/extension.h>
+
+std::vector<at::Tensor> forward_fast_block_diag_cuda(
+        at::Tensor input);
+
+std::vector<at::Tensor> forward_fast_block_diag(
+        at::Tensor input
+        ) {
+    return forward_fast_block_diag_cuda(input);
+}
+
+std::vector<at::Tensor> backward_fast_block_diag_cuda(
+        at::Tensor grad_output, 
+        at::Tensor input);
+std::vector<at::Tensor> backward_fast_block_diag(
+        at::Tensor grad_output,
+        at::Tensor input
+        ) {
+    return backward_fast_block_diag_cuda(grad_output, input);
+}
+
+PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
+    m.def("forward", &forward_fast_block_diag, "FAST BLOCK DIAG (CUDA)");
+    m.def("backward", &backward_fast_block_diag, "FAST BLOCK DIAG backward (CUDA)");
+}

lib/python3.12/site-packages/peft/tuners/boft/fbd/fbd_cuda_kernel.cu

@@ -0,0 +1,109 @@
+// Author: Yao Feng
+// Date: 2023/08
+// Description: cuda kernel for fast block diag
+
+#include <ATen/ATen.h>
+
+#include <cuda.h>
+#include <cuda_runtime.h>
+#include <vector>
+
+namespace{
+template <typename scalar_t>
+__global__ void forward_fast_block_diag_cuda_kernel(
+        const scalar_t* __restrict__ input, //[z, N, b, b]
+        scalar_t*  output, //[z, Nxb, Nxb]
+        int z, int N, int b
+    ) {
+
+    const int i = blockIdx.x * blockDim.x + threadIdx.x;
+    if (i >= z*N*b*b) {
+        return;
+    }
+    const int zi = i/(N*b*b);
+    const int Ni = (i%(N*b*b))/(b*b);
+    const int x = ((i%(N*b*b))%(b*b))/b;
+    const int y = ((i%(N*b*b))%(b*b))%b;
+
+    output[zi*N*b*N*b + (Ni*b+x)*N*b + Ni*b + y] = input[zi*N*b*b + Ni*b*b + x*b + y];
+
+}
+
+template <typename scalar_t>
+__global__ void backward_fast_block_diag_cuda_kernel(
+        const scalar_t* __restrict__ grad_output, 
+        scalar_t*  grad_input, 
+        int z, int N, int b
+    ) {
+
+    const int i = blockIdx.x * blockDim.x + threadIdx.x;
+    if (i >= z*N*b*b) {
+        return;
+    }
+    const int zi = i/(N*b*b);
+    const int Ni = (i%(N*b*b))/(b*b);
+    const int x = ((i%(N*b*b))%(b*b))/b;
+    const int y = ((i%(N*b*b))%(b*b))%b;
+    
+    grad_input[zi*N*b*b + Ni*b*b + x*b + y] = grad_output[zi*N*b*N*b + (Ni*b+x)*N*b + Ni*b + y];
+
+} // namespace
+}
+
+std::vector<at::Tensor> forward_fast_block_diag_cuda(
+    at::Tensor input
+    ){
+    const auto z = input.size(0);
+    const auto N = input.size(1);
+    const auto b = input.size(2);
+
+    // print(channel_size)
+    const int threads = 512;
+    const dim3 blocks_1 ((z*N*b*b - 1) / threads +1);
+    // initlaize output
+    auto output = at::zeros({z, N*b, N*b}, input.options());
+
+    AT_DISPATCH_FLOATING_TYPES_AND_HALF(input.type(), "forward_fast_block_diag1", ([&] {
+        forward_fast_block_diag_cuda_kernel<scalar_t><<<blocks_1, threads>>>(
+        input.data_ptr<scalar_t>(),
+        output.data_ptr<scalar_t>(),
+        z, N, b);
+      }));
+
+   
+    cudaError_t err = cudaGetLastError();
+    if (err != cudaSuccess) 
+            printf("Error in forward_fast_block_diag_cuda_kernel: %s\n", cudaGetErrorString(err));
+
+    return {output};
+}
+
+std::vector<at::Tensor> backward_fast_block_diag_cuda(
+    at::Tensor grad_output,
+    at::Tensor input
+    ){
+
+    const auto z = input.size(0);
+    const auto N = input.size(1);
+    const auto b = input.size(2);
+    
+    // print(channel_size)
+    const int threads = 512;
+    const dim3 blocks_1 ((z*N*b*b - 1) / threads +1);
+    
+    // initialize grad input
+    auto grad_input = at::zeros_like(input);
+
+    AT_DISPATCH_FLOATING_TYPES_AND_HALF(grad_output.type(), "backward_fast_block_diag", ([&] {
+        backward_fast_block_diag_cuda_kernel<scalar_t><<<blocks_1, threads>>>(
+        grad_output.data_ptr<scalar_t>(),
+        grad_input.data_ptr<scalar_t>(),
+        z, N, b);
+      }));
+    
+    cudaError_t err = cudaGetLastError();
+    if (err != cudaSuccess) 
+            printf("Error in backward_fast_block_diag_cuda_kernel: %s\n", cudaGetErrorString(err));
+
+    return {grad_input};
+}

lib/python3.12/site-packages/peft/tuners/boft/layer.py

@@ -0,0 +1,984 @@
+# Copyright 2023-present the HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# The implementation is based on "Parameter-Efficient Orthogonal Finetuning
+# via Butterfly Factorization" (https://arxiv.org/abs/2311.06243) in ICLR 2024.
+
+from __future__ import annotations
+
+import math
+import os
+import warnings
+from contextlib import contextmanager
+from typing import Any, Optional, Union
+
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from torch.autograd import Function
+
+from peft.tuners.tuners_utils import BaseTunerLayer, check_adapters_to_merge
+
+
+_FBD_CUDA = None
+
+
+# this function is a 1:1 copy from accelerate
+@contextmanager
+def patch_environment(**kwargs):
+    """
+    A context manager that will add each keyword argument passed to `os.environ` and remove them when exiting.
+
+    Will convert the values in `kwargs` to strings and upper-case all the keys.
+
+    Example:
+
+    ```python
+    >>> import os
+    >>> from accelerate.utils import patch_environment
+
+    >>> with patch_environment(FOO="bar"):
+    ...     print(os.environ["FOO"])  # prints "bar"
+    >>> print(os.environ["FOO"])  # raises KeyError
+    ```
+    """
+    existing_vars = {}
+    for key, value in kwargs.items():
+        key = key.upper()
+        if key in os.environ:
+            existing_vars[key] = os.environ[key]
+        os.environ[key] = str(value)
+
+    yield
+
+    for key in kwargs:
+        key = key.upper()
+        if key in existing_vars:
+            # restore previous value
+            os.environ[key] = existing_vars[key]
+        else:
+            os.environ.pop(key, None)
+
+
+def get_fbd_cuda():
+    global _FBD_CUDA
+
+    if _FBD_CUDA is not None:
+        return _FBD_CUDA
+
+    # This import initializes cuda context and should thus be local, see issue 1877
+    from torch.utils.cpp_extension import load
+
+    curr_dir = os.path.dirname(__file__)
+    # need ninja to build the extension
+    try:
+        with patch_environment(CC="gcc", CXX="gcc"):
+            fbd_cuda = load(
+                name="fbd_cuda",
+                sources=[f"{curr_dir}/fbd/fbd_cuda.cpp", f"{curr_dir}/fbd/fbd_cuda_kernel.cu"],
+                verbose=True,
+                # build_directory='/tmp/'  # for debugging
+            )
+            # extra_cuda_cflags = ['-std=c++14', '-ccbin=$$(which gcc-7)']) # cuda10.2 is not compatible with gcc9. Specify gcc 7
+    except Exception as e:
+        warnings.warn(f"Failed to load the CUDA extension: {e}, check if ninja is available.")
+        warnings.warn("Setting boft_n_butterfly_factor to 1 to speed up the finetuning process.")
+        fbd_cuda = None
+
+    _FBD_CUDA = fbd_cuda
+    return _FBD_CUDA
+
+
+class FastBlockDiag(Function):
+    """
+    Implements a custom autograd Function for a fast block diagonal operation using CUDA.
+
+    This function is optimized for 4D tensors where the last two dimensions are equal, representing block diagonal
+    matrices for efficient computation on CUDA devices.
+    """
+
+    @staticmethod
+    def forward(ctx, input):
+        """
+        The forward method for FastBlockDiag.
+
+        Computes the block diagonal operation on the input tensor using a CUDA-optimized function. This method assumes
+        that the input is a 4D tensor where the last two dimensions are equal, which represent the blocks to be
+        diagonalized.
+
+        Parameters:
+        ctx: A context object that can be used to stash information for backward computation.
+        input (Tensor): The input tensor of shape (N, D, H, H), where `N` is the batch size,
+                        `D` represents one additional dimension (In BOFT, the number of BOFT blocks), and `H` is the
+                        size of the square blocks along the last two dimensions (In BOFT, the block size).
+
+        Returns:
+        Tensor: The resulting tensor after applying the block diagonal operation,
+                will have the shape (N, DxH, DxH).
+        """
+        output = get_fbd_cuda().forward(input)[0]
+        ctx.save_for_backward(input)
+        return output
+
+    @staticmethod
+    def backward(ctx, grad_output):
+        (input,) = ctx.saved_tensors
+        grad_input = get_fbd_cuda().backward(grad_output, input)[0]
+        return grad_input
+
+
+class MultiplicativeDropoutLayer(nn.Module):
+    """
+    Implements the multiplicative dropout layer for BOFT.
+    """
+
+    def __init__(self, p=0.0):
+        """
+        Initializes the multiplicative dropout layer.
+
+        Parameters:
+        p (float): The probability of dropping out a block. Defaults to 0.0.
+        """
+        super().__init__()
+        self.p = p
+
+    def forward(self, x):
+        """
+        Applies multiplicative dropout to the input tensor.
+
+        Parameters:
+        x (Tensor): The input tensor of shape (N, D, H, H), where `N` is the batch size, `D` represents
+                    one additional dimension (In BOFT, the number of BOFT blocks), and `H` is the size of the square
+                    blocks along the last two dimensions (In BOFT, the block size).
+        """
+        if self.training:
+            # Ensure the last two dimensions are the same
+            if x.shape[-1] != x.shape[-2]:
+                raise ValueError("The last two dimensions of input should be the same!")
+
+            N, D, H, _ = x.shape
+
+            # Randomly select one from N
+            n_random = torch.randint(0, N, (1,)).item()
+
+            # Create a mask with 1s for matrices to be replaced with identity and 0s otherwise
+            num_to_replace = int(self.p * D)
+            num_zeros = D - num_to_replace
+
+            # Generate a flat tensor with desired number of 1s and 0s
+            mask = torch.cat([torch.ones(num_to_replace, device=x.device), torch.zeros(num_zeros, device=x.device)])
+
+            # Shuffle and reshape the mask
+            mask = mask[torch.randperm(D)].view(1, D, 1, 1)
+
+            full_mask = torch.zeros(N, D, 1, 1, device=x.device)
+            full_mask[n_random] = mask
+
+            # Use the mask to combine original matrices and identity matrices
+            eye_matrix = torch.eye(H, device=x.device).repeat(N, D, 1, 1)
+            x = (1 - full_mask) * x + full_mask * eye_matrix
+        return x
+
+
+class BOFTLayer(BaseTunerLayer):
+    """
+    Implements the BOFT layer.
+    """
+
+    # All names of layers that may contain (trainable) adapter weights
+    adapter_layer_names = ("boft_R", "boft_s")
+    # All names of other parameters that may contain adapter-related parameters
+    other_param_names = ("boft_block_size", "boft_block_num", "boft_dropout")
+
+    def __init__(self, base_layer: nn.Module, **kwargs) -> None:
+        """
+        Initializes the BOFT layer.
+
+        Note, currently only support linear layer and convolutional layer, with further support for other layers to be
+        added soon.
+
+        Parameters:
+        base_layer: the pretrained model layer
+        """
+        self.base_layer = base_layer
+        self.boft_block_size = {}
+        self.boft_block_num = {}
+        self.boft_dropout = nn.ModuleDict({})
+        self.boft_R = nn.ParameterDict({})
+        self.boft_s = nn.ParameterDict({})
+        # Mark the weight as unmerged
+        self._disable_adapters = False
+        self.merged_adapters = []
+        self.kwargs = kwargs
+
+        base_layer = self.get_base_layer()
+
+        if isinstance(base_layer, nn.Linear):
+            in_features, out_features = base_layer.in_features, base_layer.out_features
+        elif isinstance(base_layer, nn.Conv2d):
+            in_features, out_features = base_layer.in_channels, base_layer.out_channels
+        else:
+            raise ValueError(f"Unsupported layer type {type(base_layer)}")
+
+        self.in_features = in_features
+        self.out_features = out_features
+
+    def set_scale(self, adapter, scale):
+        if adapter not in self.scaling:
+            # Ignore the case where the adapter is not in the layer
+            return
+
+        warnings.warn("Scaling operation for BOFT not supported! Automatically set scale to 1.")
+
+    def scale_layer(self, scale: float) -> None:
+        if scale == 1:
+            return
+
+        for active_adapter in self.active_adapters:
+            if active_adapter not in self.boft_R.keys():
+                continue
+
+            warnings.warn("Scaling operation for BOFT not supported! Automatically set scale to 1.")
+
+    def unscale_layer(self, scale=None) -> None:
+        for active_adapter in self.active_adapters:
+            if active_adapter not in self.boft_R.keys():
+                continue
+
+            warnings.warn("Unscaling operation for BOFT not supported! Keeping scale to 1.")
+
+    def update_layer(
+        self, adapter_name, boft_block_size, boft_block_num, boft_n_butterfly_factor, boft_dropout, init_weights
+    ):
+        """
+        Update the linear layer with trainable BOFT weights. Override for other layer types.
+        """
+        # Attempt to load the CUDA extension during model initialization
+        if not get_fbd_cuda():
+            self.fbd_cuda_available = False
+            # If the CUDA extension is not available, set the butterfly factor to 1 to speed up the finetuning process
+            boft_n_butterfly_factor = 1
+        else:
+            self.fbd_cuda_available = True
+
+        # to be consistent with the paper notation
+        boft_n_butterfly_factor = boft_n_butterfly_factor - 1
+        if boft_n_butterfly_factor < 0:
+            raise ValueError(
+                f"You can only specify boft_n_butterfly_factor {boft_n_butterfly_factor + 1} to be a positive integer number."
+            )
+
+        # Initialize the MultiplicativeDropoutLayer for boft_dropout > 0.0.
+        if boft_dropout > 0.0:
+            boft_dropout_layer = MultiplicativeDropoutLayer(p=boft_dropout)
+        else:
+            boft_dropout_layer = nn.Identity()
+        self.boft_dropout.update(nn.ModuleDict({adapter_name: boft_dropout_layer}))
+
+        if boft_block_size == 0 and boft_block_num != 0:
+            if self.in_features % boft_block_num != 0:
+                raise ValueError(
+                    f"in_features ({self.in_features}) must be divisible by boft_block_num ({boft_block_num})!"
+                )
+
+            if boft_n_butterfly_factor != 0:
+                if boft_n_butterfly_factor > int(math.log2(boft_block_num)):
+                    raise ValueError(
+                        f"Invalid combination of boft_n_butterfly_factor ({boft_n_butterfly_factor + 1}) and boft_block_num ({boft_block_num})!"
+                    )
+                if boft_block_num % (2**boft_n_butterfly_factor) != 0:
+                    raise ValueError(
+                        f"boft_block_num ({boft_block_num}) must be a multiple of 2 raised to the power of boft_n_butterfly_factor ({boft_n_butterfly_factor + 1})!"
+                    )
+
+            boft_block_size = int(self.in_features // boft_block_num)
+
+        elif boft_block_size != 0 and boft_block_num == 0:
+            if self.in_features % boft_block_size != 0:
+                raise ValueError(
+                    f"in_features ({self.in_features}) must be divisible by boft_block_size ({boft_block_size})!"
+                )
+
+            if boft_n_butterfly_factor != 0:
+                if self.in_features < (boft_block_size * (2**boft_n_butterfly_factor)):
+                    raise ValueError(
+                        f"Invalid combination of in_features ({self.in_features}), boft_n_butterfly_factor ({boft_n_butterfly_factor + 1}) and boft_block_size ({boft_block_size})!"
+                    )
+                if self.in_features % (boft_block_size * (2**boft_n_butterfly_factor)) != 0:
+                    raise ValueError(
+                        f"Invalid combination of in_features ({self.in_features}), boft_n_butterfly_factor ({boft_n_butterfly_factor + 1}) and boft_block_size ({boft_block_size})!"
+                    )
+
+            boft_block_num = int(self.in_features // boft_block_size)
+
+        else:
+            raise ValueError(
+                "Something went wrong, please report this error: https://github.com/huggingface/peft/issues"
+            )
+
+        # In OFT you can specify the number of blocks to be 1
+        if boft_n_butterfly_factor != 0:
+            if boft_block_num % 2 != 0:
+                raise ValueError(f"boft_block_num ({boft_block_num}) must be an even number!")
+
+            if boft_block_size % 2 != 0:
+                raise ValueError(f"boft_block_size ({boft_block_size}) must be an even number!")
+
+        # If there is no butterfly factor, then permutation matrix P will be an identity matrix.
+        P = torch.empty((boft_n_butterfly_factor + 1, self.in_features, self.in_features))
+        for i in range(boft_n_butterfly_factor + 1):
+            perm = self.block_butterfly_perm(
+                self.in_features, int(boft_block_num / (2 ** (i))), int(boft_block_size / 2), boft_n_butterfly_factor
+            )
+            perm_mat = self.perm2mat(perm)
+            P[i] = perm_mat
+
+        self.register_buffer("boft_P", P, persistent=False)
+
+        self.boft_R[adapter_name] = nn.Parameter(
+            torch.zeros(boft_n_butterfly_factor + 1, boft_block_num, boft_block_size, boft_block_size)
+        )
+        self.boft_s[adapter_name] = nn.Parameter(torch.ones(int(self.out_features), 1))
+
+        self.reset_boft_parameters(adapter_name, init_weights)
+
+        # set the boft block size and number
+        self.boft_block_size[adapter_name] = boft_block_size
+        self.boft_block_num[adapter_name] = boft_block_num
+
+        self._move_adapter_to_device_of_base_layer(adapter_name)
+        self.set_adapter(self.active_adapters)
+
+    def reset_boft_parameters(self, adapter_name, init_weights):
+        """
+        Reset the BOFT parameters.
+        """
+        if init_weights is False:
+            nn.init.normal_(self.boft_R[adapter_name], mean=0.0, std=0.1)
+            nn.init.normal_(self.boft_s[adapter_name], mean=1.0, std=0.1)
+            return
+
+        if adapter_name in self.boft_R.keys():
+            if init_weights is True:
+                # initialize R to zero
+                nn.init.zeros_(self.boft_R[adapter_name])
+                nn.init.ones_(self.boft_s[adapter_name])
+            else:
+                raise ValueError(f"Unknown initialization {init_weights=}")
+
+    def perm2mat(self, indices):
+        """
+        Convert permutation indices to permutation matrix.
+
+        Args:
+        indices: A list of indices representing the permutation.
+        """
+        # Number of indices determines the size of the square matrix
+        n = len(indices)
+
+        # Initialize a matrix of zeros
+        perm_mat = torch.zeros((n, n))
+
+        # Set the 1s according to the indices
+        for i, idx in enumerate(indices):
+            perm_mat[i, idx] = 1
+
+        return perm_mat
+
+    def block_butterfly_perm(self, n, b, r=3, n_butterfly_factor=1):
+        """
+        Define the permutation matrix for the block butterfly permutation.
+
+        Args:
+        n: size of the permutation matrix
+        b: desired number of blocks after multiplying with the permutation matrix
+        r: base block size of the block diagonal matrix, e.g. 2x2, 3x3, 5x5 etc.
+        """
+
+        if n_butterfly_factor == 0:
+            return torch.arange(n)
+
+        if b * r * 2 > n:
+            raise ValueError("Invalid number of blocks!")
+
+        block_size = int(n // b)
+        indices = torch.arange(n)
+
+        def sort_block(b, r):
+            step = b / r
+            initial_order = torch.arange(b)
+            sorted_order = torch.empty(b, dtype=torch.long)
+
+            evens = torch.arange(0, step, 2)
+            odds = torch.arange(1, step, 2)
+            sorted_seq = torch.cat((evens, odds), dim=0)
+            for i, pos in enumerate(sorted_seq):
+                sorted_order[int(i * r) : int(i * r + r)] = initial_order[int(pos * r) : int(pos * r + r)]
+            return sorted_order
+
+        sorted_order = sort_block(block_size, r)
+
+        for i in range(0, n, block_size):
+            block_end = i + block_size
+            tmp_indices = indices[i:block_end]
+            indices[i:block_end] = tmp_indices[sorted_order]
+        return indices
+
+    def cayley_batch(self, data):
+        """
+        Perform the Cayley parametrization on a batch of skew-symmetric matrices.
+
+        Args:
+            data: A batch of skew-symmetric matrices of shape (b, r, c).
+        """
+        b, r, c = data.shape
+        # Ensure the input matrix is skew-symmetric
+        skew_mat = 0.5 * (data - data.transpose(1, 2))
+        id_mat = torch.eye(r, device=data.device).unsqueeze(0).expand(b, r, c)
+
+        # Perform the Cayley parametrization
+        Q = torch.linalg.solve(id_mat + skew_mat, id_mat - skew_mat, left=False)
+
+        return Q
+
+
+class Linear(nn.Module, BOFTLayer):
+    """
+    BOFT implemented in a dense layer.
+    """
+
+    def __init__(
+        self,
+        base_layer,
+        adapter_name: str,
+        boft_block_size: int = 8,
+        boft_block_num: int = 0,
+        boft_n_butterfly_factor: int = 0,
+        boft_dropout: float = 0.1,
+        fan_in_fan_out: bool = False,  # Set this to True if the layer to replace stores weight like (fan_in, fan_out)
+        init_weights: Union[bool, str] = True,
+        is_target_conv_1d_layer: bool = False,
+        **kwargs,
+    ) -> None:
+        super().__init__()
+        BOFTLayer.__init__(self, base_layer, **kwargs)
+        self.fan_in_fan_out = fan_in_fan_out
+
+        self._active_adapter = adapter_name
+
+        self.update_layer(
+            adapter_name, boft_block_size, boft_block_num, boft_n_butterfly_factor, boft_dropout, init_weights
+        )
+        self.is_target_conv_1d_layer = is_target_conv_1d_layer
+
+    def merge(self, safe_merge: bool = False, adapter_names: Optional[list[str]] = None) -> None:
+        """
+        Merge the active adapter weights into the base weights
+
+        Args:
+            safe_merge (`bool`, *optional*):
+                If True, the merge operation will be performed in a copy of the original weights and check for NaNs
+                before merging the weights. This is useful if you want to check if the merge operation will produce
+                NaNs. Defaults to `False`.
+            adapter_names (`List[str]`, *optional*):
+                The list of adapter names that should be merged. If None, all active adapters will be merged. Defaults
+                to `None`.
+        """
+        adapter_names = check_adapters_to_merge(self, adapter_names)
+        if not adapter_names:
+            # no adapter to merge
+            return
+
+        for active_adapter in adapter_names:
+            if active_adapter in self.boft_R.keys():
+                base_layer = self.get_base_layer()
+                if safe_merge:
+                    # Note that safe_merge will be slower than the normal merge
+                    # because of the copy operation.
+                    orig_weight = base_layer.weight.data.clone()
+                    butterfly_oft_mat, boft_s = self.get_delta_weight(active_adapter)
+                    orig_weight = torch.transpose(orig_weight, 0, 1)
+                    orig_weight = torch.mm(butterfly_oft_mat, orig_weight)
+                    orig_weight = torch.transpose(orig_weight, 0, 1)
+                    orig_weight = orig_weight * boft_s
+
+                    if not torch.isfinite(orig_weight).all():
+                        raise ValueError(
+                            f"NaNs detected in the merged weights. The adapter {active_adapter} seems to be broken"
+                        )
+
+                    self.base_layer.weight.data = orig_weight.contiguous()
+                else:
+                    butterfly_oft_mat, boft_s = self.get_delta_weight(active_adapter)
+                    orig_weight = base_layer.weight.data.clone()
+                    orig_weight = torch.transpose(orig_weight, 0, 1)
+                    orig_weight = torch.mm(butterfly_oft_mat, orig_weight)
+                    orig_weight = torch.transpose(orig_weight, 0, 1)
+                    orig_weight = orig_weight * boft_s
+
+                    self.base_layer.weight.data = orig_weight.contiguous()
+
+                self.merged_adapters.append(active_adapter)
+
+    def unmerge(self) -> None:
+        """
+        This method unmerges all merged adapter layers from the base weights.
+        """
+        if not self.merged:
+            warnings.warn("Already unmerged. Nothing to do.")
+            return
+        while len(self.merged_adapters) > 0:
+            active_adapter = self.merged_adapters.pop()
+            if active_adapter in self.boft_R.keys():
+                butterfly_oft_mat, boft_s = self.get_delta_weight(active_adapter)
+
+                orig_weight = self.get_base_layer().weight.data.clone()
+                orig_weight = torch.transpose(orig_weight, 0, 1)
+                orig_weight = torch.mm(butterfly_oft_mat.t(), orig_weight)
+                orig_weight = torch.transpose(orig_weight, 0, 1)
+
+                self.get_base_layer().weight.data = orig_weight * (1 / boft_s)
+
+    def get_delta_weight(self, adapter) -> tuple[torch.Tensor, torch.Tensor]:
+        """
+        Compute the delta weight for the given adapter.
+
+        Args:
+            adapter (str):
+                The name of the adapter for which the delta weight should be computed.
+        """
+        boft_R = self.boft_R[adapter]
+        boft_s = self.boft_s[adapter]
+
+        N, D, H, _ = boft_R.shape
+        boft_R = boft_R.view(N * D, H, H)
+        orth_rotate_butterfly = self.cayley_batch(boft_R)
+        orth_rotate_butterfly = orth_rotate_butterfly.view(N, D, H, H)
+        if self.fbd_cuda_available:
+            block_diagonal_butterfly = FastBlockDiag.apply(orth_rotate_butterfly)
+        else:
+            orth_rotate_butterfly = orth_rotate_butterfly.squeeze(0)
+            block_diagonal_butterfly = torch.block_diag(*torch.unbind(orth_rotate_butterfly))
+            block_diagonal_butterfly = block_diagonal_butterfly.unsqueeze(0)
+
+        boft_P = self.boft_P.to(block_diagonal_butterfly.device)
+        butterfly_oft_mat_batch = torch.bmm(block_diagonal_butterfly, boft_P.permute(0, 2, 1))
+        butterfly_oft_mat_batch = torch.bmm(boft_P, butterfly_oft_mat_batch)
+        butterfly_oft_mat = butterfly_oft_mat_batch[0]
+
+        for i in range(1, butterfly_oft_mat_batch.shape[0]):
+            butterfly_oft_mat = butterfly_oft_mat_batch[i] @ butterfly_oft_mat
+
+        return butterfly_oft_mat, boft_s
+
+    def forward(self, x: torch.Tensor, *args: Any, **kwargs: Any) -> torch.Tensor:
+        previous_dtype = x.dtype
+
+        if self.disable_adapters:
+            if self.merged:
+                self.unmerge()
+            result = self.base_layer(x, *args, **kwargs)
+        elif self.merged:
+            result = self.base_layer(x, *args, **kwargs)
+        else:
+            boft_rotation = torch.eye(self.in_features, device=x.device, dtype=previous_dtype)
+            boft_scale = torch.ones((int(self.out_features), 1), device=x.device, dtype=previous_dtype)
+
+            for active_adapter in self.active_adapters:
+                if active_adapter not in self.boft_R.keys():
+                    continue
+                boft_R = self.boft_R[active_adapter]
+                boft_s = self.boft_s[active_adapter]
+                dropout = self.boft_dropout[active_adapter]
+
+                N, D, H, _ = boft_R.shape
+                boft_R = boft_R.view(N * D, H, H)
+                orth_rotate_butterfly = self.cayley_batch(boft_R)
+                orth_rotate_butterfly = orth_rotate_butterfly.view(N, D, H, H)
+                orth_rotate_butterfly = dropout(orth_rotate_butterfly)
+                if self.fbd_cuda_available:
+                    block_diagonal_butterfly = FastBlockDiag.apply(orth_rotate_butterfly)
+                else:
+                    orth_rotate_butterfly = orth_rotate_butterfly.squeeze(0)
+                    block_diagonal_butterfly = torch.block_diag(*torch.unbind(orth_rotate_butterfly))
+                    block_diagonal_butterfly = block_diagonal_butterfly.unsqueeze(0)
+
+                # The BOFT author's cayley_batch, dropout and FastBlockDiag ONLY return fp32 outputs.
+                boft_P = self.boft_P.to(x)
+                block_diagonal_butterfly = block_diagonal_butterfly.to(x)
+                butterfly_oft_mat_batch = torch.bmm(block_diagonal_butterfly, boft_P.permute(0, 2, 1))
+                butterfly_oft_mat_batch = torch.bmm(boft_P, butterfly_oft_mat_batch)
+                butterfly_oft_mat = butterfly_oft_mat_batch[0]
+
+                for i in range(1, butterfly_oft_mat_batch.shape[0]):
+                    butterfly_oft_mat = butterfly_oft_mat_batch[i] @ butterfly_oft_mat
+
+                boft_rotation = butterfly_oft_mat @ boft_rotation
+                boft_scale = boft_s * boft_scale
+
+            x = x.to(self.get_base_layer().weight.data.dtype)
+
+            orig_weight = self.get_base_layer().weight.data
+            orig_weight = torch.transpose(orig_weight, 0, 1)
+            boft_rotation = boft_rotation.to(previous_dtype)
+            orig_weight = orig_weight.to(previous_dtype)
+            rotated_weight = torch.mm(boft_rotation, orig_weight)
+            rotated_weight = torch.transpose(rotated_weight, 0, 1)
+
+            scaled_rotated_weight = rotated_weight * boft_scale
+
+            scaled_rotated_weight = scaled_rotated_weight.to(previous_dtype)
+            if self.base_layer.bias is not None:
+                self.base_layer.bias = self.base_layer.bias.to(previous_dtype)
+            result = F.linear(input=x, weight=scaled_rotated_weight, bias=self.base_layer.bias)
+
+        result = result.to(previous_dtype)
+        return result
+
+    def __repr__(self) -> str:
+        rep = super().__repr__()
+        return "boft." + rep
+
+
+class Conv2d(nn.Module, BOFTLayer):
+    """
+    BOFT implemented in a Conv2d layer.
+    """
+
+    def __init__(
+        self,
+        base_layer: nn.Module,
+        adapter_name: str,
+        boft_block_size: int = 8,
+        boft_block_num: int = 0,
+        boft_n_butterfly_factor: int = 0,
+        boft_dropout: float = 0.1,
+        init_weights: Union[bool, str] = True,
+        **kwargs,
+    ) -> None:
+        super().__init__()
+        BOFTLayer.__init__(self, base_layer)
+
+        self._active_adapter = adapter_name
+        self.update_layer(
+            adapter_name, boft_block_size, boft_block_num, boft_n_butterfly_factor, boft_dropout, init_weights
+        )
+
+    def update_layer(
+        self, adapter_name, boft_block_size, boft_block_num, boft_n_butterfly_factor, boft_dropout, init_weights
+    ):
+        """
+        Update the conv2d layer with trainable BOFT weights.
+        """
+
+        # Attempt to load the CUDA extension during model initialization
+        if not get_fbd_cuda():
+            self.fbd_cuda_available = False
+            # If the CUDA extension is not available, set the butterfly factor to 1 to speed up the finetuning process
+            boft_n_butterfly_factor = 1
+        else:
+            self.fbd_cuda_available = True
+
+        # to be consistent with the paper notation
+        boft_n_butterfly_factor = boft_n_butterfly_factor - 1
+        if boft_n_butterfly_factor < 0:
+            raise ValueError(
+                f"You can only specify boft_n_butterfly_factor {boft_n_butterfly_factor + 1} to be a positive integer number."
+            )
+
+        # Initialize the MultiplicativeDropoutLayer for boft_dropout > 0.0.
+        if boft_dropout > 0.0:
+            boft_dropout_layer = MultiplicativeDropoutLayer(p=boft_dropout)
+        else:
+            boft_dropout_layer = nn.Identity()
+        self.boft_dropout.update(nn.ModuleDict({adapter_name: boft_dropout_layer}))
+
+        # layer information from the base layer
+        base_layer = self.get_base_layer()
+        conv_filter_dim = self.in_features * base_layer.kernel_size[0] * base_layer.kernel_size[0]
+
+        # Initialize the BOFT parameters.
+        if boft_block_size == 0 and boft_block_num != 0:
+            if conv_filter_dim % boft_block_num != 0:
+                raise ValueError(
+                    f"Convolutional kernel dimension ({conv_filter_dim}) must be divisible by boft_block_num ({boft_block_num})!"
+                )
+
+            if boft_n_butterfly_factor != 0:
+                if boft_n_butterfly_factor > int(math.log2(boft_block_num)):
+                    raise ValueError(
+                        f"Invalid combination of boft_n_butterfly_factor ({boft_n_butterfly_factor + 1}) and boft_block_num ({boft_block_num})!"
+                    )
+                if boft_block_num % (2**boft_n_butterfly_factor) != 0:
+                    raise ValueError(
+                        f"boft_block_num ({boft_block_num}) must be a multiple of 2 raised to the power of boft_n_butterfly_factor ({boft_n_butterfly_factor + 1})!"
+                    )
+
+            boft_block_size = int(conv_filter_dim // boft_block_num)
+
+        elif boft_block_size != 0 and boft_block_num == 0:
+            if conv_filter_dim % boft_block_size != 0:
+                raise ValueError(
+                    f"Convolutional kernel dimension ({conv_filter_dim}) must be divisible by boft_block_size ({boft_block_size})!"
+                )
+
+            if boft_n_butterfly_factor != 0:
+                if conv_filter_dim < (boft_block_size * (2**boft_n_butterfly_factor)):
+                    raise ValueError(
+                        f"Invalid combination of convolutional kernel dimension ({conv_filter_dim}), boft_n_butterfly_factor ({boft_n_butterfly_factor + 1}) and boft_block_size ({boft_block_size})!"
+                    )
+                if conv_filter_dim % (boft_block_size * (2**boft_n_butterfly_factor)) != 0:
+                    raise ValueError(
+                        f"Invalid combination of convolutional kernel dimension ({conv_filter_dim}), boft_n_butterfly_factor ({boft_n_butterfly_factor + 1}) and boft_block_size ({boft_block_size})!"
+                    )
+
+            boft_block_num = int(conv_filter_dim // boft_block_size)
+
+        else:
+            raise ValueError(
+                "Something went wrong, please report this error: https://github.com/huggingface/peft/issues"
+            )
+
+        # In OFT you can specify the number of blocks to be 1
+        if boft_n_butterfly_factor != 0:
+            if boft_block_num % 2 != 0:
+                raise ValueError(f"boft_block_num ({boft_block_num}) must be an even number!")
+
+            if boft_block_size % 2 != 0:
+                raise ValueError(f"boft_block_size ({boft_block_size}) must be an even number!")
+
+        # If there is no butterfly factor, then permutation matrix P will be an identity matrix.
+        P = torch.empty((boft_n_butterfly_factor + 1, conv_filter_dim, conv_filter_dim))
+        for i in range(boft_n_butterfly_factor + 1):
+            perm = self.block_butterfly_perm(
+                conv_filter_dim, int(boft_block_num / (2 ** (i))), int(boft_block_size / 2), boft_n_butterfly_factor
+            )
+            perm_mat = self.perm2mat(perm)
+            P[i] = perm_mat
+
+        self.register_buffer("boft_P", P, persistent=False)
+
+        self.boft_R[adapter_name] = nn.Parameter(
+            torch.zeros(boft_n_butterfly_factor + 1, boft_block_num, boft_block_size, boft_block_size)
+        )
+        self.boft_s[adapter_name] = nn.Parameter(torch.ones(1, int(self.out_features)))
+
+        self.reset_boft_parameters(adapter_name, init_weights)
+
+        # set the boft block size and number
+        self.boft_block_size[adapter_name] = boft_block_size
+        self.boft_block_num[adapter_name] = boft_block_num
+
+        self._move_adapter_to_device_of_base_layer(adapter_name)
+        self.set_adapter(self.active_adapters)
+
+    def merge(self, safe_merge: bool = False, adapter_names: Optional[list[str]] = None) -> None:
+        """
+        Merge the active adapter weights into the base weights
+
+        Args:
+            safe_merge (`bool`, *optional*):
+                If True, the merge operation will be performed in a copy of the original weights and check for NaNs
+                before merging the weights. This is useful if you want to check if the merge operation will produce
+                NaNs. Defaults to `False`.
+            adapter_names (`List[str]`, *optional*):
+                The list of adapter names that should be merged. If None, all active adapters will be merged. Defaults
+                to `None`.
+        """
+        adapter_names = check_adapters_to_merge(self, adapter_names)
+        if not adapter_names:
+            # no adapter to merge
+            return
+
+        for active_adapter in adapter_names:
+            if active_adapter in self.boft_R.keys():
+                base_layer = self.get_base_layer()
+                if safe_merge:
+                    # Note that safe_merge will be slower than the normal merge
+                    # because of the copy operation.
+                    orig_weight = base_layer.weight.data.clone()
+                    butterfly_oft_mat, boft_s = self.get_delta_weight(active_adapter)
+
+                    orig_weight = orig_weight.view(
+                        self.out_features, self.in_features * base_layer.kernel_size[0] * base_layer.kernel_size[0]
+                    )
+                    orig_weight = torch.transpose(orig_weight, 0, 1)
+                    orig_weight = torch.mm(butterfly_oft_mat, orig_weight)
+                    orig_weight = torch.transpose(orig_weight, 0, 1)
+                    orig_weight = orig_weight * boft_s
+                    orig_weight = orig_weight.view(
+                        self.out_features, self.in_features, base_layer.kernel_size[0], base_layer.kernel_size[0]
+                    )
+
+                    self.base_layer.weight.data = orig_weight.contiguous()
+                else:
+                    butterfly_oft_mat, boft_s = self.get_delta_weight(active_adapter)
+
+                    orig_weight = base_layer.weight.data.clone()
+                    orig_weight = orig_weight.view(
+                        self.out_features, self.in_features * base_layer.kernel_size[0] * base_layer.kernel_size[0]
+                    )
+                    orig_weight = torch.transpose(orig_weight, 0, 1)
+                    orig_weight = torch.mm(butterfly_oft_mat, orig_weight)
+                    orig_weight = torch.transpose(orig_weight, 0, 1)
+                    orig_weight = orig_weight * boft_s
+                    orig_weight = orig_weight.view(
+                        self.out_features, self.in_features, base_layer.kernel_size[0], base_layer.kernel_size[0]
+                    )
+
+                    self.base_layer.weight.data = orig_weight.contiguous()
+
+                self.merged_adapters.append(active_adapter)
+
+    def unmerge(self) -> None:
+        """
+        This method unmerges all merged adapter layers from the base weights.
+        """
+        if not self.merged:
+            warnings.warn("Already unmerged. Nothing to do.")
+            return
+        while len(self.merged_adapters) > 0:
+            active_adapter = self.merged_adapters.pop()
+            if active_adapter in self.boft_R.keys():
+                butterfly_oft_mat, boft_s = self.get_delta_weight(active_adapter)
+
+                orig_weight = self.get_base_layer().weight.data.clone()
+                orig_weight = orig_weight.view(
+                    self.out_features,
+                    self.in_features * self.get_base_layer().kernel_size[0] * self.get_base_layer().kernel_size[0],
+                )
+                orig_weight = torch.transpose(orig_weight, 0, 1)
+                orig_weight = torch.mm(butterfly_oft_mat.t(), orig_weight)
+                orig_weight = torch.transpose(orig_weight, 0, 1)
+                orig_weight = orig_weight * (1 / boft_s)
+                orig_weight = orig_weight.view(
+                    self.out_features,
+                    self.in_features,
+                    self.get_base_layer().kernel_size[0],
+                    self.get_base_layer().kernel_size[0],
+                )
+
+                self.get_base_layer().weight.data = orig_weight
+
+    def get_delta_weight(self, adapter) -> tuple[torch.Tensor, torch.Tensor]:
+        """
+        Compute the delta weight for the given adapter.
+
+        Args:
+            adapter (str):
+                The name of the adapter for which the delta weight should be computed.
+        """
+
+        boft_R = self.boft_R[adapter]
+        boft_s = self.boft_s[adapter].transpose(0, 1)
+
+        N, D, H, _ = boft_R.shape
+        boft_R = boft_R.view(N * D, H, H)
+        orth_rotate_butterfly = self.cayley_batch(boft_R)
+        orth_rotate_butterfly = orth_rotate_butterfly.view(N, D, H, H)
+        if self.fbd_cuda_available:
+            block_diagonal_butterfly = FastBlockDiag.apply(orth_rotate_butterfly)
+        else:
+            orth_rotate_butterfly = orth_rotate_butterfly.squeeze(0)
+            block_diagonal_butterfly = torch.block_diag(*torch.unbind(orth_rotate_butterfly))
+            block_diagonal_butterfly = block_diagonal_butterfly.unsqueeze(0)
+
+        boft_P = self.boft_P.to(block_diagonal_butterfly.device)
+        butterfly_oft_mat_batch = torch.bmm(block_diagonal_butterfly, boft_P.permute(0, 2, 1))
+        butterfly_oft_mat_batch = torch.bmm(boft_P, butterfly_oft_mat_batch)
+        butterfly_oft_mat = butterfly_oft_mat_batch[0]
+
+        for i in range(1, butterfly_oft_mat_batch.shape[0]):
+            butterfly_oft_mat = butterfly_oft_mat_batch[i] @ butterfly_oft_mat
+
+        return butterfly_oft_mat, boft_s
+
+    def forward(self, x: torch.Tensor, *args: Any, **kwargs: Any) -> torch.Tensor:
+        previous_dtype = x.dtype
+
+        if self.disable_adapters:
+            if self.merged:
+                self.unmerge()
+            result = self.base_layer(x, *args, **kwargs)
+        elif self.merged:
+            result = self.base_layer(x, *args, **kwargs)
+        else:
+            boft_rotation = torch.eye(
+                self.in_features * self.base_layer.kernel_size[0] * self.base_layer.kernel_size[0],
+                device=x.device,
+                dtype=x.dtype,
+            )
+            boft_scale = torch.ones((int(self.out_features), 1), device=x.device, dtype=x.dtype)
+
+            for active_adapter in self.active_adapters:
+                if active_adapter not in self.boft_R.keys():
+                    continue
+                boft_R = self.boft_R[active_adapter]
+                boft_s = self.boft_s[active_adapter].transpose(0, 1)
+                dropout = self.boft_dropout[active_adapter]
+
+                N, D, H, _ = boft_R.shape
+                boft_R = boft_R.view(N * D, H, H)
+                orth_rotate_butterfly = self.cayley_batch(boft_R)
+                orth_rotate_butterfly = orth_rotate_butterfly.view(N, D, H, H)
+                orth_rotate_butterfly = dropout(orth_rotate_butterfly)
+                if self.fbd_cuda_available:
+                    block_diagonal_butterfly = FastBlockDiag.apply(orth_rotate_butterfly)
+                else:
+                    orth_rotate_butterfly = orth_rotate_butterfly.squeeze(0)
+                    block_diagonal_butterfly = torch.block_diag(*torch.unbind(orth_rotate_butterfly))
+                    block_diagonal_butterfly = block_diagonal_butterfly.unsqueeze(0)
+
+                boft_P = self.boft_P.to(x)
+                block_diagonal_butterfly = block_diagonal_butterfly.to(x)
+                butterfly_oft_mat_batch = torch.bmm(block_diagonal_butterfly, boft_P.permute(0, 2, 1))
+                butterfly_oft_mat_batch = torch.bmm(boft_P, butterfly_oft_mat_batch)
+                butterfly_oft_mat = butterfly_oft_mat_batch[0]
+
+                for i in range(1, butterfly_oft_mat_batch.shape[0]):
+                    butterfly_oft_mat = butterfly_oft_mat_batch[i] @ butterfly_oft_mat
+
+                boft_rotation = butterfly_oft_mat @ boft_rotation
+                boft_scale = boft_s * boft_scale
+
+            x = x.to(self.base_layer.weight.data.dtype)
+
+            orig_weight = self.base_layer.weight.data
+            orig_weight = orig_weight.view(
+                self.out_features,
+                self.in_features * self.base_layer.kernel_size[0] * self.base_layer.kernel_size[0],
+            )
+            orig_weight = torch.transpose(orig_weight, 0, 1)
+            rotated_weight = torch.mm(boft_rotation, orig_weight)
+            rotated_weight = torch.transpose(rotated_weight, 0, 1)
+
+            scaled_rotated_weight = rotated_weight * boft_scale
+
+            scaled_rotated_weight = scaled_rotated_weight.view(
+                self.out_features, self.in_features, self.base_layer.kernel_size[0], self.base_layer.kernel_size[0]
+            )
+            result = F.conv2d(
+                input=x,
+                weight=scaled_rotated_weight,
+                bias=self.base_layer.bias,
+                padding=self.base_layer.padding[0],
+                stride=self.base_layer.stride[0],
+            )
+
+        result = result.to(previous_dtype)
+        return result
+
+    def __repr__(self) -> str:
+        rep = super().__repr__()
+        return "boft." + rep

lib/python3.12/site-packages/peft/tuners/boft/model.py

@@ -0,0 +1,353 @@
+# Copyright 2023-present the HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# The implementation is based on "Parameter-Efficient Orthogonal Finetuning
+# via Butterfly Factorization" (https://arxiv.org/abs/2311.06243) in ICLR 2024.
+
+import warnings
+from dataclasses import asdict
+from enum import Enum
+from typing import List, Optional
+
+import torch
+from torch import nn
+from tqdm import tqdm
+
+from peft.tuners.tuners_utils import (
+    BaseTuner,
+    BaseTunerLayer,
+    check_target_module_exists,
+    onload_layer,
+)
+from peft.utils import (
+    TRANSFORMERS_MODELS_TO_LORA_TARGET_MODULES_MAPPING,
+    ModulesToSaveWrapper,
+    _get_submodules,
+)
+
+from .config import BOFTConfig
+from .layer import BOFTLayer, Conv2d, Linear
+
+
+class BOFTModel(BaseTuner):
+    """
+    Creates BOFT and OFT model from a pretrained transformers model. Paper: https://arxiv.org/abs/2311.06243
+    https://arxiv.org/abs/2306.07280
+
+    Args:
+        model ([`transformers.PreTrainedModel`]): The model to be adapted.
+        config ([`BOFTConfig`]): The configuration of the BOFT model.
+        adapter_name (`str`): The name of the adapter, defaults to `"default"`.
+        low_cpu_mem_usage (`bool`, `optional`, defaults to `False`):
+            Create empty adapter weights on meta device. Useful to speed up the loading process.
+
+    Returns:
+        `torch.nn.Module`: The BOFT model.
+
+    Example::
+
+        >>> import transformers >>> from transformers import AutoModelForSeq2SeqLM, BOFTConfig >>> from peft import
+        BOFTConfig, get_peft_model
+
+        >>> config = BOFTConfig( ... boft_block_size=8, ... boft_n_butterfly_factor=1, ... target_modules=["query",
+        "value", "key", "output.dense", "mlp.fc1", "mlp.fc2"], ... boft_dropout=0.1, ... bias="boft_only", ...
+        modules_to_save=["classifier"], ... )
+
+        >>> model = transformers.Dinov2ForImageClassification.from_pretrained( ... "facebook/dinov2-large", ...
+        num_labels=100, ... ) >>> boft_model = get_peft_model(model, config)
+
+    **Attributes**:
+        - **model** ([`transformers.PreTrainedModel`]) -- The model to be adapted.
+        - **peft_config** ([`BOFTConfig`]): The configuration of the BOFT model.
+    """
+
+    prefix: str = "boft_"
+
+    def __init__(self, model, config, adapter_name, low_cpu_mem_usage: bool = False) -> None:
+        super().__init__(model, config, adapter_name, low_cpu_mem_usage=low_cpu_mem_usage)
+
+    def _check_new_adapter_config(self, config: BOFTConfig) -> None:
+        """
+        A helper method to check the config when a new adapter is being added.
+
+        Raise a ValueError if there is something wrong with the config or if it conflicts with existing adapters.
+
+        """
+        # TODO: there should be a check if any of the existing adapters actually has bias != "none", or else the check
+        # does not fully correspond to the error message.
+        if (len(self.peft_config) > 1) and (config.bias != "none"):
+            raise ValueError(
+                f"{self.__class__.__name__} supports only 1 adapter with bias. When using multiple adapters, "
+                "set bias to 'none' for all adapters."
+            )
+
+    @staticmethod
+    def _check_target_module_exists(boft_config, key):
+        return check_target_module_exists(boft_config, key)
+
+    def _create_and_replace(
+        self,
+        boft_config,
+        adapter_name,
+        target,
+        target_name,
+        parent,
+        current_key,
+        **optional_kwargs,
+    ):
+        if current_key is None:
+            raise ValueError("Current Key shouldn't be `None`")
+
+        bias = hasattr(target, "bias") and target.bias is not None
+        kwargs = {
+            "boft_block_size": boft_config.boft_block_size,
+            "boft_block_num": boft_config.boft_block_num,
+            "boft_n_butterfly_factor": boft_config.boft_n_butterfly_factor,
+            "boft_dropout": boft_config.boft_dropout,
+            "fan_in_fan_out": boft_config.fan_in_fan_out,
+            "init_weights": boft_config.init_weights,
+        }
+        kwargs["bias"] = bias
+
+        # If it is not a BOFTLayer, create a new module, else update it with new adapters
+        if not isinstance(target, BOFTLayer):
+            new_module = self._create_new_module(boft_config, adapter_name, target, **kwargs)
+            if adapter_name not in self.active_adapters:
+                # adding an additional adapter: it is not automatically trainable
+                new_module.requires_grad_(False)
+            self._replace_module(parent, target_name, new_module, target)
+        else:
+            target.update_layer(
+                adapter_name,
+                boft_block_size=boft_config.boft_block_size,
+                boft_block_num=boft_config.boft_block_num,
+                boft_n_butterfly_factor=boft_config.boft_n_butterfly_factor,
+                boft_dropout=boft_config.boft_dropout,
+                init_weights=boft_config.init_weights,
+            )
+
+    def _replace_module(self, parent, child_name, new_module, child):
+        setattr(parent, child_name, new_module)
+        # It's not necessary to set requires_grad here, as that is handled by
+        # _mark_only_adapters_as_trainable
+
+        # child layer wraps the original module, unpack it
+        if hasattr(child, "base_layer"):
+            child = child.base_layer
+
+        if not hasattr(new_module, "base_layer"):
+            new_module.weight = child.weight
+            if hasattr(child, "bias"):
+                new_module.bias = child.bias
+
+        if getattr(child, "state", None) is not None:
+            if hasattr(new_module, "base_layer"):
+                new_module.base_layer.state = child.state
+            else:
+                new_module.state = child.state
+            new_module.to(child.weight.device)
+
+        meta = torch.device("meta")
+        # dispatch to correct device
+        for name, module in new_module.named_modules():
+            if self.prefix in name:
+                if not any(p.device == meta for p in module.parameters()):
+                    module.to(child.weight.device)
+
+    def _mark_only_adapters_as_trainable(self, model: nn.Module) -> None:
+        for n, p in model.named_parameters():
+            if self.prefix not in n:
+                p.requires_grad = False
+
+        for active_adapter in self.active_adapters:
+            bias = self.peft_config[active_adapter].bias
+            if bias == "none":
+                continue
+
+            if bias == "all":
+                for n, p in model.named_parameters():
+                    if "bias" in n:
+                        p.requires_grad = True
+            elif bias == "boft_only":
+                for name, m in model.named_modules():
+                    if isinstance(m, BOFTLayer) and hasattr(m, "bias") and m.bias is not None:
+                        m.bias.requires_grad = True
+            else:
+                raise NotImplementedError(f"Requested bias: {bias}, is not implemented.")
+
+    @staticmethod
+    def _create_new_module(boft_config, adapter_name, target, **kwargs):
+        if isinstance(target, BaseTunerLayer):
+            target_base_layer = target.get_base_layer()
+        else:
+            target_base_layer = target
+
+        if isinstance(target_base_layer, torch.nn.Linear):
+            if kwargs["fan_in_fan_out"]:
+                warnings.warn(
+                    "fan_in_fan_out is set to True but the target module is `torch.nn.Linear`. "
+                    "Setting fan_in_fan_out to False."
+                )
+                kwargs["fan_in_fan_out"] = boft_config.fan_in_fan_out = False
+            new_module = Linear(target, adapter_name, **kwargs)
+        elif isinstance(target_base_layer, torch.nn.Conv2d):
+            new_module = Conv2d(target, adapter_name, **kwargs)
+        else:
+            raise ValueError(
+                f"Target module {target} is not supported. "
+                "Currently, only `torch.nn.Linear` and `torch.nn.Conv2d` are supported."
+            )
+
+        return new_module
+
+    def __getattr__(self, name: str):
+        """Forward missing attributes to the wrapped module."""
+        try:
+            return super().__getattr__(name)  # defer to nn.Module's logic
+        except AttributeError:
+            if name == "model":  # see #1892: prevent infinite recursion if class is not initialized
+                raise
+            return getattr(self.model, name)
+
+    def get_peft_config_as_dict(self, inference: bool = False):
+        config_dict = {}
+        for key, value in self.peft_config.items():
+            config = {k: v.value if isinstance(v, Enum) else v for k, v in asdict(value).items()}
+            if inference:
+                config["inference_mode"] = True
+        config_dict[key] = config
+        return config
+
+    def _set_adapter_layers(self, enabled=True):
+        for module in self.model.modules():
+            if isinstance(module, (BaseTunerLayer, ModulesToSaveWrapper)):
+                module.enable_adapters(enabled)
+
+    def enable_adapter_layers(self):
+        self._set_adapter_layers(enabled=True)
+
+    def disable_adapter_layers(self):
+        for active_adapter in self.active_adapters:
+            val = self.peft_config[active_adapter].bias
+            if val != "none":
+                msg = (
+                    f"Careful, disabling adapter layers with bias configured to be '{val}' does not produce the same "
+                    "output as the the base model would without adaption."
+                )
+                warnings.warn(msg)
+        self._set_adapter_layers(enabled=False)
+
+    def set_adapter(self, adapter_name):
+        for module in self.model.modules():
+            if isinstance(module, BOFTLayer):
+                if module.merged:
+                    warnings.warn("Adapter cannot be set when the model is merged. Unmerging the model first.")
+                    module.unmerge()
+                module.set_adapter(adapter_name)
+        self.active_adapter = adapter_name
+
+    @staticmethod
+    def _prepare_adapter_config(peft_config, model_config):
+        if peft_config.target_modules is None:
+            if model_config["model_type"] not in TRANSFORMERS_MODELS_TO_LORA_TARGET_MODULES_MAPPING:
+                raise ValueError("Please specify `target_modules` in `peft_config`")
+            peft_config.target_modules = set(
+                TRANSFORMERS_MODELS_TO_LORA_TARGET_MODULES_MAPPING[model_config["model_type"]]
+            )
+        return peft_config
+
+    def _unload_and_optionally_merge(
+        self,
+        merge=True,
+        progressbar: bool = False,
+        safe_merge: bool = False,
+        adapter_names: Optional[List[str]] = None,
+    ):
+        if merge:
+            self._check_merge_allowed()
+
+        key_list = [key for key, _ in self.model.named_modules() if self.prefix not in key]
+        desc = "Unloading " + ("and merging " if merge else "") + "model"
+        for key in tqdm(key_list, disable=not progressbar, desc=desc):
+            try:
+                parent, target, target_name = _get_submodules(self.model, key)
+            except AttributeError:
+                continue
+            with onload_layer(target):
+                if hasattr(target, "base_layer"):
+                    if merge:
+                        target.merge(safe_merge=safe_merge, adapter_names=adapter_names)
+                    self._replace_module(parent, target_name, target.get_base_layer(), target)
+                elif isinstance(target, ModulesToSaveWrapper):
+                    # save any additional trainable modules part of `modules_to_save`
+                    new_module = target.modules_to_save[target.active_adapter]
+                    if hasattr(new_module, "base_layer"):
+                        # check if the module is itself a tuner layer
+                        if merge:
+                            new_module.merge(safe_merge=safe_merge, adapter_names=adapter_names)
+                        new_module = new_module.get_base_layer()
+                    setattr(parent, target_name, new_module)
+
+        return self.model
+
+    def delete_adapter(self, adapter_name: str) -> None:
+        """
+        Deletes an existing adapter.
+
+        Args:
+            adapter_name (str): Name of the adapter to be deleted.
+        """
+        if adapter_name not in list(self.peft_config.keys()):
+            raise ValueError(f"Adapter {adapter_name} does not exist")
+        del self.peft_config[adapter_name]
+
+        key_list = [key for key, _ in self.model.named_modules() if self.prefix not in key]
+        new_adapter = None
+        for key in key_list:
+            _, target, _ = _get_submodules(self.model, key)
+            if isinstance(target, BOFTLayer):
+                target.delete_adapter(adapter_name)
+                if new_adapter is None:
+                    new_adapter = target.active_adapters[:]
+
+        self.active_adapter = new_adapter or []
+
+    def merge_and_unload(
+        self, progressbar: bool = False, safe_merge: bool = False, adapter_names: Optional[List[str]] = None
+    ) -> torch.nn.Module:
+        r"""
+        This method merges the BOFT layers into the base model. This is needed if someone wants to use the base model
+        as a standalone model.
+
+        Args:
+            progressbar (`bool`):
+                whether to show a progressbar indicating the unload and merge process
+            safe_merge (`bool`):
+                whether to activate the safe merging check to check if there is any potential Nan in the adapter
+                weights
+            adapter_names (`List[str]`, *optional*):
+                The list of adapter names that should be merged. If None, all active adapters will be merged. Defaults
+                to `None`.
+
+        """
+        return self._unload_and_optionally_merge(
+            progressbar=progressbar, safe_merge=safe_merge, adapter_names=adapter_names
+        )
+
+    def unload(self) -> torch.nn.Module:
+        """
+        Gets back the base model by removing all the boft modules without merging. This gives back the original base
+        model.
+        """
+        return self._unload_and_optionally_merge(merge=False)

lib/python3.12/site-packages/peft/tuners/cpt/__init__.py

@@ -0,0 +1,24 @@
+# Copyright 2024-present the HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+from peft.utils import register_peft_method
+
+from .config import CPTConfig
+from .model import CPTEmbedding
+
+
+__all__ = ["CPTConfig", "CPTEmbedding"]
+
+register_peft_method(name="cpt", config_cls=CPTConfig, model_cls=CPTEmbedding)

lib/python3.12/site-packages/peft/tuners/cpt/config.py

@@ -0,0 +1,98 @@
+# Copyright 2024-present the HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from dataclasses import dataclass, field
+from typing import Literal, Optional
+
+from peft.config import PromptLearningConfig
+from peft.utils import PeftType
+
+
+@dataclass
+class CPTConfig(PromptLearningConfig):
+    """
+    CPT Configuration class extending PeftConfig for Context-aware Prompt Tuning (CPT).
+
+    This class introduces additional parameters required for CPT, such as:
+    - Token type masks
+    - Prompt tuning initialization
+    - Loss weighting
+    - Projection settings
+
+    For more details, see the paper: https://arxiv.org/abs/2410.17222
+    """
+
+    # Token-related configurations
+    cpt_token_ids: Optional[list[int]] = field(
+        default=None, metadata={"help": "Tensor of token IDs used for CPT prompts."}
+    )
+    cpt_mask: Optional[list[int]] = field(default=None, metadata={"help": "Tensor mask applied to CPT tokens."})
+    cpt_tokens_type_mask: Optional[list[int]] = field(
+        default=None, metadata={"help": "Mask indicating the type of each CPT token."}
+    )
+
+    # Loss-related configurations
+    opt_weighted_loss_type: Optional[Literal["none", "decay"]] = field(
+        default="none", metadata={"help": "Type of weighted loss: 'none' or 'decay'."}
+    )
+    opt_loss_decay_factor: Optional[float] = field(
+        default=1.0, metadata={"help": "Factor for exponential decay in loss weighting."}
+    )
+
+    # Projection-related configurations
+    opt_projection_epsilon: Optional[float] = field(
+        default=0.1, metadata={"help": "Epsilon value for input projection."}
+    )
+    opt_projection_format_epsilon: Optional[float] = field(
+        default=0.1, metadata={"help": "Epsilon value for format projection."}
+    )
+
+    # Tokenizer configuration
+    tokenizer_name_or_path: Optional[str] = field(
+        default=None,
+        metadata={
+            "help": "The tokenizer to use for prompt tuning initialization. Only used if prompt_tuning_init is `TEXT`"
+        },
+    )
+    # Neet to define CPT-specific static attributes
+    is_prompt_learning = True  # Indicates that CPT is a prompt-learning method.
+
+    def __post_init__(self):
+        """
+        Post-initialization hook to set additional attributes after the config is initialized.
+        """
+        # CPT-specific static attributes
+        self.is_prompt_learning = True  # Indicates that CPT is a prompt-learning method.
+        self.num_layers = None  # Number of layers (optional, not always required).
+        self.token_dim = None  # Dimension of token embeddings.
+        self.num_attention_heads = None  # Number of attention heads (if applicable).
+        self.num_transformer_submodules = 1  # Number of transformer submodules used.
+        self.peft_type = PeftType.CPT  # Specifies that the PEFT type is CPT.
+        self.task_type = "CAUSAL_LM"  # Ensures task type is causal language modeling.
+
+        if self.cpt_token_ids is None:
+            self.cpt_token_ids = [0]
+
+        self.num_virtual_tokens = len(self.cpt_token_ids)
+
+        if self.cpt_mask is None:
+            self.cpt_mask = [1 for _ in self.cpt_token_ids]
+
+        if self.cpt_tokens_type_mask is None:
+            self.cpt_tokens_type_mask = [1 for _ in self.cpt_token_ids]
+
+        if not (
+            len(self.cpt_token_ids) == len(self.cpt_mask) == len(self.cpt_tokens_type_mask) == self.num_virtual_tokens
+        ):
+            raise ValueError("cpt_token_ids, cpt_mask and cpt_tokens_type_mask must have the same length.")

lib/python3.12/site-packages/peft/tuners/cpt/model.py

@@ -0,0 +1,200 @@
+# Copyright 2024-present the HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import copy
+
+import torch
+from torch.nn import CrossEntropyLoss
+
+from peft.utils.integrations import gather_params_ctx
+
+
+class CPTEmbedding(torch.nn.Module):
+    """
+    CPTEmbedding is a custom embedding layer designed for Context-aware Prompt Tuning (CPT) in PEFT. It initializes
+    embeddings, applies prompt-specific projections, and computes loss using label masks.
+    """
+
+    def __init__(self, config, word_embeddings):
+        """
+        Initializes the CPTEmbedding module.
+
+        Args:
+            config (Namespace):
+                Configuration object containing model hyperparameters and CPT-specific settings.
+            word_embeddings (torch.nn.Embedding):
+                The base word embedding layer used to initialize CPT embeddings.
+        """
+        super().__init__()
+        self.config = copy.deepcopy(config)
+        num_virtual_tokens = config.num_virtual_tokens
+
+        # Initialize embeddings with virtual token dimensions
+        self.embedding = torch.nn.Embedding(num_virtual_tokens, config.token_dim)
+
+        # Initialize embeddings using text-based prompt tuning, if configured
+        if not config.inference_mode:
+            assert config.num_virtual_tokens == len(config.cpt_token_ids)
+
+            init_token_ids = torch.LongTensor(config.cpt_token_ids).to(word_embeddings.weight.device)
+            with gather_params_ctx(word_embeddings.parameters()):
+                word_embedding_weights = word_embeddings(init_token_ids).detach().clone()
+            word_embedding_weights = word_embedding_weights.to(torch.float32)
+            self.embedding.weight = torch.nn.Parameter(word_embedding_weights)
+
+        # Initialize delta embedding with zero weights
+        self.delta_embedding = torch.nn.Embedding(num_virtual_tokens, config.token_dim)
+        self.delta_embedding.weight.data = torch.zeros_like(self.delta_embedding.weight).to(torch.float32)
+
+        # Apply hook for backward gradient updates
+        self.set_updated_tokens()
+
+    def forward(self, indices):
+        """
+        Computes the prompt embeddings and applies delta adjustments.
+
+        Args:
+            indices (torch.Tensor):
+                Indices of the tokens to be embedded.
+
+        Returns:
+            torch.Tensor:
+                Sum of prompt embeddings and delta embeddings.
+        """
+        with torch.no_grad():
+            prompt_embeddings = self.embedding(indices)
+
+        self.delta_embedding.weight.data = self.get_projection()  # Apply epsilon-based projection
+
+        delta_prompt_embeddings = self.delta_embedding(indices)
+
+        return prompt_embeddings + delta_prompt_embeddings
+
+    def set_updated_tokens(self):
+        """
+        Sets up a backward hook to selectively update token gradients based on the CPT token type mask.
+        """
+        tensor_ICL_mask = torch.Tensor(self.config.cpt_tokens_type_mask).long()
+        mask_input_template = torch.remainder(tensor_ICL_mask, 4) == 1
+        mask_input = torch.remainder(tensor_ICL_mask, 4) == 2
+        mask_output_template = torch.remainder(tensor_ICL_mask, 4) == 3
+        mask = mask_input_template | mask_input | mask_output_template
+        mask = mask.view(-1, 1)
+
+        def backward_hook(grad):
+            grad = grad * mask.to(grad.device)  # Apply mask to gradients
+            return grad
+
+        self.delta_embedding.weight.register_hook(backward_hook)
+
+    def get_epsilon(self):
+        cpt_tokens_type_mask = self.config.cpt_tokens_type_mask
+
+        MIN_VALUE = 1e-10
+
+        # Calculate normalized epsilon values for input, output, and format tokens
+        normalized_format_eps = self.config.opt_projection_format_epsilon * torch.sqrt(
+            torch.Tensor([self.config.token_dim / 2048])
+        )
+        normalized_input_eps = self.config.opt_projection_epsilon * torch.sqrt(
+            torch.Tensor([self.config.token_dim / 2048])
+        )
+
+        epsilon = torch.ones_like(torch.Tensor(cpt_tokens_type_mask)).to(torch.float32) * MIN_VALUE
+        cpt_tokens_type_mask = torch.Tensor(cpt_tokens_type_mask).long()
+
+        epsilon[(cpt_tokens_type_mask > 0) & (torch.remainder(cpt_tokens_type_mask, 4) == 1)] = normalized_format_eps
+        epsilon[(cpt_tokens_type_mask > 0) & (torch.remainder(cpt_tokens_type_mask, 4) == 3)] = normalized_format_eps
+        epsilon[(cpt_tokens_type_mask > 0) & (torch.remainder(cpt_tokens_type_mask, 4) == 2)] = normalized_input_eps
+
+        return epsilon
+
+    def get_projection(self):
+        """
+        Applies epsilon-based projection to the delta embeddings to control their norm.
+        """
+
+        # Apply projection to control delta embedding norm
+        with torch.no_grad():
+            new_embeddings_weights = self.delta_embedding.weight.clone().to(self.delta_embedding.weight.device)
+            token_norm = torch.norm(new_embeddings_weights, p=2, dim=1)
+
+            projection_mask = token_norm > 0
+            if torch.any(projection_mask):
+                epsilon = self.get_epsilon().to(self.delta_embedding.weight.device)
+                new_embeddings_weights[projection_mask] *= (
+                    epsilon[projection_mask] / (token_norm[projection_mask].clamp(min=epsilon[projection_mask]))
+                ).view(-1, 1)
+            return new_embeddings_weights
+
+    @staticmethod
+    def calculate_loss(base_model_output, labels, cpt_type_mask, config):
+        """
+        Computes the loss for CPT models with optional exponential decay.
+
+        Args:
+            base_model_output (ModelOutput):
+                Output from the base model containing logits.
+            labels (torch.Tensor):
+                Ground-truth labels for the input tokens.
+            cpt_type_mask (torch.Tensor):
+                Token type mask used for filtering valid loss terms.
+            config (Namespace):
+                Configuration object containing loss-related hyperparameters.
+
+        Returns:
+            ModelOutput:
+                The base model output with computed loss.
+        """
+
+        device = base_model_output.logits.device
+
+        lm_logits = base_model_output.logits
+        labels = labels.to(device)
+
+        # Shift logits and labels for token prediction
+        shift_logits = lm_logits[..., :-1, :].contiguous()
+        shift_labels = labels[..., 1:].contiguous()
+        shift_cpt_type_mask = cpt_type_mask[..., 1:].contiguous()
+
+        shift_labels_bool = (shift_labels.clone().detach() != -100).bool()
+        batch_size, seq_length, vocab_size = shift_logits.shape
+
+        # Compute cross-entropy loss
+        loss_fct = CrossEntropyLoss(reduction="none", ignore_index=-100)
+        loss = loss_fct(
+            shift_logits.view(batch_size * seq_length, vocab_size), shift_labels.view(batch_size * seq_length)
+        )
+        loss = loss.view(batch_size, seq_length)
+        # Apply exponential decay weights to the loss
+        shift_labels_weights = shift_labels_bool.clone().detach().float()
+
+        for i in range(batch_size):
+            idx_labels = (shift_cpt_type_mask[i] > 0) & (shift_cpt_type_mask[i] % 4 == 0)
+            labels_ids = shift_cpt_type_mask[i][idx_labels].unique()
+
+            exponential_decay = torch.ones_like(shift_cpt_type_mask[i]).to(device=device).float()
+            decay_value = 1
+            for label_mask_idx in torch.flip(labels_ids, [0]):
+                exponential_decay[shift_cpt_type_mask[i] == label_mask_idx] = decay_value
+                decay_value *= config.opt_loss_decay_factor
+            if config.opt_weighted_loss_type == "decay":
+                shift_labels_weights[i] *= exponential_decay
+
+        # Compute the weighted mean loss
+        loss = (loss[shift_labels_bool] * shift_labels_weights[shift_labels_bool]).mean()
+
+        base_model_output.loss = loss
+
+        return base_model_output

lib/python3.12/site-packages/peft/tuners/fourierft/__init__.py

@@ -0,0 +1,24 @@
+# Copyright 2024-present the HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from peft.utils import register_peft_method
+
+from .config import FourierFTConfig
+from .layer import FourierFTLayer, FourierFTLinear
+from .model import FourierFTModel
+
+
+__all__ = ["FourierFTConfig", "FourierFTLayer", "FourierFTLinear", "FourierFTModel"]
+
+register_peft_method(name="fourierft", model_cls=FourierFTModel, config_cls=FourierFTConfig)

lib/python3.12/site-packages/peft/tuners/fourierft/config.py

@@ -0,0 +1,205 @@
+# Copyright 2024-present the HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Optional, Union
+
+from peft.config import PeftConfig
+from peft.utils import PeftType
+
+
+@dataclass
+class FourierFTConfig(PeftConfig):
+    """
+    This is the configuration class to store the configuration of a [`FourierFTModel`].
+
+    Args:
+        n_frequency (`int`):
+            Num of learnable frequencies for the Discrete Fourier Transform. 'n_frequency' is an integer that is
+            greater than 0 and less than or equal to d^2 (assuming the weight W has dimensions of d by d).
+            Additionally, it is the number of trainable parameters required to update each delta W weight.
+            'n_frequency' will affect the performance and efficiency for PEFT. Specifically, it has little impact on
+            training speed, but higher values of it (typically) result in larger GPU memory costs and better accuracy.
+            With the same `target_modules`, the number of parameters of LoRA is (2*d*r/n_frequency) times that of
+            FourierFT. The following examples of settings regarding 'n_frequency' can be used as reference for users.
+            For NLU tasks with the RoBERTa-large model, adopting 'n_frequency': 1000 can almost achieve similar results
+            as 'r': 8 in LoRA. At this time, the number of parameters of LoRA is about 16 times that of FourierFT. For
+            image classification tasks with Vit-large models, adopting 'n_frequency': 3000 can almost achieve similar
+            results as 'r': 16 in LoRA, where the number of parameters of LoRA is about 11 times that of FourierFT.
+        scaling (`float`):
+            The scaling value for the delta W matrix. This is an important hyperparameter used for scaling, similar to
+            the 'lora_alpha' parameter in the LoRA method. 'scaling' can be determined during the hyperparameter search
+            process. However, if users want to skip this process, one can refer to the settings in the following
+            scenarios. This parameter can be set to 100.0 or 150.0 for both RoBERTa-base and RoBERTa-large models
+            across all NLU (GLUE) tasks. This parameter can be set to 300.0 for both LLaMA family models for all
+            instruction tuning. This parameter can be set to 300.0 for both ViT-base and ViT-large models across all
+            image classification tasks.
+        random_loc_seed (`int`):
+            Seed for the random location of the frequencies, i.e., the spectral entry matrix.
+        target_modules (`Union[list[str],str]`):
+            List of module names or regex expression of the module names to replace with FourierFT. For example, ['q',
+            'v'] or '.*decoder.*(SelfAttention|EncDecAttention).*(q|v)$'. Only linear layers are supported.
+        exclude_modules (`Optional[Union[List[str], str]]`):
+            The names of the modules to not apply the adapter. When passing a string, a regex match will be performed.
+            When passing a list of strings, either an exact match will be performed or it is checked if the name of the
+            module ends with any of the passed strings.
+        fan_in_fan_out (`bool`):
+            Set this to True if the layer to replace stores weight like (fan_in, fan_out).
+        bias (`str`):
+            Bias type for FourierFT. Can be 'none', 'all' or 'fourier_only'.
+        modules_to_save (`list[str]`):
+            List of modules apart from FourierFT layers to be set as trainable and saved in the final checkpoint. For
+            example, in Sequence Classification or Token Classification tasks, the final layer `classifier/score` are
+            randomly initialized and as such need to be trainable and saved.
+        layers_to_transform (`Union[list[int],int]`):
+            The layer indexes to transform, is this argument is specified, PEFT will transform only the layers indexes
+            that are specified inside this list. If a single integer is passed, PEFT will transform only the layer at
+            this index.
+        layers_pattern (`Optional[Union[List[str], str]]`):
+            The layer pattern name, used only if `layers_to_transform` is different to None and if the layer pattern is
+            not in the common layers pattern. This should target the `nn.ModuleList` of the model, which is often
+            called `'layers'` or `'h'`.
+        n_frequency_pattern (`dict`):
+            The mapping from layer names or regexp expression to n_frequency which are different from the default
+            specified. For example, `{model.decoder.layers.0.encoder_attn.k_proj: 1000`}.
+        init_weights (`bool`):
+            The initialization of the Fourier weights. Set this to False if the spectrum are initialized to a standard
+            normal distribution. Set this to True if the spectrum are initialized to zeros.
+    """
+
+    n_frequency: int = field(
+        default=1000,
+        metadata={
+            "help": (
+                "Num of learnable frequencies for the Discrete Fourier Transform. 'n_frequency' is an integer that is"
+                "greater than 0 and less than or equal to d^2 (assuming the weight W has dimensions of d by d)."
+                "Additionally, it is the number of trainable parameters required to update each delta W weight."
+                "'n_frequency' will affect the performance and efficiency for PEFT. Specifically, it has little impact on"
+                "training speed, but higher values of it (typically) result in larger GPU memory costs and better accuracy."
+                "With the same `target_modules`, the number of parameters of LoRA is (2*d*r/n_frequency) times that of FourierFT."
+                "The following examples of settings regarding 'n_frequency' can be used as reference for users. For NLU"
+                "tasks with the RoBERTa-large model, adopting 'n_frequency': 1000 can almost achieve similar results as"
+                "'r': 8 in LoRA. At this time, the number of parameters of LoRA is about 16 times that of FourierFT."
+                "For image classification tasks with Vit-large models, adopting 'n_frequency': 3000 can almost achieve"
+                "similar results as 'r': 16 in LoRA, where the number of parameters of LoRA is about 11 times that of FourierFT."
+            )
+        },
+    )
+    scaling: float = field(
+        default=150.0,
+        metadata={
+            "help": (
+                "The scaling value for the delta W matrix. This is an important hyperparameter used for scaling, similar to the"
+                "'lora_alpha' parameter in the LoRA method. 'scaling' can be determined during the hyperparameter search process."
+                "However, if users want to skip this process, one can refer to the settings in the following scenarios."
+                "This parameter can be set to 100.0 or 150.0 for both RoBERTa-base and RoBERTa-large models across all NLU (GLUE) tasks."
+                "This parameter can be set to 300.0 for both LLaMA family models for all instruction tuning."
+                "This parameter can be set to 300.0 for both ViT-base and ViT-large models across all image classification tasks."
+            )
+        },
+    )
+    random_loc_seed: Optional[int] = field(
+        default=777, metadata={"help": "Seed for the random location of the frequencies."}
+    )
+    fan_in_fan_out: bool = field(
+        default=False,
+        metadata={"help": "Set this to True if the layer to replace stores weight like (fan_in, fan_out)"},
+    )
+    target_modules: Optional[Union[list[str], str]] = field(
+        default=None,
+        metadata={
+            "help": (
+                "List of module names or regex expression of the module names to replace with FourierFT."
+                "For example, ['q', 'v'] or '.*decoder.*(SelfAttention|EncDecAttention).*(q|v)$'. "
+                "Only linear layers are supported."
+            )
+        },
+    )
+    exclude_modules: Optional[Union[list[str], str]] = field(
+        default=None,
+        metadata={"help": "List of module names or regex expression of the module names to exclude from fourierft."},
+    )
+    bias: str = field(
+        default="none", metadata={"help": "Bias type for FourierFT. Can be 'none', 'all' or 'fourier_only'."}
+    )
+    modules_to_save: Optional[list[str]] = field(
+        default=None,
+        metadata={
+            "help": (
+                "List of modules apart from FourierFT layers to be set as trainable and saved in the final checkpoint. For"
+                " example, in Sequence Classification or Token Classification tasks, the final layer"
+                " `classifier/score` are randomly initialized and as such need to be trainable and saved."
+            )
+        },
+    )
+    layers_to_transform: Optional[Union[list[int], int]] = field(
+        default=None,
+        metadata={
+            "help": (
+                "The layer indexes to transform, is this argument is specified, PEFT will transform only the layers"
+                " indexes that are specified inside this list. If a single integer is passed, PEFT will transform only"
+                " the layer at this index."
+            )
+        },
+    )
+    layers_pattern: Optional[Union[list[str], str]] = field(
+        default=None,
+        metadata={
+            "help": (
+                "The layer pattern name, used only if `layers_to_transform` is different to None and if the layer"
+                " pattern is not in the common layers pattern. This should target the `nn.ModuleList` of the "
+                "model, which is often called `'layers'` or `'h'`."
+            )
+        },
+    )
+    n_frequency_pattern: Optional[dict] = field(
+        default_factory=dict,
+        metadata={
+            "help": (
+                "The mapping from layer names or regexp expression to n_frequency which are different from the default specified."
+                "For example, `{model.decoder.layers.0.encoder_attn.k_proj: 500`}."
+            )
+        },
+    )
+    init_weights: bool = field(
+        default=False,
+        metadata={
+            "help": (
+                "The initialization of the Fourier weights. Set this to False if the spectrum should be initialized to a standard normal distribution."
+                "Set this to True if the spectrum should be initialized to zeros."
+            )
+        },
+    )
+
+    def __post_init__(self):
+        super().__post_init__()
+        self.peft_type = PeftType.FOURIERFT
+        self.target_modules = (
+            set(self.target_modules) if isinstance(self.target_modules, list) else self.target_modules
+        )
+        self.exclude_modules = (
+            set(self.exclude_modules) if isinstance(self.exclude_modules, list) else self.exclude_modules
+        )
+        # if target_modules is a regex expression, then layers_to_transform should be None
+        if isinstance(self.target_modules, str) and self.layers_to_transform is not None:
+            raise ValueError("`layers_to_transform` cannot be used when `target_modules` is a str.")
+
+        # if target_modules is a regex expression, then layers_pattern should be None
+        if isinstance(self.target_modules, str) and self.layers_pattern is not None:
+            raise ValueError("`layers_pattern` cannot be used when `target_modules` is a str.")
+        # check for layers_to_transform and layers_pattern
+        if self.layers_pattern and not self.layers_to_transform:
+            raise ValueError("When `layers_pattern` is specified, `layers_to_transform` must also be specified. ")