smpeft/mapping.py

# coding=utf-8
# Original License:
# Copyright 2023-present the HuggingFace Inc. team.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from .peft_model import (
    PeftModel,
    PeftModelForCausalLM,
    PeftModelForSeq2SeqLM,
    PeftModelForSequenceClassification,
    PeftModelForTokenClassification,
)
from .sama import SamaConfig, SamaTuner
from .utils import PromptLearningConfig

from transformers import PreTrainedModel

MODEL_TYPE_TO_PEFT_MODEL_MAPPING = {
    "SEQ_CLS": PeftModelForSequenceClassification,
    "SEQ_2_SEQ_LM": PeftModelForSeq2SeqLM,
    "CAUSAL_LM": PeftModelForCausalLM,
    "TOKEN_CLS": PeftModelForTokenClassification,
}

PEFT_TYPE_TO_CONFIG_MAPPING: dict = {
    "SAMA": SamaConfig
}

PEFT_TYPE_TO_TUNER_MAPPING: dict = {
    "SAMA": SamaTuner
}

TRANSFORMERS_MODELS_TO_LORA_TARGET_MODULES_MAPPING = {
    "t5": ["q", "v"],
    "mt5": ["q", "v"],
    "bart": ["q_proj", "v_proj"],
    "gpt2": ["c_attn"],
    "bloom": ["query_key_value"],
    "blip-2": ["q", "v", "q_proj", "v_proj"],
    "opt": ["q_proj", "v_proj"],
    "gptj": ["q_proj", "v_proj"],
    "gpt_neox": ["query_key_value"],
    "gpt_neo": ["q_proj", "v_proj"],
    "bert": ["query", "value"],
    "roberta": ["query", "value"],
    "xlm-roberta": ["query", "value"],
    "electra": ["query", "value"],
    "deberta-v2": ["query_proj", "value_proj"],
    "deberta": ["in_proj"],
    "layoutlm": ["query", "value"],
    "llama": ["q_proj", "v_proj"],
    "chatglm": ["query_key_value"],
    "gpt_bigcode": ["c_attn"],
    "mpt": ["Wqkv"],
    "RefinedWebModel": ["query_key_value"],
    "RefinedWeb": ["query_key_value"],
    "falcon": ["query_key_value"],
    "btlm": ["c_proj", "c_attn"],
    "codegen": ["qkv_proj"],
    "mistral": ["q_proj", "v_proj"],
    "mixtral": ["q_proj", "v_proj"],
    "stablelm": ["q_proj", "v_proj"],
    "phi": ["q_proj", "v_proj", "fc1", "fc2"],
    "gemma": ["q_proj", "v_proj"],
}



def get_peft_config(config_dict):
    """
    Returns a Peft config object from a dictionary.

    Args:
        config_dict (`Dict[str, Any]`): Dictionary containing the configuration parameters.
    """

    return PEFT_TYPE_TO_CONFIG_MAPPING[config_dict["peft_type"]](**config_dict)


def _prepare_prompt_learning_config(peft_config, model_config):
    if peft_config.num_layers is None:
        if "num_hidden_layers" in model_config:
            num_layers = model_config["num_hidden_layers"]
        elif "num_layers" in model_config:
            num_layers = model_config["num_layers"]
        elif "n_layer" in model_config:
            num_layers = model_config["n_layer"]
        else:
            raise ValueError("Please specify `num_layers` in `peft_config`")
        peft_config.num_layers = num_layers

    if peft_config.token_dim is None:
        if "hidden_size" in model_config:
            token_dim = model_config["hidden_size"]
        elif "n_embd" in model_config:
            token_dim = model_config["n_embd"]
        elif "d_model" in model_config:
            token_dim = model_config["d_model"]
        else:
            raise ValueError("Please specify `token_dim` in `peft_config`")
        peft_config.token_dim = token_dim

    if peft_config.num_attention_heads is None:
        if "num_attention_heads" in model_config:
            num_attention_heads = model_config["num_attention_heads"]
        elif "n_head" in model_config:
            num_attention_heads = model_config["n_head"]
        elif "num_heads" in model_config:
            num_attention_heads = model_config["num_heads"]
        elif "encoder_attention_heads" in model_config:
            num_attention_heads = model_config["encoder_attention_heads"]
        else:
            raise ValueError("Please specify `num_attention_heads` in `peft_config`")
        peft_config.num_attention_heads = num_attention_heads

    if getattr(peft_config, "encoder_hidden_size", None) is None:
        setattr(peft_config, "encoder_hidden_size", token_dim)

    return peft_config


def _prepare_lora_config(peft_config, model_config):
    if peft_config.target_modules is None:
        if model_config["model_type"] not in TRANSFORMERS_MODELS_TO_LORA_TARGET_MODULES_MAPPING:
            raise ValueError("Please specify `target_modules` in `peft_config`")
        peft_config.target_modules = TRANSFORMERS_MODELS_TO_LORA_TARGET_MODULES_MAPPING[model_config["model_type"]]
    if len(peft_config.target_modules) == 1:
        peft_config.fan_in_fan_out = True
        peft_config.enable_lora = [True, False, True]
    if peft_config.inference_mode:
        peft_config.merge_weights = True
    return peft_config
    


def get_peft_model(model, peft_config,
                    adapter_name: str = "default"):
    """
    Returns a Peft model object from a model and a config.

    Args:
        model ([`transformers.PreTrainedModel`]): Model to be wrapped.
        peft_config ([`PeftConfig`]): Configuration object containing the parameters of the Peft model.
    """
    model_config = model.config.to_dict()
    peft_config.base_model_name_or_path = model.__dict__.get("name_or_path", None)
    if peft_config.task_type not in MODEL_TYPE_TO_PEFT_MODEL_MAPPING.keys():
        if peft_config.peft_type == "LORA" or "QUANTA":
            peft_config = _prepare_lora_config(peft_config, model_config)
            return PeftModel(model, peft_config)
    if not isinstance(peft_config, PromptLearningConfig):
        if peft_config.peft_type == "LORA" or "QUANTA":
            peft_config = _prepare_lora_config(peft_config, model_config)
    else:
        peft_config = _prepare_prompt_learning_config(peft_config, model_config)
    # assert False
    return MODEL_TYPE_TO_PEFT_MODEL_MAPPING[peft_config.task_type](
        model,
        peft_config,
        adapter_name=adapter_name,
    )


# def get_peft_model(
#     model: PreTrainedModel,
#     peft_config,
#     adapter_name: str = "default",
#     mixed: bool = False,
#     autocast_adapter_dtype: bool = True,
#     revision: Optional[str] = None,
#     low_cpu_mem_usage: bool = False,
# ) -> PeftModel | PeftMixedModel:
#     """
#     Returns a Peft model object from a model and a config, where the model will be modified in-place.

#     Args:
#         model ([`transformers.PreTrainedModel`]):
#             Model to be wrapped.
#         peft_config ([`PeftConfig`]):
#             Configuration object containing the parameters of the Peft model.
#         adapter_name (`str`, `optional`, defaults to `"default"`):
#             The name of the adapter to be injected, if not provided, the default adapter name is used ("default").
#         mixed (`bool`, `optional`, defaults to `False`):
#             Whether to allow mixing different (compatible) adapter types.
#         autocast_adapter_dtype (`bool`, *optional*):
#             Whether to autocast the adapter dtype. Defaults to `True`. Right now, this will only cast adapter weights
#             using float16 or bfloat16 to float32, as this is typically required for stable training, and only affect
#             select PEFT tuners.
#         revision (`str`, `optional`, defaults to `main`):
#             The revision of the base model. If this isn't set, the saved peft model will load the `main` revision for
#             the base model
#         low_cpu_mem_usage (`bool`, `optional`, defaults to `False`):
#             Create empty adapter weights on meta device. Useful to speed up the loading process. Leave this setting as
#             False if you intend on training the model, unless the adapter weights will be replaced by different weights
#             before training starts.
#     """
#     model_config = BaseTuner.get_model_config(model)
#     old_name = peft_config.base_model_name_or_path
#     new_name = model.__dict__.get("name_or_path", None)
#     peft_config.base_model_name_or_path = new_name

#     # Especially in notebook environments there could be a case that a user wants to experiment with different
#     # configuration values. However, it is likely that there won't be any changes for new configs on an already
#     # initialized PEFT model. The best we can do is warn the user about it.
#     if any(isinstance(module, BaseTunerLayer) for module in model.modules()):
#         warnings.warn(
#             "You are trying to modify a model with PEFT for a second time. If you want to reload the model with a "
#             "different config, make sure to call `.unload()` before."
#         )

#     if (old_name is not None) and (old_name != new_name):
#         warnings.warn(
#             f"The PEFT config's `base_model_name_or_path` was renamed from '{old_name}' to '{new_name}'. "
#             "Please ensure that the correct base model is loaded when loading this checkpoint."
#         )

#     if revision is not None:
#         if peft_config.revision is not None and peft_config.revision != revision:
#             warnings.warn(
#                 f"peft config has already set base model revision to {peft_config.revision}, overwriting with revision {revision}"
#             )
#         peft_config.revision = revision

#     if (
#         (isinstance(peft_config, PEFT_TYPE_TO_CONFIG_MAPPING["LORA"]))
#         and (peft_config.init_lora_weights == "eva")
#         and not low_cpu_mem_usage
#     ):
#         warnings.warn(
#             "lora with eva initialization used with low_cpu_mem_usage=False. "
#             "Setting low_cpu_mem_usage=True can improve the maximum batch size possible for eva initialization."
#         )

#     prefix = PEFT_TYPE_TO_PREFIX_MAPPING.get(peft_config.peft_type)
#     if prefix and adapter_name in prefix:
#         warnings.warn(
#             f"Adapter name '{adapter_name}' should not be contained in the prefix '{prefix}'. "
#             "This may lead to reinitialization of the adapter weights during loading."
#         )

#     if mixed:
#         # note: PeftMixedModel does not support autocast_adapter_dtype, so don't pass it
#         return PeftMixedModel(model, peft_config, adapter_name=adapter_name)

#     # We explicitly exclude prompt learning here since prompt learning is specific to the task and needs special
#     # handling in the PEFT model's forward method.
#     if peft_config.task_type not in MODEL_TYPE_TO_PEFT_MODEL_MAPPING.keys() and not peft_config.is_prompt_learning:
#         return PeftModel(
#             model,
#             peft_config,
#             adapter_name=adapter_name,
#             autocast_adapter_dtype=autocast_adapter_dtype,
#             low_cpu_mem_usage=low_cpu_mem_usage,
#         )

#     return MODEL_TYPE_TO_PEFT_MODEL_MAPPING[peft_config.task_type](
#         model,
#         peft_config,
#         adapter_name=adapter_name,
#         autocast_adapter_dtype=autocast_adapter_dtype,
#         low_cpu_mem_usage=low_cpu_mem_usage,
#     )