Buckets:

MisterAI
/

LocalAI_Demo_backends

Files

xet

MisterAI/LocalAI_Demo_backends / cpu-diffusers /venv /lib /python3.10 /site-packages /peft /mapping_func.py

MisterAI

28 days ago

download

raw

8.96 kB

# Copyright 2024-present the HuggingFace Inc. team.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import annotations

import warnings
from typing import TYPE_CHECKING, Literal, Optional, overload

from torch import nn

from .auto import MODEL_TYPE_TO_PEFT_MODEL_MAPPING
from .config import PeftConfig, PromptLearningConfig
from .mapping import PEFT_TYPE_TO_CONFIG_MAPPING, PEFT_TYPE_TO_PREFIX_MAPPING
from .mixed_model import PeftMixedModel
from .peft_model import PeftModel
from .tuners.tuners_utils import BaseTuner, BaseTunerLayer


if TYPE_CHECKING:
    from .tuners import LoraConfig, LoraModel

    # A PeftModel forwards missing attributes to the wrapped tuner model via __getattr__, so its interface is
    # effectively the intersection of both classes. Python's type system cannot express intersections, so we emulate
    # them for static type checkers with classes that inherit from both, with PeftModel first in the MRO to mirror the
    # runtime attribute lookup order. These classes only exist for type checkers; at runtime, get_peft_model returns a
    # plain PeftModel (subclass).
    class _LoraPeftModel(PeftModel, LoraModel):  # type: ignore[misc]
        ...

    class _TunerPeftModel(PeftModel, BaseTuner):  # type: ignore[misc]
        ...

    # The overloads only affect static type checking, the returned types are chosen based on what attributes are
    # reachable through __getattr__ forwarding: LoRA has extra methods like add_weighted_adapter, other tuners provide
    # the BaseTuner API (e.g. merge_and_unload), and prompt learning wraps the base model directly, i.e. there is no
    # tuner model whose attributes could be forwarded. Overloads need to be defined before the actual function.
    @overload
    def get_peft_model(
        model: nn.Module,
        peft_config: PeftConfig,
        adapter_name: str = ...,
        *,
        mixed: Literal[True],
        autocast_adapter_dtype: bool = ...,
        revision: Optional[str] = ...,
        low_cpu_mem_usage: bool = ...,
    ) -> PeftMixedModel: ...

    @overload
    def get_peft_model(
        model: nn.Module,
        peft_config: LoraConfig,
        adapter_name: str = ...,
        mixed: Literal[False] = ...,
        autocast_adapter_dtype: bool = ...,
        revision: Optional[str] = ...,
        low_cpu_mem_usage: bool = ...,
    ) -> _LoraPeftModel: ...

    @overload
    def get_peft_model(
        model: nn.Module,
        peft_config: PromptLearningConfig,
        adapter_name: str = ...,
        mixed: Literal[False] = ...,
        autocast_adapter_dtype: bool = ...,
        revision: Optional[str] = ...,
        low_cpu_mem_usage: bool = ...,
    ) -> PeftModel: ...

    @overload
    def get_peft_model(
        model: nn.Module,
        peft_config: PeftConfig,
        adapter_name: str = ...,
        mixed: Literal[False] = ...,
        autocast_adapter_dtype: bool = ...,
        revision: Optional[str] = ...,
        low_cpu_mem_usage: bool = ...,
    ) -> _TunerPeftModel: ...

    @overload
    def get_peft_model(
        model: nn.Module,
        peft_config: PeftConfig,
        adapter_name: str = ...,
        mixed: bool = ...,
        autocast_adapter_dtype: bool = ...,
        revision: Optional[str] = ...,
        low_cpu_mem_usage: bool = ...,
    ) -> PeftModel | PeftMixedModel: ...


def get_peft_model(
    model: nn.Module,
    peft_config: PeftConfig,
    adapter_name: str = "default",
    mixed: bool = False,
    autocast_adapter_dtype: bool = True,
    revision: Optional[str] = None,
    low_cpu_mem_usage: bool = False,
) -> PeftModel | PeftMixedModel:
    """
    Returns a Peft model object from a model and a config, where the model will be modified in-place.
    Args:
        model ([`torch.nn.Module`]):
            Model to be wrapped. Typically this is a Transformers model but any `nn.Module` can work, with the caveat
            that task-specific features (`peft_config.task_type`) require the model to follow Transformers conventions.
        peft_config ([`PeftConfig`]):
            Configuration object containing the parameters of the Peft model.
        adapter_name (`str`, `optional`, defaults to `"default"`):
            The name of the adapter to be injected, if not provided, the default adapter name is used ("default").
        mixed (`bool`, `optional`, defaults to `False`):
            Whether to allow mixing different (compatible) adapter types.
        autocast_adapter_dtype (`bool`, *optional*):
            Whether to autocast the adapter dtype. Defaults to `True`. Right now, this will only cast adapter weights
            using float16 or bfloat16 to float32, as this is typically required for stable training, and only affect
            select PEFT tuners.
        revision (`str`, `optional`, defaults to `main`):
            The revision of the base model. If this isn't set, the saved peft model will load the `main` revision for
            the base model
        low_cpu_mem_usage (`bool`, `optional`, defaults to `False`):
            Create empty adapter weights on meta device. Useful to speed up the loading process. Leave this setting as
            False if you intend on training the model, unless the adapter weights will be replaced by different weights
            before training starts.
    """
    old_name = peft_config.base_model_name_or_path
    # Transformers from-config models use name_or_path == ""; that is not a Hub id.
    new_name = model.__dict__.get("name_or_path", None) or None
    peft_config.base_model_name_or_path = new_name

    # Especially in notebook environments there could be a case that a user wants to experiment with different
    # configuration values. However, it is likely that there won't be any changes for new configs on an already
    # initialized PEFT model. The best we can do is warn the user about it.
    if any(isinstance(module, BaseTunerLayer) for module in model.modules()):
        warnings.warn(
            "You are trying to modify a model with PEFT for a second time. If you want to reload the model with a "
            "different config, make sure to call `.unload()` before."
        )

    if (old_name is not None) and (old_name != new_name):
        warnings.warn(
            f"The PEFT config's `base_model_name_or_path` was renamed from '{old_name}' to '{new_name}'. "
            "Please ensure that the correct base model is loaded when loading this checkpoint."
        )

    if revision is not None:
        if peft_config.revision is not None and peft_config.revision != revision:
            warnings.warn(
                f"peft config has already set base model revision to {peft_config.revision}, overwriting with revision {revision}"
            )
        peft_config.revision = revision

    if (
        (isinstance(peft_config, PEFT_TYPE_TO_CONFIG_MAPPING["LORA"]))
        and (peft_config.init_lora_weights == "eva")
        and not low_cpu_mem_usage
    ):
        warnings.warn(
            "lora with eva initialization used with low_cpu_mem_usage=False. "
            "Setting low_cpu_mem_usage=True can improve the maximum batch size possible for eva initialization."
        )

    prefix = PEFT_TYPE_TO_PREFIX_MAPPING.get(peft_config.peft_type)
    if prefix and adapter_name in prefix:
        warnings.warn(
            f"Adapter name '{adapter_name}' should not be contained in the prefix '{prefix}'. "
            "This may lead to reinitialization of the adapter weights during loading."
        )

    if mixed:
        # note: PeftMixedModel does not support autocast_adapter_dtype, so don't pass it
        return PeftMixedModel(model, peft_config, adapter_name=adapter_name)

    # We explicitly exclude prompt learning here since prompt learning is specific to the task and needs special
    # handling in the PEFT model's forward method.
    if peft_config.task_type not in MODEL_TYPE_TO_PEFT_MODEL_MAPPING.keys() and not peft_config.is_prompt_learning:
        return PeftModel(
            model,
            peft_config,
            adapter_name=adapter_name,
            autocast_adapter_dtype=autocast_adapter_dtype,
            low_cpu_mem_usage=low_cpu_mem_usage,
        )

    return MODEL_TYPE_TO_PEFT_MODEL_MAPPING[peft_config.task_type](
        model,
        peft_config,
        adapter_name=adapter_name,
        autocast_adapter_dtype=autocast_adapter_dtype,
        low_cpu_mem_usage=low_cpu_mem_usage,
    )

Xet Storage Details

Size:: 8.96 kB
Xet hash:: a2a37385809db53f14f99e0c21dc3628bdb0eb940e37eb4c58af721b83836605

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.

	# Copyright 2024-present the HuggingFace Inc. team.
	#
	# Licensed under the Apache License, Version 2.0 (the "License");
	# you may not use this file except in compliance with the License.
	# You may obtain a copy of the License at
	#
	# http://www.apache.org/licenses/LICENSE-2.0
	#
	# Unless required by applicable law or agreed to in writing, software
	# distributed under the License is distributed on an "AS IS" BASIS,
	# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	# See the License for the specific language governing permissions and
	# limitations under the License.

	from __future__ import annotations

	import warnings
	from typing import TYPE_CHECKING, Literal, Optional, overload

	from torch import nn

	from .auto import MODEL_TYPE_TO_PEFT_MODEL_MAPPING
	from .config import PeftConfig, PromptLearningConfig
	from .mapping import PEFT_TYPE_TO_CONFIG_MAPPING, PEFT_TYPE_TO_PREFIX_MAPPING
	from .mixed_model import PeftMixedModel
	from .peft_model import PeftModel
	from .tuners.tuners_utils import BaseTuner, BaseTunerLayer


	if TYPE_CHECKING:
	from .tuners import LoraConfig, LoraModel

	# A PeftModel forwards missing attributes to the wrapped tuner model via __getattr__, so its interface is
	# effectively the intersection of both classes. Python's type system cannot express intersections, so we emulate
	# them for static type checkers with classes that inherit from both, with PeftModel first in the MRO to mirror the
	# runtime attribute lookup order. These classes only exist for type checkers; at runtime, get_peft_model returns a
	# plain PeftModel (subclass).
	class _LoraPeftModel(PeftModel, LoraModel): # type: ignore[misc]
	...

	class _TunerPeftModel(PeftModel, BaseTuner): # type: ignore[misc]
	...

	# The overloads only affect static type checking, the returned types are chosen based on what attributes are
	# reachable through __getattr__ forwarding: LoRA has extra methods like add_weighted_adapter, other tuners provide
	# the BaseTuner API (e.g. merge_and_unload), and prompt learning wraps the base model directly, i.e. there is no
	# tuner model whose attributes could be forwarded. Overloads need to be defined before the actual function.
	@overload
	def get_peft_model(
	model: nn.Module,
	peft_config: PeftConfig,
	adapter_name: str = ...,
	*,
	mixed: Literal[True],
	autocast_adapter_dtype: bool = ...,
	revision: Optional[str] = ...,
	low_cpu_mem_usage: bool = ...,
	) -> PeftMixedModel: ...

	@overload
	def get_peft_model(
	model: nn.Module,
	peft_config: LoraConfig,
	adapter_name: str = ...,
	mixed: Literal[False] = ...,
	autocast_adapter_dtype: bool = ...,
	revision: Optional[str] = ...,
	low_cpu_mem_usage: bool = ...,
	) -> _LoraPeftModel: ...

	@overload
	def get_peft_model(
	model: nn.Module,
	peft_config: PromptLearningConfig,
	adapter_name: str = ...,
	mixed: Literal[False] = ...,
	autocast_adapter_dtype: bool = ...,
	revision: Optional[str] = ...,
	low_cpu_mem_usage: bool = ...,
	) -> PeftModel: ...

	@overload
	def get_peft_model(
	model: nn.Module,
	peft_config: PeftConfig,
	adapter_name: str = ...,
	mixed: Literal[False] = ...,
	autocast_adapter_dtype: bool = ...,
	revision: Optional[str] = ...,
	low_cpu_mem_usage: bool = ...,
	) -> _TunerPeftModel: ...

	@overload
	def get_peft_model(
	model: nn.Module,
	peft_config: PeftConfig,
	adapter_name: str = ...,
	mixed: bool = ...,
	autocast_adapter_dtype: bool = ...,
	revision: Optional[str] = ...,
	low_cpu_mem_usage: bool = ...,
	) -> PeftModel \| PeftMixedModel: ...


	def get_peft_model(
	model: nn.Module,
	peft_config: PeftConfig,
	adapter_name: str = "default",
	mixed: bool = False,
	autocast_adapter_dtype: bool = True,
	revision: Optional[str] = None,
	low_cpu_mem_usage: bool = False,
	) -> PeftModel \| PeftMixedModel:
	"""
	Returns a Peft model object from a model and a config, where the model will be modified in-place.

	Args:
	model ([`torch.nn.Module`]):
	Model to be wrapped. Typically this is a Transformers model but any `nn.Module` can work, with the caveat
	that task-specific features (`peft_config.task_type`) require the model to follow Transformers conventions.
	peft_config ([`PeftConfig`]):
	Configuration object containing the parameters of the Peft model.
	adapter_name (`str`, `optional`, defaults to `"default"`):
	The name of the adapter to be injected, if not provided, the default adapter name is used ("default").
	mixed (`bool`, `optional`, defaults to `False`):
	Whether to allow mixing different (compatible) adapter types.
	autocast_adapter_dtype (`bool`, optional):
	Whether to autocast the adapter dtype. Defaults to `True`. Right now, this will only cast adapter weights
	using float16 or bfloat16 to float32, as this is typically required for stable training, and only affect
	select PEFT tuners.
	revision (`str`, `optional`, defaults to `main`):
	The revision of the base model. If this isn't set, the saved peft model will load the `main` revision for
	the base model
	low_cpu_mem_usage (`bool`, `optional`, defaults to `False`):
	Create empty adapter weights on meta device. Useful to speed up the loading process. Leave this setting as
	False if you intend on training the model, unless the adapter weights will be replaced by different weights
	before training starts.
	"""
	old_name = peft_config.base_model_name_or_path
	# Transformers from-config models use name_or_path == ""; that is not a Hub id.
	new_name = model.__dict__.get("name_or_path", None) or None
	peft_config.base_model_name_or_path = new_name

	# Especially in notebook environments there could be a case that a user wants to experiment with different
	# configuration values. However, it is likely that there won't be any changes for new configs on an already
	# initialized PEFT model. The best we can do is warn the user about it.
	if any(isinstance(module, BaseTunerLayer) for module in model.modules()):
	warnings.warn(
	"You are trying to modify a model with PEFT for a second time. If you want to reload the model with a "
	"different config, make sure to call `.unload()` before."
	)

	if (old_name is not None) and (old_name != new_name):
	warnings.warn(
	f"The PEFT config's `base_model_name_or_path` was renamed from '{old_name}' to '{new_name}'. "
	"Please ensure that the correct base model is loaded when loading this checkpoint."
	)

	if revision is not None:
	if peft_config.revision is not None and peft_config.revision != revision:
	warnings.warn(
	f"peft config has already set base model revision to {peft_config.revision}, overwriting with revision {revision}"
	)
	peft_config.revision = revision

	if (
	(isinstance(peft_config, PEFT_TYPE_TO_CONFIG_MAPPING["LORA"]))
	and (peft_config.init_lora_weights == "eva")
	and not low_cpu_mem_usage
	):
	warnings.warn(
	"lora with eva initialization used with low_cpu_mem_usage=False. "
	"Setting low_cpu_mem_usage=True can improve the maximum batch size possible for eva initialization."
	)

	prefix = PEFT_TYPE_TO_PREFIX_MAPPING.get(peft_config.peft_type)
	if prefix and adapter_name in prefix:
	warnings.warn(
	f"Adapter name '{adapter_name}' should not be contained in the prefix '{prefix}'. "
	"This may lead to reinitialization of the adapter weights during loading."
	)

	if mixed:
	# note: PeftMixedModel does not support autocast_adapter_dtype, so don't pass it
	return PeftMixedModel(model, peft_config, adapter_name=adapter_name)

	# We explicitly exclude prompt learning here since prompt learning is specific to the task and needs special
	# handling in the PEFT model's forward method.
	if peft_config.task_type not in MODEL_TYPE_TO_PEFT_MODEL_MAPPING.keys() and not peft_config.is_prompt_learning:
	return PeftModel(
	model,
	peft_config,
	adapter_name=adapter_name,
	autocast_adapter_dtype=autocast_adapter_dtype,
	low_cpu_mem_usage=low_cpu_mem_usage,
	)

	return MODEL_TYPE_TO_PEFT_MODEL_MAPPING[peft_config.task_type](
	model,
	peft_config,
	adapter_name=adapter_name,
	autocast_adapter_dtype=autocast_adapter_dtype,
	low_cpu_mem_usage=low_cpu_mem_usage,
	)