src/animatediff/utils/device.py

import logging
from functools import lru_cache
from math import ceil
from typing import Union

import torch

logger = logging.getLogger(__name__)


def device_info_str(device: torch.device) -> str:
    device_info = torch.cuda.get_device_properties(device)
    return (
        f"{device_info.name} {ceil(device_info.total_memory / 1024 ** 3)}GB, "
        + f"CC {device_info.major}.{device_info.minor}, {device_info.multi_processor_count} SM(s)"
    )


@lru_cache(maxsize=4)
def supports_bfloat16(device: Union[str, torch.device]) -> bool:
    """A non-exhaustive check for bfloat16 support on a given device.
    Weird that torch doesn't have a global function for this. If your device
    does support bfloat16 and it's not listed here, go ahead and add it.
    """
    device = torch.device(device)  # make sure device is a torch.device
    match device.type:
        case "cpu":
            ret = False
        case "cuda":
            with device:
                ret = torch.cuda.is_bf16_supported()
        case "xla":
            ret = True
        case "mps":
            ret = True
        case _:
            ret = False
    return ret


@lru_cache(maxsize=4)
def maybe_bfloat16(
    device: Union[str, torch.device],
    fallback: torch.dtype = torch.float32,
) -> torch.dtype:
    """Returns torch.bfloat16 if available, otherwise the fallback dtype (default float32)"""
    device = torch.device(device)  # make sure device is a torch.device
    return torch.bfloat16 if supports_bfloat16(device) else fallback


def dtype_for_model(model: str, device: torch.device) -> torch.dtype:
    match model:
        case "unet":
            return torch.float32 if device.type == "cpu" else torch.float16
        case "tenc":
            return torch.float32 if device.type == "cpu" else torch.float16
        case "vae":
            return maybe_bfloat16(device, fallback=torch.float32)
        case unknown:
            raise ValueError(f"Invalid model {unknown}")


def get_model_dtypes(
    device: Union[str, torch.device],
    force_half_vae: bool = False,
) -> tuple[torch.dtype, torch.dtype, torch.dtype]:
    device = torch.device(device)  # make sure device is a torch.device
    unet_dtype = dtype_for_model("unet", device)
    tenc_dtype = dtype_for_model("tenc", device)
    vae_dtype = dtype_for_model("vae", device)

    if device.type == "cpu":
        logger.warn("Device explicitly set to CPU, will run everything in fp32")
        logger.warn("This is likely to be *incredibly* slow, but I don't tell you how to live.")

    if force_half_vae:
        if device.type == "cpu":
            logger.critical("Can't force VAE to fp16 mode on CPU! Exiting...")
            raise RuntimeError("Can't force VAE to fp16 mode on CPU!")
        if vae_dtype == torch.bfloat16:
            logger.warn("Forcing VAE to use fp16 despite bfloat16 support! This is a bad idea!")
            logger.warn("If you're not sure why you're doing this, you probably shouldn't be.")
            vae_dtype = torch.float16
        else:
            logger.warn("Forcing VAE to use fp16 instead of fp32 on CUDA! This may result in black outputs!")
            logger.warn("Running a VAE in fp16 can result in black images or poor output quality.")
            logger.warn("I don't tell you how to live, but you probably shouldn't do this.")
            vae_dtype = torch.float16

    logger.info(f"Selected data types: {unet_dtype=}, {tenc_dtype=}, {vae_dtype=}")
    return unet_dtype, tenc_dtype, vae_dtype


def get_memory_format(device: Union[str, torch.device]) -> torch.memory_format:
    device = torch.device(device)  # make sure device is a torch.device
    # if we have a cuda device
    if device.type == "cuda":
        device_info = torch.cuda.get_device_properties(device)
        # Volta and newer seem to like channels_last. This will probably bite me on TU11x cards.
        if device_info.major >= 7:
            ret = torch.channels_last
        else:
            ret = torch.contiguous_format
    elif device.type == "xpu":
        # Intel ARC GPUs/XPUs like channels_last
        ret = torch.channels_last
    else:
        # TODO: Does MPS like channels_last? do other devices?
        ret = torch.contiguous_format
    if ret == torch.channels_last:
        logger.info("Using channels_last memory format for UNet and VAE")
    return ret