ComfyUI-Easy-Use/py/dit/pixArt/loader.py

import torch
import math
import comfy.supported_models_base
import comfy.latent_formats
import comfy.model_patcher
import comfy.model_base
import comfy.utils
import comfy.conds
from comfy import model_management
from .diffusers_convert import convert_state_dict

# checkpointbf
class EXM_PixArt(comfy.supported_models_base.BASE):
    unet_config = {}
    unet_extra_config = {}
    latent_format = comfy.latent_formats.SD15

    def __init__(self, model_conf):
        self.model_target = model_conf.get("target")
        self.unet_config = model_conf.get("unet_config", {})
        self.sampling_settings = model_conf.get("sampling_settings", {})
        self.latent_format = self.latent_format()
        # UNET is handled by extension
        self.unet_config["disable_unet_model_creation"] = True

    def model_type(self, state_dict, prefix=""):
        return comfy.model_base.ModelType.EPS


class EXM_PixArt_Model(comfy.model_base.BaseModel):
    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)

    def extra_conds(self, **kwargs):
        out = super().extra_conds(**kwargs)

        img_hw = kwargs.get("img_hw", None)
        if img_hw is not None:
            out["img_hw"] = comfy.conds.CONDRegular(torch.tensor(img_hw))

        aspect_ratio = kwargs.get("aspect_ratio", None)
        if aspect_ratio is not None:
            out["aspect_ratio"] = comfy.conds.CONDRegular(torch.tensor(aspect_ratio))

        cn_hint = kwargs.get("cn_hint", None)
        if cn_hint is not None:
            out["cn_hint"] = comfy.conds.CONDRegular(cn_hint)

        return out


def load_pixart(model_path, model_conf=None):
    state_dict = comfy.utils.load_torch_file(model_path)
    state_dict = state_dict.get("model", state_dict)

    # prefix
    for prefix in ["model.diffusion_model.", ]:
        if any(True for x in state_dict if x.startswith(prefix)):
            state_dict = {k[len(prefix):]: v for k, v in state_dict.items()}

    # diffusers
    if "adaln_single.linear.weight" in state_dict:
        state_dict = convert_state_dict(state_dict)  # Diffusers

    # guess auto config
    if model_conf is None:
        model_conf = guess_pixart_config(state_dict)

    parameters = comfy.utils.calculate_parameters(state_dict)
    unet_dtype = model_management.unet_dtype(model_params=parameters)
    load_device = comfy.model_management.get_torch_device()
    offload_device = comfy.model_management.unet_offload_device()

    # ignore fp8/etc and use directly for now
    manual_cast_dtype = model_management.unet_manual_cast(unet_dtype, load_device)
    if manual_cast_dtype:
        print(f"PixArt: falling back to {manual_cast_dtype}")
        unet_dtype = manual_cast_dtype

    model_conf = EXM_PixArt(model_conf)  # convert to object
    model = EXM_PixArt_Model(  # same as comfy.model_base.BaseModel
        model_conf,
        model_type=comfy.model_base.ModelType.EPS,
        device=model_management.get_torch_device()
    )

    if model_conf.model_target == "PixArtMS":
        from .models.PixArtMS import PixArtMS
        model.diffusion_model = PixArtMS(**model_conf.unet_config)
    elif model_conf.model_target == "PixArt":
        from .models.PixArt import PixArt
        model.diffusion_model = PixArt(**model_conf.unet_config)
    elif model_conf.model_target == "PixArtMSSigma":
        from .models.PixArtMS import PixArtMS
        model.diffusion_model = PixArtMS(**model_conf.unet_config)
        model.latent_format = comfy.latent_formats.SDXL()
    elif model_conf.model_target == "ControlPixArtMSHalf":
        from .models.PixArtMS import PixArtMS
        from .models.pixart_controlnet import ControlPixArtMSHalf
        model.diffusion_model = PixArtMS(**model_conf.unet_config)
        model.diffusion_model = ControlPixArtMSHalf(model.diffusion_model)
    elif model_conf.model_target == "ControlPixArtHalf":
        from .models.PixArt import PixArt
        from .models.pixart_controlnet import ControlPixArtHalf
        model.diffusion_model = PixArt(**model_conf.unet_config)
        model.diffusion_model = ControlPixArtHalf(model.diffusion_model)
    else:
        raise NotImplementedError(f"Unknown model target '{model_conf.model_target}'")

    m, u = model.diffusion_model.load_state_dict(state_dict, strict=False)
    if len(m) > 0: print("Missing UNET keys", m)
    if len(u) > 0: print("Leftover UNET keys", u)
    model.diffusion_model.dtype = unet_dtype
    model.diffusion_model.eval()
    model.diffusion_model.to(unet_dtype)

    model_patcher = comfy.model_patcher.ModelPatcher(
        model,
        load_device=load_device,
        offload_device=offload_device,
    )
    return model_patcher


def guess_pixart_config(sd):
    """
    Guess config based on converted state dict.
    """
    # Shared settings based on DiT_XL_2 - could be enumerated
    config = {
        "num_heads": 16,  # get from attention
        "patch_size": 2,  # final layer I guess?
        "hidden_size": 1152,  # pos_embed.shape[2]
    }
    config["depth"] = sum([key.endswith(".attn.proj.weight") for key in sd.keys()]) or 28

    try:
        # this is not present in the diffusers version for sigma?
        config["model_max_length"] = sd["y_embedder.y_embedding"].shape[0]
    except KeyError:
        # need better logic to guess this
        config["model_max_length"] = 300

    if "pos_embed" in sd:
        config["input_size"] = int(math.sqrt(sd["pos_embed"].shape[1])) * config["patch_size"]
        config["pe_interpolation"] = config["input_size"] // (512 // 8)  # dumb guess

    target_arch = "PixArtMS"
    if config["model_max_length"] == 300:
        # Sigma
        target_arch = "PixArtMSSigma"
        config["micro_condition"] = False
        if "input_size" not in config:
            # The diffusers weights for 1K/2K are exactly the same...?
            # replace patch embed logic with HyDiT?
            print(f"PixArt: diffusers weights - 2K model will be broken, use manual loading!")
            config["input_size"] = 1024 // 8
    else:
        # Alpha
        if "csize_embedder.mlp.0.weight" in sd:
            # MS (microconds)
            target_arch = "PixArtMS"
            config["micro_condition"] = True
            if "input_size" not in config:
                config["input_size"] = 1024 // 8
                config["pe_interpolation"] = 2
        else:
            # PixArt
            target_arch = "PixArt"
            if "input_size" not in config:
                config["input_size"] = 512 // 8
                config["pe_interpolation"] = 1

    print("PixArt guessed config:", target_arch, config)
    return {
        "target": target_arch,
        "unet_config": config,
        "sampling_settings": {
            "beta_schedule": "sqrt_linear",
            "linear_start": 0.0001,
            "linear_end": 0.02,
            "timesteps": 1000,
        }
    }

# lora
class EXM_PixArt_ModelPatcher(comfy.model_patcher.ModelPatcher):
    def calculate_weight(self, patches, weight, key):
        """
        This is almost the same as the comfy function, but stripped down to just the LoRA patch code.
        The problem with the original code is the q/k/v keys being combined into one for the attention.
        In the diffusers code, they're treated as separate keys, but in the reference code they're recombined (q+kv|qkv).
        This means, for example, that the [1152,1152] weights become [3456,1152] in the state dict.
        The issue with this is that the LoRA weights are [128,1152],[1152,128] and become [384,1162],[3456,128] instead.

        This is the best thing I could think of that would fix that, but it's very fragile.
         - Check key shape to determine if it needs the fallback logic
         - Cut the input into parts based on the shape (undoing the torch.cat)
         - Do the matrix multiplication logic
         - Recombine them to match the expected shape
        """
        for p in patches:
            alpha = p[0]
            v = p[1]
            strength_model = p[2]
            if strength_model != 1.0:
                weight *= strength_model

            if isinstance(v, list):
                v = (self.calculate_weight(v[1:], v[0].clone(), key),)

            if len(v) == 2:
                patch_type = v[0]
                v = v[1]

            if patch_type == "lora":
                mat1 = comfy.model_management.cast_to_device(v[0], weight.device, torch.float32)
                mat2 = comfy.model_management.cast_to_device(v[1], weight.device, torch.float32)
                if v[2] is not None:
                    alpha *= v[2] / mat2.shape[0]
                try:
                    mat1 = mat1.flatten(start_dim=1)
                    mat2 = mat2.flatten(start_dim=1)

                    ch1 = mat1.shape[0] // mat2.shape[1]
                    ch2 = mat2.shape[0] // mat1.shape[1]
                    ### Fallback logic for shape mismatch ###
                    if mat1.shape[0] != mat2.shape[1] and ch1 == ch2 and (mat1.shape[0] / mat2.shape[1]) % 1 == 0:
                        mat1 = mat1.chunk(ch1, dim=0)
                        mat2 = mat2.chunk(ch1, dim=0)
                        weight += torch.cat(
                            [alpha * torch.mm(mat1[x], mat2[x]) for x in range(ch1)],
                            dim=0,
                        ).reshape(weight.shape).type(weight.dtype)
                    else:
                        weight += (alpha * torch.mm(mat1, mat2)).reshape(weight.shape).type(weight.dtype)
                except Exception as e:
                    print("ERROR", key, e)
        return weight

    def clone(self):
        n = EXM_PixArt_ModelPatcher(self.model, self.load_device, self.offload_device, self.size, self.current_device,
                                    weight_inplace_update=self.weight_inplace_update)
        n.patches = {}
        for k in self.patches:
            n.patches[k] = self.patches[k][:]

        n.object_patches = self.object_patches.copy()
        n.model_options = copy.deepcopy(self.model_options)
        n.model_keys = self.model_keys
        return n


def replace_model_patcher(model):
    n = EXM_PixArt_ModelPatcher(
        model=model.model,
        size=model.size,
        load_device=model.load_device,
        offload_device=model.offload_device,
        current_device=model.current_device,
        weight_inplace_update=model.weight_inplace_update,
    )
    n.patches = {}
    for k in model.patches:
        n.patches[k] = model.patches[k][:]

    n.object_patches = model.object_patches.copy()
    n.model_options = copy.deepcopy(model.model_options)
    return n


def find_peft_alpha(path):
    def load_json(json_path):
        with open(json_path) as f:
            data = json.load(f)
        alpha = data.get("lora_alpha")
        alpha = alpha or data.get("alpha")
        if not alpha:
            print(" Found config but `lora_alpha` is missing!")
        else:
            print(f" Found config at {json_path} [alpha:{alpha}]")
        return alpha

    # For some weird reason peft doesn't include the alpha in the actual model
    print("PixArt: Warning! This is a PEFT LoRA. Trying to find config...")
    files = [
        f"{os.path.splitext(path)[0]}.json",
        f"{os.path.splitext(path)[0]}.config.json",
        os.path.join(os.path.dirname(path), "adapter_config.json"),
    ]
    for file in files:
        if os.path.isfile(file):
            return load_json(file)

    print(" Missing config/alpha! assuming alpha of 8. Consider converting it/adding a config json to it.")
    return 8.0


def load_pixart_lora(model, lora, lora_path, strength):
    k_back = lambda x: x.replace(".lora_up.weight", "")
    # need to convert the actual weights for this to work.
    if any(True for x in lora.keys() if x.endswith("adaln_single.linear.lora_A.weight")):
        lora = convert_lora_state_dict(lora, peft=True)
        alpha = find_peft_alpha(lora_path)
        lora.update({f"{k_back(x)}.alpha": torch.tensor(alpha) for x in lora.keys() if "lora_up" in x})
    else:  # OneTrainer
        lora = convert_lora_state_dict(lora, peft=False)

    key_map = {k_back(x): f"diffusion_model.{k_back(x)}.weight" for x in lora.keys() if "lora_up" in x}  # fake

    loaded = comfy.lora.load_lora(lora, key_map)
    if model is not None:
        # switch to custom model patcher when using LoRAs
        if isinstance(model, EXM_PixArt_ModelPatcher):
            new_modelpatcher = model.clone()
        else:
            new_modelpatcher = replace_model_patcher(model)
        k = new_modelpatcher.add_patches(loaded, strength)
    else:
        k = ()
        new_modelpatcher = None

    k = set(k)
    for x in loaded:
        if (x not in k):
            print("NOT LOADED", x)

    return new_modelpatcher