Quantumbraid's picture
download
raw
15.2 kB
import math
import torch
from .qwen_image import QwenImageModel
import os
from typing import TYPE_CHECKING, List, Optional
import yaml
from toolkit import train_tools
from toolkit.config_modules import GenerateImageConfig, ModelConfig
from PIL import Image
from toolkit.models.base_model import BaseModel
from toolkit.basic import flush
from toolkit.prompt_utils import PromptEmbeds
from toolkit.samplers.custom_flowmatch_sampler import (
CustomFlowMatchEulerDiscreteScheduler,
)
from toolkit.accelerator import get_accelerator, unwrap_model
from optimum.quanto import freeze, QTensor
import torch.nn.functional as F
from diffusers import (
QwenImageTransformer2DModel,
AutoencoderKLQwenImage,
)
from transformers import Qwen2_5_VLForConditionalGeneration, Qwen2Tokenizer
from tqdm import tqdm
if TYPE_CHECKING:
from toolkit.data_transfer_object.data_loader import DataLoaderBatchDTO
try:
from .qwen_image_pipelines import QwenImageEditPlusCustomPipeline
from diffusers.pipelines.qwenimage.pipeline_qwenimage_edit_plus import (
CONDITION_IMAGE_SIZE,
VAE_IMAGE_SIZE,
)
except ImportError:
raise ImportError(
"Diffusers is out of date. Update diffusers to the latest version by doing 'pip uninstall diffusers' and then 'pip install -r requirements.txt'"
)
class QwenImageEditPlusModel(QwenImageModel):
arch = "qwen_image_edit_plus"
_qwen_image_keep_visual = True
_qwen_pipeline = QwenImageEditPlusCustomPipeline
def __init__(
self,
device,
model_config: ModelConfig,
dtype="bf16",
custom_pipeline=None,
noise_scheduler=None,
**kwargs,
):
super().__init__(
device, model_config, dtype, custom_pipeline, noise_scheduler, **kwargs
)
self.is_flow_matching = True
self.is_transformer = True
self.target_lora_modules = ["QwenImageTransformer2DModel"]
# set true for models that encode control image into text embeddings
self.encode_control_in_text_embeddings = True
# control images will come in as a list for encoding some things if true
self.has_multiple_control_images = True
# do not resize control images
self.use_raw_control_images = True
def load_model(self):
super().load_model()
def get_generation_pipeline(self):
scheduler = QwenImageModel.get_train_scheduler()
pipeline: QwenImageEditPlusCustomPipeline = QwenImageEditPlusCustomPipeline(
scheduler=scheduler,
text_encoder=unwrap_model(self.text_encoder[0]),
tokenizer=self.tokenizer[0],
processor=self.processor,
vae=unwrap_model(self.vae),
transformer=unwrap_model(self.transformer),
)
pipeline = pipeline.to(self.device_torch)
return pipeline
def generate_single_image(
self,
pipeline: QwenImageEditPlusCustomPipeline,
gen_config: GenerateImageConfig,
conditional_embeds: PromptEmbeds,
unconditional_embeds: PromptEmbeds,
generator: torch.Generator,
extra: dict,
):
self.model.to(self.device_torch, dtype=self.torch_dtype)
sc = self.get_bucket_divisibility()
gen_config.width = int(gen_config.width // sc * sc)
gen_config.height = int(gen_config.height // sc * sc)
control_img_list = []
if gen_config.ctrl_img is not None:
control_img = Image.open(gen_config.ctrl_img)
control_img = control_img.convert("RGB")
control_img_list.append(control_img)
elif gen_config.ctrl_img_1 is not None:
control_img = Image.open(gen_config.ctrl_img_1)
control_img = control_img.convert("RGB")
control_img_list.append(control_img)
if gen_config.ctrl_img_2 is not None:
control_img = Image.open(gen_config.ctrl_img_2)
control_img = control_img.convert("RGB")
control_img_list.append(control_img)
if gen_config.ctrl_img_3 is not None:
control_img = Image.open(gen_config.ctrl_img_3)
control_img = control_img.convert("RGB")
control_img_list.append(control_img)
# flush for low vram if we are doing that
# flush_between_steps = self.model_config.low_vram
flush_between_steps = False
# Fix a bug in diffusers/torch
def callback_on_step_end(pipe, i, t, callback_kwargs):
if flush_between_steps:
flush()
latents = callback_kwargs["latents"]
return {"latents": latents}
if self.model_config.low_vram:
# set vae to tile decode
pipeline.vae.enable_tiling()
img = pipeline(
image=control_img_list,
prompt_embeds=conditional_embeds.text_embeds,
prompt_embeds_mask=conditional_embeds.attention_mask.to(
self.device_torch, dtype=torch.int64
),
negative_prompt_embeds=unconditional_embeds.text_embeds,
negative_prompt_embeds_mask=unconditional_embeds.attention_mask.to(
self.device_torch, dtype=torch.int64
),
height=gen_config.height,
width=gen_config.width,
num_inference_steps=gen_config.num_inference_steps,
true_cfg_scale=gen_config.guidance_scale,
latents=gen_config.latents,
generator=generator,
callback_on_step_end=callback_on_step_end,
do_cfg_norm=gen_config.do_cfg_norm,
**extra,
).images[0]
if self.model_config.low_vram:
# restore no tiling
pipeline.vae.disable_tiling()
return img
def condition_noisy_latents(
self, latents: torch.Tensor, batch: "DataLoaderBatchDTO"
):
# we get the control image from the batch
return latents.detach()
def get_prompt_embeds(self, prompt: List, control_images=None) -> PromptEmbeds:
# todo handle not caching text encoder
if self.pipeline.text_encoder.device != self.device_torch:
self.pipeline.text_encoder.to(self.device_torch)
if control_images is None:
raise ValueError("Missing control images for QwenImageEditPlusModel")
if not isinstance(control_images, list):
control_images = [control_images]
# expects a list of list of control images List[List[Tensor]] where each item corresponds to a batch item,
# and each item in the inner list corresponds to a control image for that batch item.
# for single image/caching, it may come in as just List[Tensor], so we handle that case by wrapping it in another list
if not isinstance(control_images[0], list):
control_images = [control_images]
if len(prompt) != len(control_images):
raise ValueError("Number of prompts must match number of control image sets")
prompt_embeds_list = []
prompt_embeds_mask_list = []
for b in range(len(prompt)):
batch_control_images = control_images[b]
for i in range(len(batch_control_images)):
if len(batch_control_images[i].shape) == 3:
batch_control_images[i] = batch_control_images[i].unsqueeze(0)
# control images are 0 - 1 scale, shape (bs, ch, height, width)
ratio = batch_control_images[i].shape[2] / batch_control_images[i].shape[3]
height = math.sqrt(CONDITION_IMAGE_SIZE * ratio)
width = height / ratio
width = round(width / 32) * 32
height = round(height / 32) * 32
batch_control_images[i] = F.interpolate(
batch_control_images[i], size=(height, width), mode="bilinear"
)
prompt_embeds, prompt_embeds_mask = self.pipeline.encode_prompt(
prompt,
image=batch_control_images,
device=self.device_torch,
num_images_per_prompt=1,
)
# diffusers >=0.37 returns None when all tokens are valid (no padding)
if prompt_embeds_mask is None:
prompt_embeds_mask = torch.ones(
prompt_embeds.shape[:2], device=prompt_embeds.device, dtype=torch.int64
)
prompt_embeds_list.append(prompt_embeds)
prompt_embeds_mask_list.append(prompt_embeds_mask)
pe = PromptEmbeds(torch.cat(prompt_embeds_list, dim=0))
pe.attention_mask = torch.cat(prompt_embeds_mask_list, dim=0)
return pe
def get_noise_prediction(
self,
latent_model_input: torch.Tensor,
timestep: torch.Tensor, # 0 to 1000 scale
text_embeddings: PromptEmbeds,
batch: "DataLoaderBatchDTO" = None,
**kwargs,
):
with torch.no_grad():
batch_size, num_channels_latents, height, width = latent_model_input.shape
if self.vae.device != self.device_torch:
self.vae.to(self.device_torch)
control_image_res = VAE_IMAGE_SIZE
if self.model_config.model_kwargs.get("match_target_res", False):
# use the current target size to set the control image res
control_image_res = height * self.pipeline.vae_scale_factor * width * self.pipeline.vae_scale_factor
# pack image tokens
latent_model_input = latent_model_input.view(
batch_size, num_channels_latents, height // 2, 2, width // 2, 2
)
latent_model_input = latent_model_input.permute(0, 2, 4, 1, 3, 5)
latent_model_input = latent_model_input.reshape(
batch_size, (height // 2) * (width // 2), num_channels_latents * 4
)
raw_packed_latents = latent_model_input
img_h2, img_w2 = height // 2, width // 2
# build distinct instances per batch item, per mamad8
img_shapes = [[(1, img_h2, img_w2)] for _ in range(batch_size)]
# pack controls
if batch is None:
raise ValueError("Batch is required for QwenImageEditPlusModel")
# split the latents into batch items so we can concat the controls
packed_latents_list = torch.chunk(latent_model_input, batch_size, dim=0)
packed_latents_with_controls_list = []
batch_control_tensor_list = batch.control_tensor_list
if batch_control_tensor_list is None and batch.control_tensor is not None:
batch_control_tensor_list = []
for b in range(batch_size):
batch_control_tensor_list.append(batch.control_tensor[b : b + 1])
if batch_control_tensor_list is not None:
b = 0
for control_tensor_list in batch_control_tensor_list:
# control tensor list is a list of tensors for this batch item
controls = []
# pack control
for control_img in control_tensor_list:
# control images are 0 - 1 scale, shape (1, ch, height, width)
control_img = control_img.to(
self.device_torch, dtype=self.torch_dtype
)
# if it is only 3 dim, add batch dim
if len(control_img.shape) == 3:
control_img = control_img.unsqueeze(0)
ratio = control_img.shape[2] / control_img.shape[3]
c_height = math.sqrt(control_image_res * ratio)
c_width = c_height / ratio
c_width = round(c_width / 32) * 32
c_height = round(c_height / 32) * 32
control_img = F.interpolate(
control_img, size=(c_height, c_width), mode="bilinear"
)
# scale to -1 to 1
control_img = control_img * 2 - 1
control_latent = self.encode_images(
control_img,
device=self.device_torch,
dtype=self.torch_dtype,
)
clb, cl_num_channels_latents, cl_height, cl_width = (
control_latent.shape
)
control = control_latent.view(
1,
cl_num_channels_latents,
cl_height // 2,
2,
cl_width // 2,
2,
)
control = control.permute(0, 2, 4, 1, 3, 5)
control = control.reshape(
1,
(cl_height // 2) * (cl_width // 2),
num_channels_latents * 4,
)
img_shapes[b].append((1, cl_height // 2, cl_width // 2))
controls.append(control)
# stack controls on dim 1
control = torch.cat(controls, dim=1).to(
packed_latents_list[b].device,
dtype=packed_latents_list[b].dtype,
)
# concat with latents
packed_latents_with_control = torch.cat(
[packed_latents_list[b], control], dim=1
)
packed_latents_with_controls_list.append(
packed_latents_with_control
)
b += 1
latent_model_input = torch.cat(packed_latents_with_controls_list, dim=0)
prompt_embeds_mask = text_embeddings.attention_mask.to(
self.device_torch, dtype=torch.int64
)
enc_hs = text_embeddings.text_embeds.to(self.device_torch, self.torch_dtype)
noise_pred = self.transformer(
hidden_states=latent_model_input.to(
self.device_torch, self.torch_dtype
).detach(),
timestep=(timestep / 1000).detach(),
guidance=None,
encoder_hidden_states=enc_hs.detach(),
encoder_hidden_states_mask=prompt_embeds_mask.detach(),
img_shapes=img_shapes,
return_dict=False,
**kwargs,
)[0]
noise_pred = noise_pred[:, : raw_packed_latents.size(1)]
# unpack
noise_pred = noise_pred.view(
batch_size, height // 2, width // 2, num_channels_latents, 2, 2
)
noise_pred = noise_pred.permute(0, 3, 1, 4, 2, 5)
noise_pred = noise_pred.reshape(batch_size, num_channels_latents, height, width)
return noise_pred

Xet Storage Details

Size:
15.2 kB
·
Xet hash:
26fd86061852bdd06c5207c37006d12f643b5dbe586b064ec0819f6ccd2af7dd

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.