Buckets:
| from typing import List, Optional, Union | |
| import numpy as np | |
| import torch | |
| import PIL.Image | |
| from dataclasses import dataclass | |
| from diffusers.image_processor import VaeImageProcessor | |
| from diffusers.schedulers import FlowMatchEulerDiscreteScheduler | |
| from diffusers.utils import ( | |
| logging, | |
| ) | |
| from diffusers.utils.torch_utils import randn_tensor | |
| from diffusers.pipelines.pipeline_utils import DiffusionPipeline | |
| from diffusers.utils import BaseOutput | |
| from toolkit.models.v2.vae.flux2_kl import AutoEncoder | |
| from .model import Flux2 | |
| from einops import rearrange | |
| from transformers import AutoProcessor, Mistral3ForConditionalGeneration | |
| from .sampling import ( | |
| get_schedule, | |
| batched_prc_img, | |
| batched_prc_txt, | |
| encode_image_refs, | |
| scatter_ids, | |
| ) | |
| class Flux2ImagePipelineOutput(BaseOutput): | |
| images: Union[List[PIL.Image.Image], np.ndarray] | |
| logger = logging.get_logger(__name__) # pylint: disable=invalid-name | |
| SYSTEM_MESSAGE = """You are an AI that reasons about image descriptions. You give structured responses focusing on object relationships, object | |
| attribution and actions without speculation.""" | |
| OUTPUT_LAYERS_MISTRAL = [10, 20, 30] | |
| OUTPUT_LAYERS_QWEN3 = [9, 18, 27] | |
| MAX_LENGTH = 512 | |
| class Flux2Pipeline(DiffusionPipeline): | |
| model_cpu_offload_seq = "text_encoder->transformer->vae" | |
| _callback_tensor_inputs = ["latents", "prompt_embeds"] | |
| def __init__( | |
| self, | |
| scheduler: FlowMatchEulerDiscreteScheduler, | |
| vae: AutoEncoder, | |
| text_encoder: Mistral3ForConditionalGeneration, | |
| tokenizer: AutoProcessor, | |
| transformer: Flux2, | |
| text_encoder_type: str = "mistral", # "mistral" or "qwen" | |
| is_guidance_distilled: bool = False, | |
| ): | |
| super().__init__() | |
| self.register_modules( | |
| vae=vae, | |
| text_encoder=text_encoder, | |
| tokenizer=tokenizer, | |
| transformer=transformer, | |
| scheduler=scheduler, | |
| ) | |
| self.vae_scale_factor = 16 # 8x plus 2x pixel shuffle | |
| self.num_channels_latents = 128 | |
| self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae_scale_factor) | |
| self.default_sample_size = 64 | |
| self.text_encoder_type = text_encoder_type | |
| self.is_guidance_distilled = is_guidance_distilled | |
| def format_input( | |
| self, | |
| txt: list[str], | |
| ) -> list[list[dict]]: | |
| # Remove [IMG] tokens from prompts to avoid Pixtral validation issues | |
| # when truncation is enabled. The processor counts [IMG] tokens and fails | |
| # if the count changes after truncation. | |
| cleaned_txt = [prompt.replace("[IMG]", "") for prompt in txt] | |
| return [ | |
| [ | |
| { | |
| "role": "system", | |
| "content": [{"type": "text", "text": SYSTEM_MESSAGE}], | |
| }, | |
| {"role": "user", "content": [{"type": "text", "text": prompt}]}, | |
| ] | |
| for prompt in cleaned_txt | |
| ] | |
| def _get_mistral_prompt_embeds( | |
| self, | |
| prompt: Union[str, List[str]] = None, | |
| device: Optional[torch.device] = None, | |
| dtype: Optional[torch.dtype] = None, | |
| max_sequence_length: int = 512, | |
| ): | |
| device = device or self._execution_device | |
| dtype = dtype or self.text_encoder.dtype | |
| if not isinstance(prompt, list): | |
| prompt = [prompt] | |
| # Format input messages | |
| messages_batch = self.format_input(txt=prompt) | |
| # Process all messages at once | |
| # with image processing a too short max length can throw an error in here. | |
| try: | |
| # tokenization kwargs ride in processor_kwargs (same values end up | |
| # in the same place; loose **kwargs just warn on new transformers) | |
| inputs = self.tokenizer.apply_chat_template( | |
| messages_batch, | |
| add_generation_prompt=False, | |
| tokenize=True, | |
| return_dict=True, | |
| return_tensors="pt", | |
| processor_kwargs={ | |
| "padding": "max_length", | |
| "truncation": True, | |
| "max_length": max_sequence_length, | |
| }, | |
| ) | |
| except ValueError as e: | |
| print( | |
| f"Error processing input: {e}, your max length is probably too short, when you have images in the input." | |
| ) | |
| raise e | |
| # Move to device | |
| input_ids = inputs["input_ids"].to(device) | |
| attention_mask = inputs["attention_mask"].to(device) | |
| # Forward pass through the model | |
| output = self.text_encoder( | |
| input_ids=input_ids, | |
| attention_mask=attention_mask, | |
| output_hidden_states=True, | |
| use_cache=False, | |
| ) | |
| out = torch.stack( | |
| [output.hidden_states[k] for k in OUTPUT_LAYERS_MISTRAL], dim=1 | |
| ) | |
| prompt_embeds = rearrange(out, "b c l d -> b l (c d)") | |
| # they don't return attention mask, so we create it here | |
| return prompt_embeds, None | |
| def _get_qwen_prompt_embeds( | |
| self, | |
| prompt: Union[str, List[str]] = None, | |
| device: Optional[torch.device] = None, | |
| dtype: Optional[torch.dtype] = None, | |
| max_sequence_length: int = 512, | |
| ): | |
| device = device or self._execution_device | |
| dtype = dtype or self.text_encoder.dtype | |
| if not isinstance(prompt, list): | |
| prompt = [prompt] | |
| all_input_ids = [] | |
| all_attention_masks = [] | |
| for p in prompt: | |
| messages = [{"role": "user", "content": p}] | |
| text = self.tokenizer.apply_chat_template( | |
| messages, | |
| tokenize=False, | |
| add_generation_prompt=True, | |
| enable_thinking=False, | |
| ) | |
| model_inputs = self.tokenizer( | |
| text, | |
| return_tensors="pt", | |
| padding="max_length", | |
| truncation=True, | |
| max_length=max_sequence_length, | |
| ) | |
| all_input_ids.append(model_inputs["input_ids"]) | |
| all_attention_masks.append(model_inputs["attention_mask"]) | |
| input_ids = torch.cat(all_input_ids, dim=0).to(device) | |
| attention_mask = torch.cat(all_attention_masks, dim=0).to(device) | |
| output = self.text_encoder( | |
| input_ids=input_ids, | |
| attention_mask=attention_mask, | |
| output_hidden_states=True, | |
| use_cache=False, | |
| ) | |
| out = torch.stack([output.hidden_states[k] for k in OUTPUT_LAYERS_QWEN3], dim=1) | |
| prompt_embeds = rearrange(out, "b c l d -> b l (c d)") | |
| # they dont use attention mask | |
| return prompt_embeds, None | |
| def encode_prompt( | |
| self, | |
| prompt: Union[str, List[str]], | |
| device: Optional[torch.device] = None, | |
| num_images_per_prompt: int = 1, | |
| prompt_embeds: Optional[torch.Tensor] = None, | |
| prompt_embeds_mask: Optional[torch.Tensor] = None, | |
| max_sequence_length: int = 512, | |
| ): | |
| device = device or self._execution_device | |
| prompt = [prompt] if isinstance(prompt, str) else prompt | |
| batch_size = len(prompt) if prompt_embeds is None else prompt_embeds.shape[0] | |
| if prompt_embeds is None: | |
| if self.text_encoder_type == "mistral": | |
| prompt_embeds, prompt_embeds_mask = self._get_mistral_prompt_embeds( | |
| prompt, device, max_sequence_length=max_sequence_length | |
| ) | |
| elif self.text_encoder_type == "qwen": | |
| prompt_embeds, prompt_embeds_mask = self._get_qwen_prompt_embeds( | |
| prompt, device, max_sequence_length=max_sequence_length | |
| ) | |
| else: | |
| raise ValueError( | |
| f"Unsupported text_encoder_type: {self.text_encoder_type}" | |
| ) | |
| _, seq_len, _ = prompt_embeds.shape | |
| prompt_embeds = prompt_embeds.repeat(1, num_images_per_prompt, 1) | |
| prompt_embeds = prompt_embeds.view( | |
| batch_size * num_images_per_prompt, seq_len, -1 | |
| ) | |
| return prompt_embeds, prompt_embeds_mask | |
| def prepare_latents( | |
| self, | |
| batch_size, | |
| num_channels_latents, | |
| height, | |
| width, | |
| dtype, | |
| device, | |
| generator, | |
| latents=None, | |
| ): | |
| height = int(height) // self.vae_scale_factor | |
| width = int(width) // self.vae_scale_factor | |
| shape = (batch_size, num_channels_latents, height, width) | |
| if latents is not None: | |
| return latents.to(device=device, dtype=dtype) | |
| if isinstance(generator, list) and len(generator) != batch_size: | |
| raise ValueError( | |
| f"You have passed a list of generators of length {len(generator)}, but requested an effective batch" | |
| f" size of {batch_size}. Make sure the batch size matches the length of the generators." | |
| ) | |
| latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype) | |
| return latents | |
| def guidance_scale(self): | |
| return self._guidance_scale | |
| def num_timesteps(self): | |
| return self._num_timesteps | |
| def current_timestep(self): | |
| return self._current_timestep | |
| def interrupt(self): | |
| return self._interrupt | |
| def __call__( | |
| self, | |
| prompt: Union[str, List[str]] = None, | |
| negative_prompt: Optional[Union[str, List[str]]] = None, | |
| height: Optional[int] = None, | |
| width: Optional[int] = None, | |
| num_inference_steps: int = 50, | |
| guidance_scale: Optional[float] = None, | |
| num_images_per_prompt: int = 1, | |
| generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, | |
| latents: Optional[torch.Tensor] = None, | |
| prompt_embeds: Optional[torch.Tensor] = None, | |
| prompt_embeds_mask: Optional[torch.Tensor] = None, | |
| negative_prompt_embeds: Optional[torch.Tensor] = None, | |
| negative_prompt_embeds_mask: Optional[torch.Tensor] = None, | |
| output_type: Optional[str] = "pil", | |
| return_dict: bool = True, | |
| max_sequence_length: int = 512, | |
| control_img_list: Optional[List[PIL.Image.Image]] = None, | |
| ): | |
| height = height or self.default_sample_size * self.vae_scale_factor | |
| width = width or self.default_sample_size * self.vae_scale_factor | |
| do_guidance = ( | |
| guidance_scale is not None | |
| and guidance_scale > 1.0 | |
| and not self.is_guidance_distilled | |
| ) | |
| self._guidance_scale = guidance_scale | |
| self._current_timestep = None | |
| self._interrupt = False | |
| # 2. Define call parameters | |
| if prompt is not None and isinstance(prompt, str): | |
| batch_size = 1 | |
| elif prompt is not None and isinstance(prompt, list): | |
| batch_size = len(prompt) | |
| else: | |
| batch_size = prompt_embeds.shape[0] | |
| device = self._execution_device | |
| # 3. Encode the prompt | |
| prompt_embeds, _ = self.encode_prompt( | |
| prompt=prompt, | |
| prompt_embeds=prompt_embeds, | |
| prompt_embeds_mask=prompt_embeds_mask, | |
| device=device, | |
| num_images_per_prompt=num_images_per_prompt, | |
| max_sequence_length=max_sequence_length, | |
| ) | |
| txt, txt_ids = batched_prc_txt(prompt_embeds) | |
| neg_txt, neg_txt_ids = None, None | |
| if do_guidance: | |
| negative_prompt_embeds, _ = self.encode_prompt( | |
| prompt=negative_prompt, | |
| prompt_embeds=negative_prompt_embeds, | |
| prompt_embeds_mask=negative_prompt_embeds_mask, | |
| device=device, | |
| num_images_per_prompt=num_images_per_prompt, | |
| max_sequence_length=max_sequence_length, | |
| ) | |
| neg_txt, neg_txt_ids = batched_prc_txt(negative_prompt_embeds) | |
| # 4. Prepare latent variables\ | |
| latents = self.prepare_latents( | |
| batch_size * num_images_per_prompt, | |
| self.num_channels_latents, | |
| height, | |
| width, | |
| prompt_embeds.dtype, | |
| device, | |
| generator, | |
| latents, | |
| ) | |
| packed_latents, img_ids = batched_prc_img(latents) | |
| timesteps = get_schedule(num_inference_steps, packed_latents.shape[1]) | |
| self._num_timesteps = len(timesteps) | |
| guidance_vec = torch.full( | |
| (packed_latents.shape[0],), | |
| guidance_scale, | |
| device=packed_latents.device, | |
| dtype=packed_latents.dtype, | |
| ) | |
| if control_img_list is not None and len(control_img_list) > 0: | |
| img_cond_seq, img_cond_seq_ids = encode_image_refs( | |
| self.vae, control_img_list | |
| ) | |
| else: | |
| img_cond_seq, img_cond_seq_ids = None, None | |
| # 6. Denoising loop | |
| i = 0 | |
| with self.progress_bar(total=num_inference_steps) as progress_bar: | |
| for t_curr, t_prev in zip(timesteps[:-1], timesteps[1:]): | |
| if self.interrupt: | |
| continue | |
| t_vec = torch.full( | |
| (packed_latents.shape[0],), | |
| t_curr, | |
| dtype=packed_latents.dtype, | |
| device=packed_latents.device, | |
| ) | |
| self._current_timestep = t_curr | |
| img_input = packed_latents | |
| img_input_ids = img_ids | |
| if img_cond_seq is not None: | |
| assert img_cond_seq_ids is not None, ( | |
| "You need to provide either both or neither of the sequence conditioning" | |
| ) | |
| img_input = torch.cat((img_input, img_cond_seq), dim=1) | |
| img_input_ids = torch.cat((img_input_ids, img_cond_seq_ids), dim=1) | |
| pred = self.transformer( | |
| x=img_input, | |
| x_ids=img_input_ids, | |
| timesteps=t_vec, | |
| ctx=txt, | |
| ctx_ids=txt_ids, | |
| guidance=guidance_vec, | |
| ) | |
| if do_guidance: | |
| pred_uncond = self.transformer( | |
| x=img_input, | |
| x_ids=img_input_ids, | |
| timesteps=t_vec, | |
| ctx=neg_txt, | |
| ctx_ids=neg_txt_ids, | |
| guidance=guidance_vec, | |
| ) | |
| pred = pred_uncond + guidance_scale * (pred - pred_uncond) | |
| if img_cond_seq is not None: | |
| pred = pred[:, : packed_latents.shape[1]] | |
| packed_latents = packed_latents + (t_prev - t_curr) * pred | |
| i += 1 | |
| progress_bar.update(1) | |
| self._current_timestep = None | |
| # 7. Post-processing | |
| latents = torch.cat(scatter_ids(packed_latents, img_ids)).squeeze(2) | |
| if output_type == "latent": | |
| image = latents | |
| else: | |
| latents = latents.to(self.vae.dtype) | |
| image = self.vae.decode(latents).float() | |
| image = self.image_processor.postprocess(image, output_type=output_type) | |
| # Offload all models | |
| self.maybe_free_model_hooks() | |
| if not return_dict: | |
| return (image,) | |
| return Flux2ImagePipelineOutput(images=image) | |
Xet Storage Details
- Size:
- 15.6 kB
- Xet hash:
- e29a9fa65ae0740d65244a3ebf0e1d37e0c3a5b3d45313a4206a3dc9ceabdcc1
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.