Buckets:
| """Boogu-Image edit (TI2I) integration for ai-toolkit. | |
| The edit model is the same Lumina2-style transformer + Qwen3-VL encoder as the | |
| base T2I model, with reference-image conditioning. A reference image feeds the | |
| model in TWO places: | |
| 1. Into the Qwen3-VL instruction encoder as image content alongside the edit | |
| instruction (so the *text embeddings* already encode the reference image). | |
| This is why ``encode_control_in_text_embeddings = True``. | |
| 2. Into the transformer as reference-image VAE latents | |
| (``ref_image_hidden_states``), which the ref-image refiner + double-stream | |
| blocks attend to. | |
| Everything else (transformer, VAE, scheduler, time/velocity convention, saving) | |
| is inherited from ``BooguImageModel`` -- this file only overrides the pieces | |
| that change for TI2I. | |
| """ | |
| import math | |
| from typing import TYPE_CHECKING, List, Optional | |
| import torch | |
| import torch.nn.functional as F | |
| from PIL import Image | |
| from torchvision.transforms.functional import to_tensor | |
| from toolkit.advanced_prompt_embeds import AdvancedPromptEmbeds | |
| from toolkit.config_modules import GenerateImageConfig, ModelConfig | |
| from .boogu_image import BooguImageModel | |
| from .src.pipeline import ( | |
| BooguImagePipeline, | |
| pad_instruction_features, | |
| run_boogu_transformer, | |
| ) | |
| if TYPE_CHECKING: | |
| from toolkit.data_transfer_object.data_loader import DataLoaderBatchDTO | |
| # Edit release (clean bf16 safetensors); same layout as the base repo. | |
| BOOGU_EDIT_PATH = "Boogu/Boogu-Image-0.1-Edit" | |
| # System prompt the edit model was trained with (SYSTEM_PROMPT_4_TI2I upstream). | |
| SYSTEM_PROMPT_TI2I = ( | |
| "Describe the key features of the input image (color, shape, size, texture, " | |
| "objects, background), then explain how the user's text instruction should " | |
| "alter or modify the image. Generate a new image that meets the user's " | |
| "requirements while maintaining consistency with the original input where " | |
| "appropriate." | |
| ) | |
| class BooguImageEditModel(BooguImageModel): | |
| arch = "boogu_image_edit" | |
| default_repo = BOOGU_EDIT_PATH | |
| def __init__( | |
| self, | |
| device, | |
| model_config: ModelConfig, | |
| dtype="bf16", | |
| custom_pipeline=None, | |
| noise_scheduler=None, | |
| **kwargs, | |
| ): | |
| super().__init__( | |
| device, model_config, dtype, custom_pipeline, noise_scheduler, **kwargs | |
| ) | |
| # The reference image is encoded into the Qwen3-VL instruction features, | |
| # so get_prompt_embeds receives the control image(s). | |
| self.encode_control_in_text_embeddings = True | |
| # Boogu supports up to 5 reference images -> they arrive as a list. | |
| self.has_multiple_control_images = True | |
| # Reference images keep their own aspect/size (not resized to the target). | |
| self.use_raw_control_images = True | |
| def text_embedding_space_version(self): | |
| # Distinct from the base T2I cache: the edit features fold in the ref image. | |
| return self.arch + "_v1" | |
| # ------------------------------------------------------------------ | |
| # Reference-image helpers | |
| # ------------------------------------------------------------------ | |
| def _vlm_resize_hw(self, h, w, max_pixels, max_side, factor=16): | |
| """Boogu's VLM image downscale (BooguImageProcessor.get_new_height_width). | |
| Scale down (never up) to fit BOTH ``max_pixels`` (area) and | |
| ``max_side_length``, then round each dim down to a multiple of ``factor`` | |
| (the image processor's ``vae_scale_factor`` = 16 for this model). The Qwen | |
| processor's own smart_resize runs afterwards, exactly as upstream. | |
| """ | |
| longest = h if h > w else w | |
| ratio_side = max_side / longest | |
| ratio_pixels = (max_pixels / (h * w)) ** 0.5 | |
| ratio = min(ratio_pixels, ratio_side, 1.0) | |
| nh = max(factor, int(h * ratio) // factor * factor) | |
| nw = max(factor, int(w * ratio) // factor * factor) | |
| return nh, nw | |
| def _ref_target_pixels(self, target_pixels: Optional[int]) -> int: | |
| """Decide the pixel budget each reference image is resized to fit within. | |
| - default: ``control_image_max_pixels`` model_kwarg (1 MP) -- a hard cap so | |
| raw, full-size control images don't blow up the token count / VRAM. | |
| - ``match_target_res`` model_kwarg: use the target generation area instead, | |
| matching Boogu's recommendation of ``max_input_image_pixels ~= H*W``. | |
| """ | |
| max_pixels = int( | |
| self.model_config.model_kwargs.get("control_image_max_pixels", 1024 * 1024) | |
| ) | |
| if ( | |
| self.model_config.model_kwargs.get("match_target_res", False) | |
| and target_pixels | |
| ): | |
| return int(target_pixels) | |
| return max_pixels | |
| def _encode_ref_latents( | |
| self, control_tensors, target_pixels: Optional[int] = None | |
| ) -> List[torch.Tensor]: | |
| """Encode ``[0, 1]`` reference image tensors to VAE latents. | |
| Returns a list of ``(16, h, w)`` latents (one per reference image). Each | |
| control image is resized so its area fits within the pixel budget (see | |
| ``_ref_target_pixels``) -- preserving aspect ratio -- then snapped so the | |
| latent grid is divisible by the patch size. ``control_tensors`` is a list | |
| of ``(C, H, W)`` or ``(1, C, H, W)`` tensors in ``[0, 1]``. | |
| """ | |
| sc = self.get_bucket_divisibility() # 16: VAE(8) * patch(2) | |
| budget = self._ref_target_pixels(target_pixels) | |
| match = self.model_config.model_kwargs.get("match_target_res", False) | |
| latents = [] | |
| for img in control_tensors: | |
| if img.dim() == 3: | |
| img = img.unsqueeze(0) | |
| img = img.to(self.device_torch, dtype=self.torch_dtype) | |
| h, w = img.shape[2], img.shape[3] | |
| # match_target_res: scale area *to* the budget; otherwise only scale | |
| # *down* when the image is larger than the budget. | |
| area = h * w | |
| if match or area > budget: | |
| ratio = h / w | |
| new_h = math.sqrt(budget * ratio) | |
| new_w = new_h / ratio | |
| else: | |
| new_h, new_w = float(h), float(w) | |
| # snap to a multiple of the bucket divisibility so the VAE latent grid | |
| # is patchifiable (the transformer rearranges 2x2 latent patches). | |
| new_h = max(sc, int(round(new_h / sc)) * sc) | |
| new_w = max(sc, int(round(new_w / sc)) * sc) | |
| if (new_h, new_w) != (h, w): | |
| img = F.interpolate(img, size=(new_h, new_w), mode="bilinear") | |
| # encode_images expects [-1, 1]; control tensors arrive in [0, 1]. | |
| latent = self.encode_images( | |
| img * 2 - 1, device=self.device_torch, dtype=self.torch_dtype | |
| ) | |
| latents.append(latent[0]) # drop batch dim -> (16, h, w) | |
| return latents | |
| def _batch_ref_latents_from_batch( | |
| self, | |
| batch: "DataLoaderBatchDTO", | |
| batch_size: int, | |
| target_pixels: Optional[int] = None, | |
| ) -> Optional[List[List[torch.Tensor]]]: | |
| """Build the transformer's ``ref_image_hidden_states`` from a train batch.""" | |
| control_list = batch.control_tensor_list | |
| if control_list is None and batch.control_tensor is not None: | |
| control_list = [batch.control_tensor[b : b + 1] for b in range(batch_size)] | |
| if control_list is None: | |
| return None | |
| if len(control_list) != batch_size: | |
| raise ValueError("Control tensor list length does not match batch size") | |
| return [ | |
| self._encode_ref_latents(controls, target_pixels=target_pixels) | |
| for controls in control_list | |
| ] | |
| # ------------------------------------------------------------------ | |
| # Conditioning | |
| # ------------------------------------------------------------------ | |
| def get_prompt_embeds(self, prompt, control_images=None) -> AdvancedPromptEmbeds: | |
| if isinstance(prompt, str): | |
| prompt = [prompt] | |
| if control_images is None: | |
| raise ValueError("BooguImageEditModel requires control (reference) images") | |
| # Normalize to List[List[Tensor]] (per-prompt list of reference images), the | |
| # same convention qwen_image_edit_plus uses. | |
| if not isinstance(control_images, list): | |
| control_images = [control_images] | |
| if not isinstance(control_images[0], list): | |
| control_images = [control_images] | |
| if len(prompt) != len(control_images): | |
| raise ValueError( | |
| "Number of prompts must match number of control image sets" | |
| ) | |
| if self.text_encoder.device == torch.device("cpu"): | |
| self.text_encoder.to(self.device_torch) | |
| device = self.text_encoder.device | |
| features_list = [] | |
| for p, ctrl in zip(prompt, control_images): | |
| # Keep reference images as tensors the whole way (no GPU->CPU->PIL | |
| # round-trip). Match Boogu's VLM preprocessing: downscale each control | |
| # image to fit max_pixels (384^2) AND max_side_length (768) -- the MLLM | |
| # only needs a coarse understanding of the reference (high-res detail | |
| # flows through the VAE ref latents), and this keeps the instruction | |
| # sequence well under the transformer rope axes_lens (~144 tokens/ref). | |
| max_pixels = int( | |
| self.model_config.model_kwargs.get("vlm_max_pixels", 384 * 384) | |
| ) | |
| max_side = int( | |
| self.model_config.model_kwargs.get("vlm_max_side_length", 768) | |
| ) | |
| images = [] | |
| for img in ctrl: | |
| if img.dim() == 4: | |
| img = img[0] | |
| img = img.to(device) | |
| nh, nw = self._vlm_resize_hw( | |
| img.shape[1], img.shape[2], max_pixels, max_side | |
| ) | |
| if (nh, nw) != (img.shape[1], img.shape[2]): | |
| img = ( | |
| F.interpolate( | |
| img.unsqueeze(0), | |
| size=(nh, nw), | |
| mode="bicubic", | |
| antialias=True, | |
| ) | |
| .squeeze(0) | |
| .clamp(0, 1) | |
| ) | |
| images.append(img) | |
| # Build just the text template with image placeholders (tokenize=False), | |
| # then let the processor expand the image tokens from the real grid size. | |
| user_content = [{"type": "image"} for _ in images] | |
| user_content.append({"type": "text", "text": p}) | |
| messages = [ | |
| { | |
| "role": "system", | |
| "content": [{"type": "text", "text": SYSTEM_PROMPT_TI2I}], | |
| }, | |
| {"role": "user", "content": user_content}, | |
| ] | |
| text = self.tokenizer.apply_chat_template( | |
| messages, tokenize=False, add_generation_prompt=False | |
| ) | |
| # do_rescale=False: control tensors are already [0, 1] (the image | |
| # normalizer maps them to [-1, 1]). No size override -- the images are | |
| # already at Boogu's target size, the processor just snaps to its grid. | |
| inputs = self.tokenizer( | |
| text=[text], | |
| images=images, | |
| return_tensors="pt", | |
| do_rescale=False, | |
| ) | |
| model_inputs = {} | |
| for k, v in inputs.items(): | |
| if isinstance(v, torch.Tensor): | |
| v = v.to(device) | |
| # cast image pixels to the encoder dtype; leave ids/masks as ints | |
| if v.is_floating_point(): | |
| v = v.to(self.torch_dtype) | |
| model_inputs[k] = v | |
| with torch.no_grad(): | |
| output = self.text_encoder(**model_inputs) | |
| features_list.append(output.last_hidden_state[0].to(self.torch_dtype)) | |
| return AdvancedPromptEmbeds(text_embeds=features_list) | |
| def get_noise_prediction( | |
| self, | |
| latent_model_input: torch.Tensor, # (B, 16, h, w) | |
| timestep: torch.Tensor, # 0..1000 scale (1000 = pure noise) | |
| text_embeddings: AdvancedPromptEmbeds, | |
| batch: "DataLoaderBatchDTO" = None, | |
| **kwargs, | |
| ): | |
| if self.model.device == torch.device("cpu"): | |
| self.model.to(self.device_torch) | |
| with torch.no_grad(): | |
| # target pixel area from the noise latents (h, w are VAE-downsampled) | |
| _, _, lh, lw = latent_model_input.shape | |
| target_pixels = (lh * self.vae_scale_factor) * (lw * self.vae_scale_factor) | |
| ref_latents = ( | |
| self._batch_ref_latents_from_batch( | |
| batch, latent_model_input.shape[0], target_pixels=target_pixels | |
| ) | |
| if batch is not None | |
| else None | |
| ) | |
| # toolkit timestep (0..1000, 1000=noise) -> Boogu native time (0=noise, 1=clean) | |
| t01 = timestep.to(self.device_torch, dtype=torch.float32) / 1000.0 | |
| if t01.dim() == 0: | |
| t01 = t01.unsqueeze(0) | |
| if t01.shape[0] != latent_model_input.shape[0]: | |
| t01 = t01.expand(latent_model_input.shape[0]) | |
| boogu_t = 1.0 - t01 | |
| instr_feats, instr_mask = pad_instruction_features( | |
| text_embeddings.text_embeds, self.device_torch, self.torch_dtype | |
| ) | |
| # Model predicts clean - noise; negate to return the toolkit velocity. | |
| raw_velocity = run_boogu_transformer( | |
| self.transformer, | |
| latent_model_input.to(self.device_torch, self.torch_dtype), | |
| boogu_t, | |
| instr_feats, | |
| instr_mask, | |
| self.get_freqs_cis(), | |
| ref_image_hidden_states=ref_latents, | |
| ) | |
| return -raw_velocity | |
| # ------------------------------------------------------------------ | |
| # Sampling previews | |
| # ------------------------------------------------------------------ | |
| def generate_single_image( | |
| self, | |
| pipeline: BooguImagePipeline, | |
| gen_config: GenerateImageConfig, | |
| conditional_embeds: AdvancedPromptEmbeds, | |
| unconditional_embeds: AdvancedPromptEmbeds, | |
| generator: torch.Generator, | |
| extra: dict, | |
| ): | |
| if self.model.device == torch.device("cpu"): | |
| self.model.to(self.device_torch) | |
| sc = self.get_bucket_divisibility() | |
| gen_config.width = int(gen_config.width // sc * sc) | |
| gen_config.height = int(gen_config.height // sc * sc) | |
| # Load the reference image(s) for the transformer ref latents. The MLLM | |
| # side already saw them (baked into conditional/unconditional embeds). | |
| ctrl_paths = [ | |
| p | |
| for p in ( | |
| gen_config.ctrl_img, | |
| gen_config.ctrl_img_1, | |
| gen_config.ctrl_img_2, | |
| gen_config.ctrl_img_3, | |
| ) | |
| if p is not None | |
| ] | |
| ref_latents = None | |
| if ctrl_paths: | |
| ctrl_tensors = [ | |
| to_tensor(Image.open(path).convert("RGB")) for path in ctrl_paths | |
| ] | |
| target_pixels = gen_config.width * gen_config.height | |
| # one batch item (preview batch size is 1) -> List[List[(16, h, w)]] | |
| ref_latents = [ | |
| self._encode_ref_latents(ctrl_tensors, target_pixels=target_pixels) | |
| ] | |
| img = pipeline( | |
| conditional_embeds=conditional_embeds, | |
| unconditional_embeds=unconditional_embeds, | |
| height=gen_config.height, | |
| width=gen_config.width, | |
| num_inference_steps=gen_config.num_inference_steps, | |
| guidance_scale=gen_config.guidance_scale, | |
| latents=gen_config.latents, | |
| generator=generator, | |
| ref_latents=ref_latents, | |
| )[0] | |
| return img | |
| def get_base_model_version(self): | |
| return "boogu_image_edit.0.1" | |
Xet Storage Details
- Size:
- 16.1 kB
- Xet hash:
- 1f6810313851505b534c6b63d0916a01907c5fcec83399abf1a09a6dd544bd4b
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.