Buckets:
| """Packing / sampling helpers for Boogu-Image (base T2I). | |
| This module glues the Qwen3-VL instruction features and the image latents into | |
| the call the Boogu transformer expects, and provides a minimal flow-matching | |
| sampler used to render preview images during training. | |
| Time convention | |
| --------------- | |
| Boogu's native flow time is ``t in [0, 1]`` with ``t=0`` pure noise and ``t=1`` | |
| clean; the transformer predicts ``clean - noise``. ai-toolkit's scheduler uses | |
| the opposite convention (``t=1`` noise, velocity ``noise - clean``). The | |
| conversion lives in ``BooguImageModel.get_noise_prediction``; this sampler runs | |
| entirely in Boogu's native domain via :func:`run_boogu_transformer`. | |
| """ | |
| from __future__ import annotations | |
| import math | |
| from typing import List, Optional | |
| import numpy as np | |
| import torch | |
| from PIL import Image | |
| from diffusers.utils.torch_utils import randn_tensor | |
| from .transformer import BooguImageTransformer2DModel | |
| # --------------------------------------------------------------------------- | |
| # Instruction feature padding. | |
| # --------------------------------------------------------------------------- | |
| def pad_instruction_features( | |
| features_list: List[torch.Tensor], | |
| device: torch.device, | |
| dtype: torch.dtype, | |
| ) -> tuple[torch.Tensor, torch.Tensor]: | |
| """Right-pad per-sample ``(L_i, D)`` instruction features into a batch. | |
| Captions are stored per-sample at their natural length and only padded to the | |
| batch max here, right before the model call. Returns ``(features (B, L, D), | |
| attention_mask (B, L))`` with the mask 1 for real tokens, 0 for padding. | |
| """ | |
| lengths = [f.shape[0] for f in features_list] | |
| max_len = max(lengths) | |
| dim = features_list[0].shape[-1] | |
| batch_size = len(features_list) | |
| features = torch.zeros(batch_size, max_len, dim, device=device, dtype=dtype) | |
| mask = torch.zeros(batch_size, max_len, dtype=torch.long, device=device) | |
| for i, f in enumerate(features_list): | |
| n = f.shape[0] | |
| features[i, :n] = f.to(device, dtype) | |
| mask[i, :n] = 1 | |
| return features, mask | |
| # --------------------------------------------------------------------------- | |
| # Time-shift schedule (mirrors the released Boogu base scheduler: v1 shift). | |
| # --------------------------------------------------------------------------- | |
| def _lin_shift( | |
| num_tokens: float, | |
| x1: float = 256.0, | |
| y1: float = 0.5, | |
| x2: float = 4096.0, | |
| y2: float = 1.15, | |
| ) -> float: | |
| """Linear token-count -> mu mapping (Boogu base_shift/max_shift defaults).""" | |
| m = (y2 - y1) / (x2 - x1) | |
| b = y1 - m * x1 | |
| return m * num_tokens + b | |
| def boogu_time_schedule( | |
| num_steps: int, | |
| num_patch_tokens: int, | |
| device: Optional[torch.device] = None, | |
| ) -> torch.Tensor: | |
| """Boogu native-domain timesteps (0=noise .. 1=clean) with v1 time shift. | |
| Returns a length ``num_steps + 1`` tensor; the trailing ``1.0`` is the clean | |
| endpoint, matching the ``_timesteps`` tail in the reference scheduler. | |
| """ | |
| t_arr = np.linspace(0.0, 1.0, num_steps + 1, dtype=np.float32)[:-1] | |
| mu = _lin_shift(max(1, int(num_patch_tokens))) | |
| eps = 1e-8 | |
| t1 = np.clip(1.0 - t_arr, eps, 1.0 - eps) | |
| num = math.exp(mu) | |
| denom = num + (1.0 / t1 - 1.0) | |
| t_arr = (1.0 - num / denom).astype(np.float32) | |
| times = np.concatenate([t_arr, np.ones(1, dtype=np.float32)]) | |
| return torch.from_numpy(times).to(device=device, dtype=torch.float32) | |
| # --------------------------------------------------------------------------- | |
| # Transformer call (Boogu native time domain). | |
| # --------------------------------------------------------------------------- | |
| def run_boogu_transformer( | |
| transformer: BooguImageTransformer2DModel, | |
| latents: torch.Tensor, # (B, 16, H, W) | |
| boogu_t: torch.Tensor, # (B,) in [0, 1], 0=noise, 1=clean | |
| instruction_features: torch.Tensor, # (B, L, instruction_feat_dim) | |
| instruction_mask: torch.Tensor, # (B, L) 1 for real tokens | |
| freqs_cis, # precomputed per-axis rotary tables | |
| ref_image_hidden_states=None, # edit/TI2I: List[List[(16, H, W)]] per batch item | |
| ) -> torch.Tensor: | |
| """Run the transformer and return the raw model velocity (``clean - noise``). | |
| Shapes pass straight through: the prediction comes back as ``(B, 16, H, W)`` | |
| in the same latent layout as ``latents``. ``ref_image_hidden_states`` stays | |
| ``None`` for the base T2I model and carries reference-image VAE latents for | |
| the edit (TI2I) model. | |
| """ | |
| out = transformer( | |
| hidden_states=latents, | |
| timestep=boogu_t, | |
| instruction_hidden_states=instruction_features, | |
| freqs_cis=freqs_cis, | |
| instruction_attention_mask=instruction_mask, | |
| ref_image_hidden_states=ref_image_hidden_states, | |
| return_dict=False, | |
| ) | |
| return out | |
| # --------------------------------------------------------------------------- | |
| # Minimal sampling pipeline (for training previews). | |
| # --------------------------------------------------------------------------- | |
| class BooguImagePipeline: | |
| """Lightweight flow-matching sampler used by ai-toolkit's preview generation.""" | |
| def __init__(self, model): | |
| # ``model`` is the BooguImageModel so we can reuse its encode/decode and | |
| # latent helpers without duplicating state. | |
| self.model = model | |
| def device(self): | |
| return self.model.device_torch | |
| def to(self, *args, **kwargs): | |
| return self | |
| def __call__( | |
| self, | |
| conditional_embeds, | |
| unconditional_embeds, | |
| height: int = 1024, | |
| width: int = 1024, | |
| num_inference_steps: int = 50, | |
| guidance_scale: float = 4.0, | |
| latents: Optional[torch.Tensor] = None, | |
| generator: Optional[torch.Generator] = None, | |
| ref_latents=None, # edit/TI2I: List[List[(16, H, W)]] reference VAE latents | |
| **kwargs, | |
| ) -> List[Image.Image]: | |
| model = self.model | |
| device = model.device_torch | |
| dtype = model.torch_dtype | |
| transformer = model.transformer | |
| patch = model.patch_size | |
| ae_scale = model.vae_scale_factor # 8 | |
| latent_channels = transformer.config.in_channels | |
| h_lat = height // ae_scale | |
| w_lat = width // ae_scale | |
| num_patch_tokens = (h_lat // patch) * (w_lat // patch) | |
| freqs_cis = model.get_freqs_cis() | |
| do_cfg = guidance_scale > 1.0 | |
| if latents is None: | |
| shape = (1, latent_channels, h_lat, w_lat) | |
| latents = randn_tensor( | |
| shape, generator=generator, device=device, dtype=torch.float32 | |
| ) | |
| # In Boogu's domain t=0 is pure noise, so the initial latent IS the noise. | |
| latents = latents.to(device, dtype=torch.float32) | |
| cond_feats, cond_mask = pad_instruction_features( | |
| conditional_embeds.text_embeds, device, dtype | |
| ) | |
| if do_cfg: | |
| uncond_feats, uncond_mask = pad_instruction_features( | |
| unconditional_embeds.text_embeds, device, dtype | |
| ) | |
| times = boogu_time_schedule(num_inference_steps, num_patch_tokens, device) | |
| for t, t_next in zip(times[:-1], times[1:]): | |
| boogu_t = t.expand(latents.shape[0]) | |
| v_cond = run_boogu_transformer( | |
| transformer, | |
| latents.to(dtype), | |
| boogu_t, | |
| cond_feats, | |
| cond_mask, | |
| freqs_cis, | |
| ref_image_hidden_states=ref_latents, | |
| ) | |
| if do_cfg: | |
| v_uncond = run_boogu_transformer( | |
| transformer, | |
| latents.to(dtype), | |
| boogu_t, | |
| uncond_feats, | |
| uncond_mask, | |
| freqs_cis, | |
| ref_image_hidden_states=ref_latents, | |
| ) | |
| v = v_uncond + guidance_scale * (v_cond - v_uncond) | |
| else: | |
| v = v_cond | |
| latents = latents + v.to(torch.float32) * (t_next - t) | |
| images = model.decode_latents(latents, device=device, dtype=dtype) | |
| images = images.float().clamp(-1.0, 1.0) | |
| images = ((images + 1.0) * 127.5).round().to(torch.uint8) | |
| images = images.permute(0, 2, 3, 1).cpu().numpy() | |
| return [Image.fromarray(arr) for arr in images] | |
Xet Storage Details
- Size:
- 8.35 kB
- Xet hash:
- 706825c633ffcd14b8d10b151193d1720153fc89bdb332f33bccd89114f39a7c
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.