Buckets:
"""Boogu-Image transformer (vendored & trimmed for ai-toolkit).
Adapted from the Boogu-Image repository
(boogu/models/transformers/transformer_boogu.py). Apache-2.0.
Differences from the upstream file:
* The TeaCache / TaylorSeer inference caches are removed (training/finetuning
never use them), along with the triton RMSNorm and flash-attn fast paths.
* Prompt-tuning (``PromptEmbedding``) is dropped -- the base model does not use it.
* Gradient checkpointing is wired through every heavy block stack (the refiner
loops as well as the double-/single-stream loops) and is gated on
``torch.is_grad_enabled()`` so it is a no-op during sampling/inference.
The mixed-stream topology, weight names and numerics are otherwise identical to
upstream so the released checkpoints load unchanged.
"""
import itertools
from typing import Any, Dict, List, Optional, Tuple, Union
import torch
import torch.nn as nn
from diffusers.configuration_utils import ConfigMixin, register_to_config
from toolkit.models.v2._mixin import OstrisModelMixin
from diffusers.loaders import PeftAdapterMixin
from diffusers.loaders.single_file_model import FromOriginalModelMixin
from diffusers.models.attention_processor import Attention
from diffusers.models.modeling_outputs import Transformer2DModelOutput
from diffusers.models.modeling_utils import ModelMixin
from diffusers.utils import (
USE_PEFT_BACKEND,
logging,
scale_lora_layers,
unscale_lora_layers,
)
from einops import rearrange
from torch.nn import RMSNorm
from .attention_processor import (
ATTENTION_BACKENDS,
_FLASH_ATTN_AVAILABLE,
BooguImageAttnProcessor,
BooguImageDoubleStreamSelfAttnProcessor,
)
from .block_lumina2 import (
Lumina2CombinedTimestepCaptionEmbedding,
LuminaFeedForward,
LuminaLayerNormContinuous,
LuminaRMSNormZero,
)
from .rope import BooguImageDoubleStreamRotaryPosEmbed
logger = logging.get_logger(__name__)
class BooguImageTransformerBlock(nn.Module):
"""Basic Boogu-Image transformer block: attention + SwiGLU MLP + RMSNorm."""
def __init__(
self,
dim: int,
num_attention_heads: int,
num_kv_heads: int,
multiple_of: int,
ffn_dim_multiplier: float,
norm_eps: float,
modulation: bool = True,
) -> None:
super().__init__()
self.head_dim = dim // num_attention_heads
self.modulation = modulation
self.attn = Attention(
query_dim=dim,
cross_attention_dim=None,
dim_head=dim // num_attention_heads,
qk_norm="rms_norm",
heads=num_attention_heads,
kv_heads=num_kv_heads,
eps=1e-5,
bias=False,
out_bias=False,
processor=BooguImageAttnProcessor(),
)
self.feed_forward = LuminaFeedForward(
dim=dim,
inner_dim=4 * dim,
multiple_of=multiple_of,
ffn_dim_multiplier=ffn_dim_multiplier,
)
if modulation:
self.norm1 = LuminaRMSNormZero(
embedding_dim=dim, norm_eps=norm_eps, norm_elementwise_affine=True
)
else:
self.norm1 = RMSNorm(dim, eps=norm_eps)
self.ffn_norm1 = RMSNorm(dim, eps=norm_eps)
self.norm2 = RMSNorm(dim, eps=norm_eps)
self.ffn_norm2 = RMSNorm(dim, eps=norm_eps)
self.initialize_weights()
def initialize_weights(self) -> None:
nn.init.xavier_uniform_(self.attn.to_q.weight)
nn.init.xavier_uniform_(self.attn.to_k.weight)
nn.init.xavier_uniform_(self.attn.to_v.weight)
nn.init.xavier_uniform_(self.attn.to_out[0].weight)
nn.init.xavier_uniform_(self.feed_forward.linear_1.weight)
nn.init.xavier_uniform_(self.feed_forward.linear_2.weight)
nn.init.xavier_uniform_(self.feed_forward.linear_3.weight)
if self.modulation:
nn.init.zeros_(self.norm1.linear.weight)
nn.init.zeros_(self.norm1.linear.bias)
def forward(
self,
hidden_states: torch.Tensor,
attention_mask: torch.Tensor,
image_rotary_emb: torch.Tensor,
temb: Optional[torch.Tensor] = None,
) -> torch.Tensor:
if self.modulation:
if temb is None:
raise ValueError("temb must be provided when modulation is enabled")
norm_hidden_states, gate_msa, scale_mlp, gate_mlp = self.norm1(
hidden_states, temb
)
attn_output = self.attn(
hidden_states=norm_hidden_states,
encoder_hidden_states=norm_hidden_states,
attention_mask=attention_mask,
image_rotary_emb=image_rotary_emb,
)
hidden_states = hidden_states + gate_msa.unsqueeze(1).tanh() * self.norm2(
attn_output
)
mlp_output = self.feed_forward(
self.ffn_norm1(hidden_states) * (1 + scale_mlp.unsqueeze(1))
)
hidden_states = hidden_states + gate_mlp.unsqueeze(
1
).tanh() * self.ffn_norm2(mlp_output)
else:
norm_hidden_states = self.norm1(hidden_states)
attn_output = self.attn(
hidden_states=norm_hidden_states,
encoder_hidden_states=norm_hidden_states,
attention_mask=attention_mask,
image_rotary_emb=image_rotary_emb,
)
hidden_states = hidden_states + self.norm2(attn_output)
mlp_output = self.feed_forward(self.ffn_norm1(hidden_states))
hidden_states = hidden_states + self.ffn_norm2(mlp_output)
return hidden_states
class BooguImageNoiseRefinerTransformerBlock(BooguImageTransformerBlock):
pass
class BooguImageRefImgRefinerTransformerBlock(BooguImageTransformerBlock):
pass
class BooguImageContextRefinerTransformerBlock(BooguImageTransformerBlock):
pass
class BooguImageSingleStreamTransformerBlock(BooguImageTransformerBlock):
pass
class BooguImageDoubleStreamTransformerBlock(nn.Module):
"""Boogu-Image double-stream block: instruction & image tokens in parallel streams."""
def __init__(
self,
dim: int,
num_attention_heads: int,
num_kv_heads: int,
multiple_of: int,
ffn_dim_multiplier: float,
norm_eps: float,
modulation: bool = True,
) -> None:
super().__init__()
self.head_dim = dim // num_attention_heads
self.num_attention_heads = num_attention_heads
self.modulation = modulation
self.hidden_size = dim
double_stream_processor = BooguImageDoubleStreamSelfAttnProcessor(
head_dim=self.head_dim,
num_attention_heads=num_attention_heads,
num_kv_heads=num_kv_heads,
qkv_bias=False,
)
# Image stream components.
self.img_instruct_attn = Attention(
query_dim=dim,
cross_attention_dim=None,
dim_head=dim // num_attention_heads,
qk_norm="rms_norm",
heads=num_attention_heads,
kv_heads=num_kv_heads,
eps=1e-5,
bias=False,
out_bias=False,
processor=double_stream_processor,
)
self.img_self_attn = Attention(
query_dim=dim,
cross_attention_dim=None,
dim_head=dim // num_attention_heads,
qk_norm="rms_norm",
heads=num_attention_heads,
kv_heads=num_kv_heads,
eps=1e-5,
bias=False,
out_bias=False,
processor=BooguImageAttnProcessor(),
)
self.img_feed_forward = LuminaFeedForward(
dim=dim,
inner_dim=4 * dim,
multiple_of=multiple_of,
ffn_dim_multiplier=ffn_dim_multiplier,
)
if modulation:
self.img_norm1 = LuminaRMSNormZero(
embedding_dim=dim, norm_eps=norm_eps, norm_elementwise_affine=True
)
self.img_norm2 = LuminaRMSNormZero(
embedding_dim=dim, norm_eps=norm_eps, norm_elementwise_affine=True
)
self.img_norm3 = LuminaRMSNormZero(
embedding_dim=dim, norm_eps=norm_eps, norm_elementwise_affine=True
)
else:
self.img_norm1 = RMSNorm(dim, eps=norm_eps)
self.img_norm2 = RMSNorm(dim, eps=norm_eps)
self.img_norm3 = RMSNorm(dim, eps=norm_eps)
self.img_ffn_norm1 = RMSNorm(dim, eps=norm_eps)
self.img_attn_norm = RMSNorm(dim, eps=norm_eps)
self.img_self_attn_norm = RMSNorm(dim, eps=norm_eps)
self.img_ffn_norm2 = RMSNorm(dim, eps=norm_eps)
# Instruction stream components.
self.instruct_feed_forward = LuminaFeedForward(
dim=dim,
inner_dim=4 * dim,
multiple_of=multiple_of,
ffn_dim_multiplier=ffn_dim_multiplier,
)
if modulation:
self.instruct_norm1 = LuminaRMSNormZero(
embedding_dim=dim, norm_eps=norm_eps, norm_elementwise_affine=True
)
self.instruct_norm2 = LuminaRMSNormZero(
embedding_dim=dim, norm_eps=norm_eps, norm_elementwise_affine=True
)
else:
self.instruct_norm1 = RMSNorm(dim, eps=norm_eps)
self.instruct_norm2 = RMSNorm(dim, eps=norm_eps)
self.instruct_ffn_norm1 = RMSNorm(dim, eps=norm_eps)
self.instruct_attn_norm = RMSNorm(dim, eps=norm_eps)
self.instruct_ffn_norm2 = RMSNorm(dim, eps=norm_eps)
self.initialize_weights()
# double_stream_processor owns its own q/k/v projections, so the wrapping
# Attention's q/k/v are unused -- drop them so they aren't saved/loaded.
for param in self.img_instruct_attn.to_q.parameters():
param.requires_grad = False
for param in self.img_instruct_attn.to_k.parameters():
param.requires_grad = False
for param in self.img_instruct_attn.to_v.parameters():
param.requires_grad = False
del self.img_instruct_attn.to_k
del self.img_instruct_attn.to_v
del self.img_instruct_attn.to_q
def initialize_weights(self) -> None:
nn.init.xavier_uniform_(self.img_instruct_attn.to_out[0].weight)
nn.init.xavier_uniform_(self.img_self_attn.to_q.weight)
nn.init.xavier_uniform_(self.img_self_attn.to_k.weight)
nn.init.xavier_uniform_(self.img_self_attn.to_v.weight)
nn.init.xavier_uniform_(self.img_self_attn.to_out[0].weight)
nn.init.xavier_uniform_(self.img_feed_forward.linear_1.weight)
nn.init.xavier_uniform_(self.img_feed_forward.linear_2.weight)
nn.init.xavier_uniform_(self.img_feed_forward.linear_3.weight)
nn.init.xavier_uniform_(self.instruct_feed_forward.linear_1.weight)
nn.init.xavier_uniform_(self.instruct_feed_forward.linear_2.weight)
nn.init.xavier_uniform_(self.instruct_feed_forward.linear_3.weight)
if self.modulation:
nn.init.zeros_(self.img_norm1.linear.weight)
nn.init.zeros_(self.img_norm1.linear.bias)
nn.init.zeros_(self.img_norm2.linear.weight)
nn.init.zeros_(self.img_norm2.linear.bias)
nn.init.zeros_(self.img_norm3.linear.weight)
nn.init.zeros_(self.img_norm3.linear.bias)
nn.init.zeros_(self.instruct_norm1.linear.weight)
nn.init.zeros_(self.instruct_norm1.linear.bias)
nn.init.zeros_(self.instruct_norm2.linear.weight)
nn.init.zeros_(self.instruct_norm2.linear.bias)
def forward(
self,
img_hidden_states: torch.Tensor, # [B, L_img, D]
instruct_hidden_states: torch.Tensor, # [B, L_instruct, D]
img_attention_mask: torch.Tensor, # [B, L_img]
joint_attention_mask: torch.Tensor, # [B, L_total]
image_rotary_emb: torch.Tensor, # [B, L_img, head_dim]
rotary_emb: torch.Tensor, # [B, L_total, head_dim]
temb: Optional[torch.Tensor] = None, # [B, 1024]
encoder_seq_lengths: List[int] = None, # [B]
seq_lengths: List[int] = None, # [B]
) -> Tuple[torch.Tensor, torch.Tensor]:
if self.modulation and temb is None:
raise ValueError("temb must be provided when modulation is enabled")
batch_size = img_hidden_states.shape[0]
L_instruct = instruct_hidden_states.shape[1]
L_img = img_hidden_states.shape[1]
if self.modulation:
# Step 1: modulation for both streams.
img_norm1_out, img_gate_msa, img_scale_mlp, img_gate_mlp = self.img_norm1(
img_hidden_states, temb
)
img_norm2_out, img_shift_mlp, _, _ = self.img_norm2(img_hidden_states, temb)
img_norm3_out, img_gate_self, _, _ = self.img_norm3(img_hidden_states, temb)
(
instruct_norm1_out,
instruct_gate_msa,
instruct_scale_mlp,
instruct_gate_mlp,
) = self.instruct_norm1(instruct_hidden_states, temb)
instruct_norm2_out, instruct_shift_mlp, _, _ = self.instruct_norm2(
instruct_hidden_states, temb
)
# Step 2: joint attention on [instruct + img].
joint_attn_out = self.img_instruct_attn.processor(
attn=self.img_instruct_attn,
img_hidden_states=img_norm1_out,
instruct_hidden_states=instruct_norm1_out,
joint_attention_mask=joint_attention_mask,
rotary_emb=rotary_emb,
encoder_seq_lengths=encoder_seq_lengths,
seq_lengths=seq_lengths,
)
instruct_attn_out = instruct_hidden_states.new_zeros(
batch_size, L_instruct, self.hidden_size
)
img_attn_out = img_hidden_states.new_zeros(
batch_size, L_img, self.hidden_size
)
for i, (encoder_seq_len, seq_len) in enumerate(
zip(encoder_seq_lengths, seq_lengths)
):
instruct_attn_out[i, :encoder_seq_len] = joint_attn_out[
i, :encoder_seq_len
]
img_attn_out[i, : seq_len - encoder_seq_len] = joint_attn_out[
i, encoder_seq_len:seq_len
]
# Step 3: image self-attention.
img_self_attn_out = self.img_self_attn(
hidden_states=img_norm3_out,
encoder_hidden_states=img_norm3_out,
attention_mask=img_attention_mask,
image_rotary_emb=image_rotary_emb,
)
# Step 4: residual updates.
img_hidden_states = img_hidden_states + img_gate_msa.unsqueeze(
1
).tanh() * self.img_attn_norm(img_attn_out)
img_hidden_states = img_hidden_states + img_gate_self.unsqueeze(
1
).tanh() * self.img_self_attn_norm(img_self_attn_out)
img_mlp_input = (
1 + img_scale_mlp.unsqueeze(1)
) * img_norm2_out + img_shift_mlp.unsqueeze(1)
img_mlp_out = self.img_feed_forward(self.img_ffn_norm1(img_mlp_input))
img_hidden_states = img_hidden_states + img_gate_mlp.unsqueeze(
1
).tanh() * self.img_ffn_norm2(img_mlp_out)
instruct_hidden_states = (
instruct_hidden_states
+ instruct_gate_msa.unsqueeze(1).tanh()
* self.instruct_attn_norm(instruct_attn_out)
)
instruct_mlp_input = (
1 + instruct_scale_mlp.unsqueeze(1)
) * instruct_norm2_out + instruct_shift_mlp.unsqueeze(1)
instruct_mlp_out = self.instruct_feed_forward(
self.instruct_ffn_norm1(instruct_mlp_input)
)
instruct_hidden_states = (
instruct_hidden_states
+ instruct_gate_mlp.unsqueeze(1).tanh()
* self.instruct_ffn_norm2(instruct_mlp_out)
)
else:
# Non-modulated branch used by context-style blocks.
img_norm1_out = self.img_norm1(img_hidden_states)
img_norm3_out = self.img_norm3(img_hidden_states)
instruct_norm1_out = self.instruct_norm1(instruct_hidden_states)
joint_attn_out = self.img_instruct_attn.processor(
attn=self.img_instruct_attn,
img_hidden_states=img_norm1_out,
instruct_hidden_states=instruct_norm1_out,
joint_attention_mask=joint_attention_mask,
rotary_emb=rotary_emb,
encoder_seq_lengths=encoder_seq_lengths,
seq_lengths=seq_lengths,
)
instruct_attn_out = instruct_hidden_states.new_zeros(
batch_size, L_instruct, self.hidden_size
)
img_attn_out = img_hidden_states.new_zeros(
batch_size, L_img, self.hidden_size
)
for i, (encoder_seq_len, seq_len) in enumerate(
zip(encoder_seq_lengths, seq_lengths)
):
instruct_attn_out[i, :encoder_seq_len] = joint_attn_out[
i, :encoder_seq_len
]
img_attn_out[i, : seq_len - encoder_seq_len] = joint_attn_out[
i, encoder_seq_len:seq_len
]
img_self_attn_out = self.img_self_attn(
hidden_states=img_norm3_out,
encoder_hidden_states=img_norm3_out,
attention_mask=img_attention_mask,
image_rotary_emb=image_rotary_emb,
)
img_hidden_states = img_hidden_states + self.img_attn_norm(img_attn_out)
img_hidden_states = img_hidden_states + self.img_self_attn_norm(
img_self_attn_out
)
img_norm2_out = self.img_norm2(img_hidden_states)
img_mlp_out = self.img_feed_forward(self.img_ffn_norm1(img_norm2_out))
img_hidden_states = img_hidden_states + self.img_ffn_norm2(img_mlp_out)
instruct_hidden_states = instruct_hidden_states + self.instruct_attn_norm(
instruct_attn_out
)
instruct_norm2_out = self.instruct_norm2(instruct_hidden_states)
instruct_mlp_out = self.instruct_feed_forward(
self.instruct_ffn_norm1(instruct_norm2_out)
)
instruct_hidden_states = instruct_hidden_states + self.instruct_ffn_norm2(
instruct_mlp_out
)
return img_hidden_states, instruct_hidden_states
class BooguImageTransformer2DModel(
ModelMixin, ConfigMixin, PeftAdapterMixin, FromOriginalModelMixin, OstrisModelMixin
):
"""Boogu-Image transformer with mixed double-stream -> single-stream topology."""
aitk_subfolder = "transformer"
@classmethod
def get_transformer_block_names(cls):
return ["double_stream_layers", "single_stream_layers"]
_supports_gradient_checkpointing = True
_no_split_modules = [
"BooguImageTransformerBlock",
"BooguImageNoiseRefinerTransformerBlock",
"BooguImageRefImgRefinerTransformerBlock",
"BooguImageContextRefinerTransformerBlock",
"BooguImageSingleStreamTransformerBlock",
"BooguImageDoubleStreamTransformerBlock",
]
_skip_layerwise_casting_patterns = ["x_embedder", "norm", "embedding"]
@register_to_config
def __init__(
self,
patch_size: int = 2,
in_channels: int = 16,
out_channels: Optional[int] = None,
hidden_size: int = 3360,
num_layers: int = 40,
num_double_stream_layers: int = 8,
num_refiner_layers: int = 2,
num_attention_heads: int = 28,
num_kv_heads: int = 7,
multiple_of: int = 256,
ffn_dim_multiplier: Optional[float] = None,
norm_eps: float = 1e-5,
axes_dim_rope: Tuple[int, int, int] = (40, 40, 40),
axes_lens: Tuple[int, int, int] = (2048, 1664, 1664),
instruction_feature_configs: Dict[str, Any] = dict(
instruction_feat_dim=4096,
reduce_type="mean",
num_instruction_feat_layers=1,
),
prompt_tuning_configs: Dict[str, Any] = dict(use_prompt_tuning=False),
timestep_scale: float = 1000.0,
) -> None:
super().__init__()
if (hidden_size // num_attention_heads) != sum(axes_dim_rope):
raise ValueError(
f"hidden_size // num_attention_heads ({hidden_size // num_attention_heads}) "
f"must equal sum(axes_dim_rope) ({sum(axes_dim_rope)})"
)
if num_double_stream_layers > num_layers:
raise ValueError(
f"num_double_stream_layers ({num_double_stream_layers}) cannot be greater "
f"than num_layers ({num_layers})"
)
self.out_channels = out_channels or in_channels
self.num_double_stream_layers = num_double_stream_layers
self.num_single_stream_layers = num_layers - num_double_stream_layers
self.instruction_feature_configs = instruction_feature_configs
self.prompt_tuning_configs = prompt_tuning_configs
self.preprocessed_instruction_feat_dim = (
self.cal_preprocessed_instruction_feat_dim(instruction_feature_configs)
)
self.rope_embedder = BooguImageDoubleStreamRotaryPosEmbed(
theta=10000,
axes_dim=axes_dim_rope,
axes_lens=axes_lens,
patch_size=patch_size,
)
self.x_embedder = nn.Linear(
in_features=patch_size * patch_size * in_channels,
out_features=hidden_size,
)
self.ref_image_patch_embedder = nn.Linear(
in_features=patch_size * patch_size * in_channels,
out_features=hidden_size,
)
self.time_caption_embed = Lumina2CombinedTimestepCaptionEmbedding(
hidden_size=hidden_size,
instruction_feat_dim=self.preprocessed_instruction_feat_dim,
norm_eps=norm_eps,
timestep_scale=timestep_scale,
)
self.noise_refiner = nn.ModuleList(
[
BooguImageNoiseRefinerTransformerBlock(
hidden_size,
num_attention_heads,
num_kv_heads,
multiple_of,
ffn_dim_multiplier,
norm_eps,
modulation=True,
)
for _ in range(num_refiner_layers)
]
)
self.ref_image_refiner = nn.ModuleList(
[
BooguImageRefImgRefinerTransformerBlock(
hidden_size,
num_attention_heads,
num_kv_heads,
multiple_of,
ffn_dim_multiplier,
norm_eps,
modulation=True,
)
for _ in range(num_refiner_layers)
]
)
self.context_refiner = nn.ModuleList(
[
BooguImageContextRefinerTransformerBlock(
hidden_size,
num_attention_heads,
num_kv_heads,
multiple_of,
ffn_dim_multiplier,
norm_eps,
modulation=False,
)
for _ in range(num_refiner_layers)
]
)
self.double_stream_layers = nn.ModuleList(
[
BooguImageDoubleStreamTransformerBlock(
hidden_size,
num_attention_heads,
num_kv_heads,
multiple_of,
ffn_dim_multiplier,
norm_eps,
modulation=True,
)
for _ in range(num_double_stream_layers)
]
)
self.single_stream_layers = nn.ModuleList(
[
BooguImageSingleStreamTransformerBlock(
hidden_size,
num_attention_heads,
num_kv_heads,
multiple_of,
ffn_dim_multiplier,
norm_eps,
modulation=True,
)
for _ in range(self.num_single_stream_layers)
]
)
self.norm_out = LuminaLayerNormContinuous(
embedding_dim=hidden_size,
conditioning_embedding_dim=min(hidden_size, 1024),
elementwise_affine=False,
eps=1e-6,
bias=True,
out_dim=patch_size * patch_size * self.out_channels,
)
# Distinguish multiple reference images (supports up to 5 ref images).
self.image_index_embedding = nn.Parameter(torch.randn(5, hidden_size))
self.gradient_checkpointing = False
# Attention defaults to torch SDPA ("native"); flip to "flash" via
# set_attention_backend when flash-attn is installed and wanted.
self.attention_backend = "native"
self.initialize_weights()
self.layers = list(self.double_stream_layers) + list(self.single_stream_layers)
def initialize_weights(self) -> None:
nn.init.xavier_uniform_(self.x_embedder.weight)
nn.init.constant_(self.x_embedder.bias, 0.0)
nn.init.xavier_uniform_(self.ref_image_patch_embedder.weight)
nn.init.constant_(self.ref_image_patch_embedder.bias, 0.0)
nn.init.zeros_(self.norm_out.linear_1.weight)
nn.init.zeros_(self.norm_out.linear_1.bias)
nn.init.zeros_(self.norm_out.linear_2.weight)
nn.init.zeros_(self.norm_out.linear_2.bias)
nn.init.normal_(self.image_index_embedding, std=0.02)
def set_attention_backend(self, backend: str) -> None:
"""Select the attention implementation for every attention module.
Args:
backend: "native" for ``F.scaled_dot_product_attention`` (the default,
no extra dependency) or "flash" for Flash Attention 2
(``flash_attn_varlen_func``). Selecting "flash" requires the
``flash_attn`` package to be installed.
"""
backend = backend.lower()
if backend not in ATTENTION_BACKENDS:
raise ValueError(
f"Unknown attention backend {backend!r}. "
f"Expected one of {ATTENTION_BACKENDS}."
)
if backend == "flash" and not _FLASH_ATTN_AVAILABLE:
raise RuntimeError(
"Flash attention 2 backend requested but the `flash_attn` package "
"is not installed. Install it with `pip install flash-attn` or use "
"the 'native' backend."
)
self.attention_backend = backend
# Processors live on the wrapping diffusers Attention modules. The single
# -stream processor is a plain object (not an nn.Module) so it isn't in
# self.modules(); reach every processor through its Attention instead.
for module in self.modules():
if isinstance(module, Attention):
processor = getattr(module, "processor", None)
if hasattr(processor, "attention_backend"):
processor.attention_backend = backend
def _ckpt(self, layer, *args):
"""Run ``layer`` with activation checkpointing when training, else directly."""
if torch.is_grad_enabled() and self.gradient_checkpointing:
return self._gradient_checkpointing_func(layer, *args)
return layer(*args)
def img_patch_embed_and_refine(
self,
hidden_states,
ref_image_hidden_states,
padded_img_mask,
padded_ref_img_mask,
noise_rotary_emb,
ref_img_rotary_emb,
l_effective_ref_img_len,
l_effective_img_len,
temb,
):
"""Embed image patches and run the refiner blocks."""
batch_size = len(hidden_states)
max_combined_img_len = max(
[
img_len + sum(ref_img_len)
for img_len, ref_img_len in zip(
l_effective_img_len, l_effective_ref_img_len
)
]
)
hidden_states = self.x_embedder(hidden_states)
ref_image_hidden_states = self.ref_image_patch_embedder(ref_image_hidden_states)
for i in range(batch_size):
shift = 0
for j, ref_img_len in enumerate(l_effective_ref_img_len[i]):
ref_image_hidden_states[i, shift : shift + ref_img_len, :] = (
ref_image_hidden_states[i, shift : shift + ref_img_len, :]
+ self.image_index_embedding[j]
)
shift += ref_img_len
for layer in self.noise_refiner:
hidden_states = self._ckpt(
layer, hidden_states, padded_img_mask, noise_rotary_emb, temb
)
flat_l_effective_ref_img_len = list(itertools.chain(*l_effective_ref_img_len))
num_ref_images = len(flat_l_effective_ref_img_len)
max_ref_img_len = max(flat_l_effective_ref_img_len)
batch_ref_img_mask = ref_image_hidden_states.new_zeros(
num_ref_images, max_ref_img_len, dtype=torch.bool
)
batch_ref_image_hidden_states = ref_image_hidden_states.new_zeros(
num_ref_images, max_ref_img_len, self.config.hidden_size
)
batch_ref_img_rotary_emb = hidden_states.new_zeros(
num_ref_images,
max_ref_img_len,
ref_img_rotary_emb.shape[-1],
dtype=ref_img_rotary_emb.dtype,
)
batch_temb = temb.new_zeros(num_ref_images, *temb.shape[1:], dtype=temb.dtype)
# Flatten reference images into a temporary batch.
idx = 0
for i in range(batch_size):
shift = 0
for ref_img_len in l_effective_ref_img_len[i]:
batch_ref_img_mask[idx, :ref_img_len] = True
batch_ref_image_hidden_states[idx, :ref_img_len] = (
ref_image_hidden_states[i, shift : shift + ref_img_len]
)
batch_ref_img_rotary_emb[idx, :ref_img_len] = ref_img_rotary_emb[
i, shift : shift + ref_img_len
]
batch_temb[idx] = temb[i]
shift += ref_img_len
idx += 1
for layer in self.ref_image_refiner:
batch_ref_image_hidden_states = self._ckpt(
layer,
batch_ref_image_hidden_states,
batch_ref_img_mask,
batch_ref_img_rotary_emb,
batch_temb,
)
# Restore reference-image sequence layout.
idx = 0
for i in range(batch_size):
shift = 0
for ref_img_len in l_effective_ref_img_len[i]:
ref_image_hidden_states[i, shift : shift + ref_img_len] = (
batch_ref_image_hidden_states[idx, :ref_img_len]
)
shift += ref_img_len
idx += 1
combined_img_hidden_states = hidden_states.new_zeros(
batch_size, max_combined_img_len, self.config.hidden_size
)
for i, (ref_img_len, img_len) in enumerate(
zip(l_effective_ref_img_len, l_effective_img_len)
):
combined_img_hidden_states[i, : sum(ref_img_len)] = ref_image_hidden_states[
i, : sum(ref_img_len)
]
combined_img_hidden_states[
i, sum(ref_img_len) : sum(ref_img_len) + img_len
] = hidden_states[i, :img_len]
return combined_img_hidden_states
def flat_and_pad_to_seq(self, hidden_states, ref_image_hidden_states):
"""Flatten patch tokens and pad to batched sequences."""
batch_size = len(hidden_states)
p = self.config.patch_size
device = hidden_states[0].device
img_sizes = [(img.size(1), img.size(2)) for img in hidden_states]
l_effective_img_len = [(H // p) * (W // p) for (H, W) in img_sizes]
if ref_image_hidden_states is not None:
ref_img_sizes = [
[(img.size(1), img.size(2)) for img in imgs]
if imgs is not None
else None
for imgs in ref_image_hidden_states
]
l_effective_ref_img_len = [
[
(ref_img_size[0] // p) * (ref_img_size[1] // p)
for ref_img_size in _ref_img_sizes
]
if _ref_img_sizes is not None
else [0]
for _ref_img_sizes in ref_img_sizes
]
else:
ref_img_sizes = [None for _ in range(batch_size)]
l_effective_ref_img_len = [[0] for _ in range(batch_size)]
max_ref_img_len = max(
[sum(ref_img_len) for ref_img_len in l_effective_ref_img_len]
)
max_img_len = max(l_effective_img_len)
# Reference-image patch embeddings.
flat_ref_img_hidden_states = []
for i in range(batch_size):
if ref_img_sizes[i] is not None:
imgs = []
for ref_img in ref_image_hidden_states[i]:
C, H, W = ref_img.size()
ref_img = rearrange(
ref_img, "c (h p1) (w p2) -> (h w) (p1 p2 c)", p1=p, p2=p
)
imgs.append(ref_img)
img = torch.cat(imgs, dim=0)
flat_ref_img_hidden_states.append(img)
else:
flat_ref_img_hidden_states.append(None)
# Noise-image patch embeddings.
flat_hidden_states = []
for i in range(batch_size):
img = hidden_states[i]
C, H, W = img.size()
img = rearrange(img, "c (h p1) (w p2) -> (h w) (p1 p2 c)", p1=p, p2=p)
flat_hidden_states.append(img)
padded_ref_img_hidden_states = torch.zeros(
batch_size,
max_ref_img_len,
flat_hidden_states[0].shape[-1],
device=device,
dtype=flat_hidden_states[0].dtype,
)
padded_ref_img_mask = torch.zeros(
batch_size, max_ref_img_len, dtype=torch.bool, device=device
)
for i in range(batch_size):
if ref_img_sizes[i] is not None:
padded_ref_img_hidden_states[i, : sum(l_effective_ref_img_len[i])] = (
flat_ref_img_hidden_states[i]
)
padded_ref_img_mask[i, : sum(l_effective_ref_img_len[i])] = True
padded_hidden_states = torch.zeros(
batch_size,
max_img_len,
flat_hidden_states[0].shape[-1],
device=device,
dtype=flat_hidden_states[0].dtype,
)
padded_img_mask = torch.zeros(
batch_size, max_img_len, dtype=torch.bool, device=device
)
for i in range(batch_size):
padded_hidden_states[i, : l_effective_img_len[i]] = flat_hidden_states[i]
padded_img_mask[i, : l_effective_img_len[i]] = True
return (
padded_hidden_states,
padded_ref_img_hidden_states,
padded_img_mask,
padded_ref_img_mask,
l_effective_ref_img_len,
l_effective_img_len,
ref_img_sizes,
img_sizes,
)
def cal_preprocessed_instruction_feat_dim(
self, instruction_feature_configs: Dict[str, Any]
):
num_instruction_feat_layers = max(
instruction_feature_configs.get("num_instruction_feat_layers", 1), 1
)
instruction_feat_dim = instruction_feature_configs.get(
"instruction_feat_dim", 4096
)
reduce_type = instruction_feature_configs.get("reduce_type", "concat")
if "cat" in reduce_type.lower():
return num_instruction_feat_layers * instruction_feat_dim
elif "mean" in reduce_type.lower():
return instruction_feat_dim
else:
raise ValueError(f"Invalid reduce_type: {reduce_type}")
def preprocess_instruction_hidden_states(
self, raw_instruction_hidden_states, instruction_feature_configs: Dict[str, Any]
):
num_instruction_feat_layers = max(
instruction_feature_configs.get("num_instruction_feat_layers", 1), 1
)
reduce_type = instruction_feature_configs.get("reduce_type", "concat")
instruction_hidden_states = None
if isinstance(raw_instruction_hidden_states, torch.Tensor):
instruction_hidden_states = raw_instruction_hidden_states
elif isinstance(raw_instruction_hidden_states, (list, tuple)):
assert len(raw_instruction_hidden_states) == num_instruction_feat_layers
if "cat" in reduce_type.lower():
instruction_hidden_states = torch.cat(
raw_instruction_hidden_states, dim=-1
)
elif "mean" in reduce_type.lower():
instruction_hidden_states = torch.mean(
torch.stack(raw_instruction_hidden_states), dim=0
)
else:
raise ValueError(f"Invalid reduce_type: {reduce_type}")
else:
raise ValueError(
"Invalid type of raw_instruction_hidden_states, expected torch.Tensor "
f"or list, but got {type(raw_instruction_hidden_states)}"
)
assert (
self.preprocessed_instruction_feat_dim
== instruction_hidden_states.shape[-1]
)
return instruction_hidden_states
def forward(
self,
hidden_states: Union[torch.Tensor, List[torch.Tensor]],
timestep: torch.Tensor,
instruction_hidden_states: torch.Tensor,
freqs_cis: torch.Tensor,
instruction_attention_mask: torch.Tensor,
ref_image_hidden_states: Optional[List[List[torch.Tensor]]] = None,
attention_kwargs: Optional[Dict[str, Any]] = None,
return_dict: bool = False,
) -> Union[torch.Tensor, Transformer2DModelOutput]:
"""context/refiner -> double-stream -> fusion -> single-stream -> projection."""
instruction_hidden_states = self.preprocess_instruction_hidden_states(
instruction_hidden_states, self.instruction_feature_configs
)
if attention_kwargs is not None:
attention_kwargs = attention_kwargs.copy()
lora_scale = attention_kwargs.pop("scale", 1.0)
else:
lora_scale = 1.0
if USE_PEFT_BACKEND:
scale_lora_layers(self, lora_scale)
elif attention_kwargs is not None and attention_kwargs.get("scale") is not None:
logger.warning(
"Passing `scale` via `attention_kwargs` when not using the PEFT "
"backend is ineffective."
)
batch_size = len(hidden_states)
is_hidden_states_tensor = isinstance(hidden_states, torch.Tensor)
if is_hidden_states_tensor:
assert hidden_states.ndim == 4
hidden_states = [_hidden_states for _hidden_states in hidden_states]
device = hidden_states[0].device
# Timestep and instruction embedding.
temb, instruction_hidden_states = self.time_caption_embed(
timestep, instruction_hidden_states, hidden_states[0].dtype
)
# Flatten and pad token sequences.
(
hidden_states,
ref_image_hidden_states,
img_mask,
ref_img_mask,
l_effective_ref_img_len,
l_effective_img_len,
ref_img_sizes,
img_sizes,
) = self.flat_and_pad_to_seq(hidden_states, ref_image_hidden_states)
# Build rotary embeddings and sequence lengths.
(
context_rotary_emb,
ref_img_rotary_emb,
noise_rotary_emb,
rotary_emb,
encoder_seq_lengths,
seq_lengths,
combined_img_rotary_emb,
combined_img_seq_lengths,
) = self.rope_embedder(
freqs_cis,
instruction_attention_mask,
l_effective_ref_img_len,
l_effective_img_len,
ref_img_sizes,
img_sizes,
device,
)
# Context refinement (non-modulated, so no temb).
for layer in self.context_refiner:
instruction_hidden_states = self._ckpt(
layer,
instruction_hidden_states,
instruction_attention_mask,
context_rotary_emb,
)
# Image patch embedding and refinement.
combined_img_hidden_states = self.img_patch_embed_and_refine(
hidden_states,
ref_image_hidden_states,
img_mask,
ref_img_mask,
noise_rotary_emb,
ref_img_rotary_emb,
l_effective_ref_img_len,
l_effective_img_len,
temb,
)
instruct_hidden_states = instruction_hidden_states
img_hidden_states = combined_img_hidden_states
# Joint mask for [instruct + image].
max_seq_len = max(seq_lengths)
joint_attention_mask = hidden_states.new_zeros(
batch_size, max_seq_len, dtype=torch.bool
)
for i, seq_len in enumerate(seq_lengths):
joint_attention_mask[i, :seq_len] = True
# Double-stream stage.
if self.num_double_stream_layers > 0:
max_img_len = max(combined_img_seq_lengths)
img_attention_mask = hidden_states.new_zeros(
batch_size, max_img_len, dtype=torch.bool
)
for i, img_seq_len in enumerate(combined_img_seq_lengths):
img_attention_mask[i, :img_seq_len] = True
for layer in self.double_stream_layers:
img_hidden_states, instruct_hidden_states = self._ckpt(
layer,
img_hidden_states,
instruct_hidden_states,
img_attention_mask,
joint_attention_mask,
combined_img_rotary_emb,
rotary_emb,
temb,
encoder_seq_lengths,
seq_lengths,
)
# Fuse streams to joint sequence.
joint_hidden_states = hidden_states.new_zeros(
batch_size, max(seq_lengths), self.config.hidden_size
)
for i, (encoder_seq_len, seq_len) in enumerate(
zip(encoder_seq_lengths, seq_lengths)
):
joint_hidden_states[i, :encoder_seq_len] = instruct_hidden_states[
i, :encoder_seq_len
]
joint_hidden_states[i, encoder_seq_len:seq_len] = img_hidden_states[
i, : seq_len - encoder_seq_len
]
hidden_states = joint_hidden_states
# Single-stream stage.
for layer in self.single_stream_layers:
hidden_states = self._ckpt(
layer, hidden_states, joint_attention_mask, rotary_emb, temb
)
# Output projection.
hidden_states = self.norm_out(hidden_states, temb)
# Reshape back to image format.
p = self.config.patch_size
output = []
for i, (img_size, img_len, seq_len) in enumerate(
zip(img_sizes, l_effective_img_len, seq_lengths)
):
height, width = img_size
img_tokens = hidden_states[i][seq_len - img_len : seq_len]
img_output = rearrange(
img_tokens,
"(h w) (p1 p2 c) -> c (h p1) (w p2)",
h=height // p,
w=width // p,
p1=p,
p2=p,
)
output.append(img_output)
if is_hidden_states_tensor:
output = torch.stack(output, dim=0)
if USE_PEFT_BACKEND:
unscale_lora_layers(self, lora_scale)
if not return_dict:
return output
return Transformer2DModelOutput(sample=output)
"""Boogu-Image transformer (vendored & trimmed for ai-toolkit).
Adapted from the Boogu-Image repository
(boogu/models/transformers/transformer_boogu.py). Apache-2.0.
Differences from the upstream file:
* The TeaCache / TaylorSeer inference caches are removed (training/finetuning
never use them), along with the triton RMSNorm and flash-attn fast paths.
* Prompt-tuning (``PromptEmbedding``) is dropped -- the base model does not use it.
* Gradient checkpointing is wired through every heavy block stack (the refiner
loops as well as the double-/single-stream loops) and is gated on
``torch.is_grad_enabled()`` so it is a no-op during sampling/inference.
The mixed-stream topology, weight names and numerics are otherwise identical to
upstream so the released checkpoints load unchanged.
"""
import itertools
from typing import Any, Dict, List, Optional, Tuple, Union
import torch
import torch.nn as nn
from diffusers.configuration_utils import ConfigMixin, register_to_config
from toolkit.models.v2._mixin import OstrisModelMixin
from diffusers.loaders import PeftAdapterMixin
from diffusers.loaders.single_file_model import FromOriginalModelMixin
from diffusers.models.attention_processor import Attention
from diffusers.models.modeling_outputs import Transformer2DModelOutput
from diffusers.models.modeling_utils import ModelMixin
from diffusers.utils import (
USE_PEFT_BACKEND,
logging,
scale_lora_layers,
unscale_lora_layers,
)
from einops import rearrange
from torch.nn import RMSNorm
from .attention_processor import (
ATTENTION_BACKENDS,
_FLASH_ATTN_AVAILABLE,
BooguImageAttnProcessor,
BooguImageDoubleStreamSelfAttnProcessor,
)
from .block_lumina2 import (
Lumina2CombinedTimestepCaptionEmbedding,
LuminaFeedForward,
LuminaLayerNormContinuous,
LuminaRMSNormZero,
)
from .rope import BooguImageDoubleStreamRotaryPosEmbed
logger = logging.get_logger(__name__)
class BooguImageTransformerBlock(nn.Module):
"""Basic Boogu-Image transformer block: attention + SwiGLU MLP + RMSNorm."""
def __init__(
self,
dim: int,
num_attention_heads: int,
num_kv_heads: int,
multiple_of: int,
ffn_dim_multiplier: float,
norm_eps: float,
modulation: bool = True,
) -> None:
super().__init__()
self.head_dim = dim // num_attention_heads
self.modulation = modulation
self.attn = Attention(
query_dim=dim,
cross_attention_dim=None,
dim_head=dim // num_attention_heads,
qk_norm="rms_norm",
heads=num_attention_heads,
kv_heads=num_kv_heads,
eps=1e-5,
bias=False,
out_bias=False,
processor=BooguImageAttnProcessor(),
)
self.feed_forward = LuminaFeedForward(
dim=dim,
inner_dim=4 * dim,
multiple_of=multiple_of,
ffn_dim_multiplier=ffn_dim_multiplier,
)
if modulation:
self.norm1 = LuminaRMSNormZero(
embedding_dim=dim, norm_eps=norm_eps, norm_elementwise_affine=True
)
else:
self.norm1 = RMSNorm(dim, eps=norm_eps)
Xet Storage Details
- Size:
- 44.7 kB
- Xet hash:
- c0769ba0f49115b84c453e36de8ee50e6429670b9bfb250dfc3856bb5ddafdea
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.