Quantumbraid's picture
download
raw
15.8 kB
"""Packing / sampling helpers for Krea 2.
Turns image latents + stacked Qwen3-VL text features into the single sequence the
``SingleStreamDiT`` consumes, and provides a minimal flow-matching sampler used to
render preview images during training.
Time convention: Krea 2 is a plain flow-matching model whose time runs ``t=1``
(pure noise) -> ``t=0`` (clean), the velocity it predicts is ``noise - clean``,
and ``x_t = (1 - t) * clean + t * noise``. This is *identical* to ai-toolkit's
convention, so unlike ideogram4 there is no flipping or negation -- the toolkit
``timestep / 1000`` flows straight through as ``t``.
"""
from __future__ import annotations
import math
from typing import List, Optional
import torch
from einops import rearrange, repeat
from PIL import Image
from diffusers.utils.torch_utils import randn_tensor
from .mmdit import SingleStreamDiT
# ---------------------------------------------------------------------------
# Text feature padding.
# ---------------------------------------------------------------------------
def pad_text_features(
features_list: List[torch.Tensor],
device: torch.device,
dtype: torch.dtype,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Right-pad a list of per-sample ``(Lt_i, F)`` features into a batch.
Each caption is stored 2D at its natural length -- the 12 stacked Qwen3-VL
hidden-state layers are flattened into the feature axis ``F = n * d`` so the
ai-toolkit batching machinery treats the list length as the batch size (it
only special-cases 2D per-sample tensors). The layer axis is restored in
``predict_velocity`` right before the MMDiT call. Padding to the batch max is
deferred to here. Returns ``(features (B, Lt, F), mask (B, Lt))``; the mask is
1 for real text tokens and 0 for padding.
"""
lengths = [f.shape[0] for f in features_list]
max_len = max(lengths)
dim = features_list[0].shape[-1]
batch_size = len(features_list)
features = torch.zeros(batch_size, max_len, dim, device=device, dtype=dtype)
mask = torch.zeros(batch_size, max_len, dtype=torch.long, device=device)
for i, f in enumerate(features_list):
ln = f.shape[0]
features[i, :ln] = f.to(device, dtype)
mask[i, :ln] = 1
return features, mask
# ---------------------------------------------------------------------------
# Latent <-> token packing and combined position / mask construction.
# ---------------------------------------------------------------------------
def prepare(
img: torch.Tensor, txtlen: int, patch: int, txtmask: torch.Tensor
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
"""Patchify the latent and build the combined text+image position / mask.
in: img (B, C, h, w) image latent
txtlen number of text tokens
patch transformer patch size
txtmask (B, txtlen) long/bool mask, 1 for real text tokens
out: (img_tokens (B, h/p*w/p, C*p*p), pos (B, txtlen+imglen, 3),
mask (B, txtlen+imglen))
"""
b, _, h, w = img.shape
h_, w_ = h // patch, w // patch
imgids = torch.zeros((h_, w_, 3), device=img.device)
imgids[..., 1] = torch.arange(h_, device=img.device)[:, None]
imgids[..., 2] = torch.arange(w_, device=img.device)[None, :]
imgpos = repeat(imgids, "h w three -> b (h w) three", b=b, three=3)
imgmask = torch.ones(b, h_ * w_, device=img.device, dtype=torch.bool)
img = rearrange(img, "b c (h ph) (w pw) -> b (h w) (c ph pw)", ph=patch, pw=patch)
txtpos = torch.zeros(b, txtlen, 3, device=img.device)
mask = torch.cat((txtmask.to(img.device).bool(), imgmask), dim=1)
pos = torch.cat((txtpos, imgpos), dim=1)
return img, pos, mask
def pack_ref_latents(
ref_latents: List[List[torch.Tensor]], patch: int, device, dtype
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
"""Patchify per-sample reference latents into padded ref tokens / pos / mask.
``ref_latents`` is a list (one entry per batch item) of lists of ``(C, h, w)``
reference latents. The i-th reference of a sample is placed on RoPE axis 0 at
index ``i + 1`` with its own y/x grid starting at 0 -- the ComfyUI Kontext
"index" placement (axis 0 is otherwise always 0, so the base weights see the
references as a new "frame" axis). Samples with fewer reference tokens are
right-padded and masked out. Returns ``(tokens (B, Lr, C*p*p),
pos (B, Lr, 3), mask (B, Lr))``.
"""
token_dim = None
seqs, ids = [], []
for refs in ref_latents:
toks, rpos = [], []
for i, ref in enumerate(refs):
ref = ref.to(device, dtype)
_, h, w = ref.shape
h_, w_ = h // patch, w // patch
toks.append(
rearrange(ref, "c (h ph) (w pw) -> (h w) (c ph pw)", ph=patch, pw=patch)
)
token_dim = toks[-1].shape[-1]
refids = torch.zeros((h_, w_, 3), device=device)
refids[..., 0] = i + 1
refids[..., 1] = torch.arange(h_, device=device)[:, None]
refids[..., 2] = torch.arange(w_, device=device)[None, :]
rpos.append(refids.reshape(-1, 3))
seqs.append(toks)
ids.append(rpos)
b = len(seqs)
# a sample may have no references (its span is fully masked/padded)
seqs = [
torch.cat(t, dim=0)
if t
else torch.zeros(0, token_dim, device=device, dtype=dtype)
for t in seqs
]
ids = [torch.cat(p, dim=0) if p else torch.zeros(0, 3, device=device) for p in ids]
max_len = max(s.shape[0] for s in seqs)
tokens = torch.zeros(b, max_len, token_dim, device=device, dtype=dtype)
pos = torch.zeros(b, max_len, 3, device=device)
mask = torch.zeros(b, max_len, device=device, dtype=torch.bool)
for i, (s, p) in enumerate(zip(seqs, ids)):
ln = s.shape[0]
tokens[i, :ln] = s
pos[i, :ln] = p
mask[i, :ln] = True
return tokens, pos, mask
def predict_velocity(
model: SingleStreamDiT,
latents: torch.Tensor, # (B, C, h, w)
t: torch.Tensor, # (B,) flow time in [0, 1] (1 = pure noise)
context: torch.Tensor, # (B, Lt, n*d) flattened stacked Qwen3-VL features
text_mask: torch.Tensor, # (B, Lt) 1 for real text tokens
ref_latents: Optional[List[List[torch.Tensor]]] = None, # per-sample (C, h, w) refs
isolate_refs: bool = False,
ref_kv_cache: Optional[dict] = None,
) -> torch.Tensor:
"""Run the MMDiT on the packed [text | image | refs] sequence.
``latents`` stay in the unpacked ``(B, C, h, w)`` latent layout; image-token
packing is internal to this function. ``context`` arrives 2D-per-sample
flattened ``(B, Lt, n*d)`` and is restored to ``(B, Lt, n, d)`` for the MMDiT.
``ref_latents`` (optional) are clean reference latents appended after the
image tokens and conditioned at t=0 ("index_timestep_zero"); the prediction
only ever covers the noisy target tokens. ``isolate_refs`` restricts ref
tokens to attending only among themselves (see ``SingleStreamDiT.forward``),
making their per-layer K/V cacheable across steps. ``ref_kv_cache`` is a
``{"kv": None, "mask": None}`` dict enabling that cache (inference only,
requires ``isolate_refs``): while ``kv`` is unset the call runs the refs
normally and fills the dict with each block's ref K/V; once filled, the ref
tokens are dropped from the sequence and the cached K/V are injected as
extra attention keys -- identical math, no ref recompute per step. Returns
the velocity ``noise - clean`` reshaped back to ``(B, C, h, w)``. No time
flip / negation: Krea's convention matches toolkit's.
"""
patch = model.config.patch
b, c, h, w = latents.shape
if ref_kv_cache is not None and not isolate_refs:
raise ValueError(
"ref_kv_cache requires isolate_refs: cached ref K/V are only "
"step-invariant when ref tokens attend solely to each other"
)
reuse_ref_kv = ref_kv_cache is not None and ref_kv_cache.get("kv") is not None
if reuse_ref_kv:
ref_latents = None # the refs are consumed from the cache instead
# Restore the stacked-layer axis flattened in pad_text_features: F -> (n, d).
n = model.config.txtlayers
context = context.reshape(
context.shape[0], context.shape[1], n, context.shape[-1] // n
)
img_tokens, pos, mask = prepare(latents, context.shape[1], patch, text_mask)
reflen = 0
ref_mask = None
if ref_latents is not None and any(len(r) > 0 for r in ref_latents):
ref_tokens, ref_pos, ref_mask = pack_ref_latents(
ref_latents, patch, img_tokens.device, img_tokens.dtype
)
reflen = ref_tokens.shape[1]
img_tokens = torch.cat((img_tokens, ref_tokens), dim=1)
pos = torch.cat((pos, ref_pos), dim=1)
mask = torch.cat((mask, ref_mask), dim=1)
capture = None
if ref_kv_cache is not None and not reuse_ref_kv and reflen > 0:
capture = []
out = model(
img=img_tokens,
context=context,
t=t,
pos=pos,
mask=mask,
reflen=reflen,
isolate_refs=isolate_refs,
ref_kv_capture=capture,
ref_kv_cache=(ref_kv_cache["kv"], ref_kv_cache["mask"])
if reuse_ref_kv
else None,
)
if capture is not None:
ref_kv_cache["kv"] = capture
ref_kv_cache["mask"] = ref_mask
# (B, imglen, c*p*p) -> (B, c, h, w)
velocity = rearrange(
out,
"b (h w) (c ph pw) -> b c (h ph) (w pw)",
ph=patch,
pw=patch,
h=h // patch,
w=w // patch,
)
return velocity
# ---------------------------------------------------------------------------
# Resolution-aware flow-matching timestep schedule.
# ---------------------------------------------------------------------------
def timesteps(
seq_len: int,
steps: int,
x1: float,
x2: float,
y1: float = 0.5,
y2: float = 1.15,
sigma: float = 1.0,
mu: Optional[float] = None,
) -> List[float]:
"""Resolution-aware flow-matching timestep schedule (t: 1 -> 0).
``mu`` is interpolated linearly in image-sequence length between (x1, y1) and
(x2, y2), then used to time-shift a uniform 1->0 grid. Pass an explicit ``mu``
to pin a constant shift regardless of resolution (the distilled turbo
checkpoint was trained at a fixed mu=1.15).
"""
ts = torch.linspace(1, 0, steps + 1)
if mu is None:
slope = (y2 - y1) / (x2 - x1)
mu = slope * seq_len + (y1 - slope * x1)
ts = math.exp(mu) / (math.exp(mu) + (1.0 / ts - 1.0) ** sigma)
return ts.tolist()
# ---------------------------------------------------------------------------
# Minimal sampling pipeline (for training previews).
# ---------------------------------------------------------------------------
class Krea2Pipeline:
"""Lightweight flow-matching sampler used by ai-toolkit's preview generation."""
def __init__(self, model):
# ``model`` is the Krea2Model so we can reuse its encode/decode and config.
self.model = model
@property
def device(self):
return self.model.device_torch
def to(self, *args, **kwargs):
return self
def set_progress_bar_config(self, **kwargs):
pass
@torch.no_grad()
def __call__(
self,
conditional_embeds,
unconditional_embeds,
height: int = 1024,
width: int = 1024,
num_inference_steps: int = 28,
guidance_scale: float = 4.5,
latents: Optional[torch.Tensor] = None,
generator: Optional[torch.Generator] = None,
ref_latents: Optional[List[List[torch.Tensor]]] = None,
**kwargs,
) -> List[Image.Image]:
model = self.model
device = model.device_torch
dtype = model.torch_dtype
transformer: SingleStreamDiT = model.transformer
patch = model.patch_size
ae_scale = model.vae_scale_factor # 8
mkw = model.model_config.model_kwargs
y1 = float(mkw.get("schedule_y1", 0.5))
y2 = float(mkw.get("schedule_y2", 1.15))
minres = int(mkw.get("schedule_min_res", 256))
maxres = int(mkw.get("schedule_max_res", 1280))
mu = mkw.get("schedule_mu", None)
mu = float(mu) if mu is not None else None
do_cfg = guidance_scale > 0 and unconditional_embeds is not None
gh = height // (ae_scale * patch)
gw = width // (ae_scale * patch)
latent_channels = transformer.config.channels
# Starting gaussian noise in the (B, C, h8, w8) latent layout.
if latents is None:
shape = (1, latent_channels, height // ae_scale, width // ae_scale)
latents = randn_tensor(
shape, generator=generator, device=device, dtype=torch.float32
)
latents = latents.to(device, dtype=torch.float32)
cond_feats, cond_mask = pad_text_features(
conditional_embeds.text_embeds, device, dtype
)
if do_cfg:
uncond_feats, uncond_mask = pad_text_features(
unconditional_embeds.text_embeds, device, dtype
)
# min_res / max_res define the (x1,y1)-(x2,y2) interpolation endpoints for mu.
align = ae_scale * patch
x1 = (minres // align) ** 2
x2 = (maxres // align) ** 2
ts = timesteps(gh * gw, num_inference_steps, x1, x2, y1=y1, y2=y2, mu=mu)
# With the kv_cache model kwarg (isolated ref attention) the ref K/V
# are step-invariant, so the very first model call doubles as the
# precompute pass: it runs with the refs in the sequence and fills this
# cache; every later call (including step 1's uncond pass) drops the
# ref tokens and reuses it.
isolate = model.kv_cache
ref_cache = None
if isolate and ref_latents is not None and any(len(r) > 0 for r in ref_latents):
ref_cache = {"kv": None, "mask": None}
# Euler integration of the flow ODE (with optional CFG).
num_steps = len(ts) - 1
for i, (tcurr, tprev) in enumerate(zip(ts[:-1], ts[1:])):
t = torch.full((latents.shape[0],), tcurr, dtype=dtype, device=device)
v_cond = predict_velocity(
transformer,
latents.to(dtype),
t,
cond_feats,
cond_mask,
ref_latents=ref_latents,
isolate_refs=isolate,
ref_kv_cache=ref_cache,
)
if do_cfg:
v_uncond = predict_velocity(
transformer,
latents.to(dtype),
t,
uncond_feats,
uncond_mask,
ref_latents=ref_latents,
isolate_refs=isolate,
ref_kv_cache=ref_cache,
)
v = v_cond + guidance_scale * (v_cond - v_uncond)
else:
v = v_cond
v = v.to(torch.float32)
# inference engine preview: x0 estimate for this step
emit = getattr(model, "_emit_sample_step", None)
if emit is not None and getattr(model, "sample_step_hook", None) is not None:
emit(latents - float(tcurr) * v, i, num_steps)
latents = latents + (tprev - tcurr) * v
images = model.decode_latents(latents, device=device, dtype=dtype)
images = images.float().clamp(-1.0, 1.0)
images = ((images + 1.0) * 127.5).round().to(torch.uint8)
images = images.permute(0, 2, 3, 1).cpu().numpy()
return [Image.fromarray(arr) for arr in images]

Xet Storage Details

Size:
15.8 kB
·
Xet hash:
f9af47bfcfacead893fc72ec9de174111cd73caea15425277fd96006fa5162f2

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.