Buckets:
| """Packing / sampling helpers for Krea 2. | |
| Turns image latents + stacked Qwen3-VL text features into the single sequence the | |
| ``SingleStreamDiT`` consumes, and provides a minimal flow-matching sampler used to | |
| render preview images during training. | |
| Time convention: Krea 2 is a plain flow-matching model whose time runs ``t=1`` | |
| (pure noise) -> ``t=0`` (clean), the velocity it predicts is ``noise - clean``, | |
| and ``x_t = (1 - t) * clean + t * noise``. This is *identical* to ai-toolkit's | |
| convention, so unlike ideogram4 there is no flipping or negation -- the toolkit | |
| ``timestep / 1000`` flows straight through as ``t``. | |
| """ | |
| from __future__ import annotations | |
| import math | |
| from typing import List, Optional | |
| import torch | |
| from einops import rearrange, repeat | |
| from PIL import Image | |
| from diffusers.utils.torch_utils import randn_tensor | |
| from .mmdit import SingleStreamDiT | |
| # --------------------------------------------------------------------------- | |
| # Text feature padding. | |
| # --------------------------------------------------------------------------- | |
| def pad_text_features( | |
| features_list: List[torch.Tensor], | |
| device: torch.device, | |
| dtype: torch.dtype, | |
| ) -> tuple[torch.Tensor, torch.Tensor]: | |
| """Right-pad a list of per-sample ``(Lt_i, F)`` features into a batch. | |
| Each caption is stored 2D at its natural length -- the 12 stacked Qwen3-VL | |
| hidden-state layers are flattened into the feature axis ``F = n * d`` so the | |
| ai-toolkit batching machinery treats the list length as the batch size (it | |
| only special-cases 2D per-sample tensors). The layer axis is restored in | |
| ``predict_velocity`` right before the MMDiT call. Padding to the batch max is | |
| deferred to here. Returns ``(features (B, Lt, F), mask (B, Lt))``; the mask is | |
| 1 for real text tokens and 0 for padding. | |
| """ | |
| lengths = [f.shape[0] for f in features_list] | |
| max_len = max(lengths) | |
| dim = features_list[0].shape[-1] | |
| batch_size = len(features_list) | |
| features = torch.zeros(batch_size, max_len, dim, device=device, dtype=dtype) | |
| mask = torch.zeros(batch_size, max_len, dtype=torch.long, device=device) | |
| for i, f in enumerate(features_list): | |
| ln = f.shape[0] | |
| features[i, :ln] = f.to(device, dtype) | |
| mask[i, :ln] = 1 | |
| return features, mask | |
| # --------------------------------------------------------------------------- | |
| # Latent <-> token packing and combined position / mask construction. | |
| # --------------------------------------------------------------------------- | |
| def prepare( | |
| img: torch.Tensor, txtlen: int, patch: int, txtmask: torch.Tensor | |
| ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: | |
| """Patchify the latent and build the combined text+image position / mask. | |
| in: img (B, C, h, w) image latent | |
| txtlen number of text tokens | |
| patch transformer patch size | |
| txtmask (B, txtlen) long/bool mask, 1 for real text tokens | |
| out: (img_tokens (B, h/p*w/p, C*p*p), pos (B, txtlen+imglen, 3), | |
| mask (B, txtlen+imglen)) | |
| """ | |
| b, _, h, w = img.shape | |
| h_, w_ = h // patch, w // patch | |
| imgids = torch.zeros((h_, w_, 3), device=img.device) | |
| imgids[..., 1] = torch.arange(h_, device=img.device)[:, None] | |
| imgids[..., 2] = torch.arange(w_, device=img.device)[None, :] | |
| imgpos = repeat(imgids, "h w three -> b (h w) three", b=b, three=3) | |
| imgmask = torch.ones(b, h_ * w_, device=img.device, dtype=torch.bool) | |
| img = rearrange(img, "b c (h ph) (w pw) -> b (h w) (c ph pw)", ph=patch, pw=patch) | |
| txtpos = torch.zeros(b, txtlen, 3, device=img.device) | |
| mask = torch.cat((txtmask.to(img.device).bool(), imgmask), dim=1) | |
| pos = torch.cat((txtpos, imgpos), dim=1) | |
| return img, pos, mask | |
| def pack_ref_latents( | |
| ref_latents: List[List[torch.Tensor]], patch: int, device, dtype | |
| ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: | |
| """Patchify per-sample reference latents into padded ref tokens / pos / mask. | |
| ``ref_latents`` is a list (one entry per batch item) of lists of ``(C, h, w)`` | |
| reference latents. The i-th reference of a sample is placed on RoPE axis 0 at | |
| index ``i + 1`` with its own y/x grid starting at 0 -- the ComfyUI Kontext | |
| "index" placement (axis 0 is otherwise always 0, so the base weights see the | |
| references as a new "frame" axis). Samples with fewer reference tokens are | |
| right-padded and masked out. Returns ``(tokens (B, Lr, C*p*p), | |
| pos (B, Lr, 3), mask (B, Lr))``. | |
| """ | |
| token_dim = None | |
| seqs, ids = [], [] | |
| for refs in ref_latents: | |
| toks, rpos = [], [] | |
| for i, ref in enumerate(refs): | |
| ref = ref.to(device, dtype) | |
| _, h, w = ref.shape | |
| h_, w_ = h // patch, w // patch | |
| toks.append( | |
| rearrange(ref, "c (h ph) (w pw) -> (h w) (c ph pw)", ph=patch, pw=patch) | |
| ) | |
| token_dim = toks[-1].shape[-1] | |
| refids = torch.zeros((h_, w_, 3), device=device) | |
| refids[..., 0] = i + 1 | |
| refids[..., 1] = torch.arange(h_, device=device)[:, None] | |
| refids[..., 2] = torch.arange(w_, device=device)[None, :] | |
| rpos.append(refids.reshape(-1, 3)) | |
| seqs.append(toks) | |
| ids.append(rpos) | |
| b = len(seqs) | |
| # a sample may have no references (its span is fully masked/padded) | |
| seqs = [ | |
| torch.cat(t, dim=0) | |
| if t | |
| else torch.zeros(0, token_dim, device=device, dtype=dtype) | |
| for t in seqs | |
| ] | |
| ids = [torch.cat(p, dim=0) if p else torch.zeros(0, 3, device=device) for p in ids] | |
| max_len = max(s.shape[0] for s in seqs) | |
| tokens = torch.zeros(b, max_len, token_dim, device=device, dtype=dtype) | |
| pos = torch.zeros(b, max_len, 3, device=device) | |
| mask = torch.zeros(b, max_len, device=device, dtype=torch.bool) | |
| for i, (s, p) in enumerate(zip(seqs, ids)): | |
| ln = s.shape[0] | |
| tokens[i, :ln] = s | |
| pos[i, :ln] = p | |
| mask[i, :ln] = True | |
| return tokens, pos, mask | |
| def predict_velocity( | |
| model: SingleStreamDiT, | |
| latents: torch.Tensor, # (B, C, h, w) | |
| t: torch.Tensor, # (B,) flow time in [0, 1] (1 = pure noise) | |
| context: torch.Tensor, # (B, Lt, n*d) flattened stacked Qwen3-VL features | |
| text_mask: torch.Tensor, # (B, Lt) 1 for real text tokens | |
| ref_latents: Optional[List[List[torch.Tensor]]] = None, # per-sample (C, h, w) refs | |
| isolate_refs: bool = False, | |
| ref_kv_cache: Optional[dict] = None, | |
| ) -> torch.Tensor: | |
| """Run the MMDiT on the packed [text | image | refs] sequence. | |
| ``latents`` stay in the unpacked ``(B, C, h, w)`` latent layout; image-token | |
| packing is internal to this function. ``context`` arrives 2D-per-sample | |
| flattened ``(B, Lt, n*d)`` and is restored to ``(B, Lt, n, d)`` for the MMDiT. | |
| ``ref_latents`` (optional) are clean reference latents appended after the | |
| image tokens and conditioned at t=0 ("index_timestep_zero"); the prediction | |
| only ever covers the noisy target tokens. ``isolate_refs`` restricts ref | |
| tokens to attending only among themselves (see ``SingleStreamDiT.forward``), | |
| making their per-layer K/V cacheable across steps. ``ref_kv_cache`` is a | |
| ``{"kv": None, "mask": None}`` dict enabling that cache (inference only, | |
| requires ``isolate_refs``): while ``kv`` is unset the call runs the refs | |
| normally and fills the dict with each block's ref K/V; once filled, the ref | |
| tokens are dropped from the sequence and the cached K/V are injected as | |
| extra attention keys -- identical math, no ref recompute per step. Returns | |
| the velocity ``noise - clean`` reshaped back to ``(B, C, h, w)``. No time | |
| flip / negation: Krea's convention matches toolkit's. | |
| """ | |
| patch = model.config.patch | |
| b, c, h, w = latents.shape | |
| if ref_kv_cache is not None and not isolate_refs: | |
| raise ValueError( | |
| "ref_kv_cache requires isolate_refs: cached ref K/V are only " | |
| "step-invariant when ref tokens attend solely to each other" | |
| ) | |
| reuse_ref_kv = ref_kv_cache is not None and ref_kv_cache.get("kv") is not None | |
| if reuse_ref_kv: | |
| ref_latents = None # the refs are consumed from the cache instead | |
| # Restore the stacked-layer axis flattened in pad_text_features: F -> (n, d). | |
| n = model.config.txtlayers | |
| context = context.reshape( | |
| context.shape[0], context.shape[1], n, context.shape[-1] // n | |
| ) | |
| img_tokens, pos, mask = prepare(latents, context.shape[1], patch, text_mask) | |
| reflen = 0 | |
| ref_mask = None | |
| if ref_latents is not None and any(len(r) > 0 for r in ref_latents): | |
| ref_tokens, ref_pos, ref_mask = pack_ref_latents( | |
| ref_latents, patch, img_tokens.device, img_tokens.dtype | |
| ) | |
| reflen = ref_tokens.shape[1] | |
| img_tokens = torch.cat((img_tokens, ref_tokens), dim=1) | |
| pos = torch.cat((pos, ref_pos), dim=1) | |
| mask = torch.cat((mask, ref_mask), dim=1) | |
| capture = None | |
| if ref_kv_cache is not None and not reuse_ref_kv and reflen > 0: | |
| capture = [] | |
| out = model( | |
| img=img_tokens, | |
| context=context, | |
| t=t, | |
| pos=pos, | |
| mask=mask, | |
| reflen=reflen, | |
| isolate_refs=isolate_refs, | |
| ref_kv_capture=capture, | |
| ref_kv_cache=(ref_kv_cache["kv"], ref_kv_cache["mask"]) | |
| if reuse_ref_kv | |
| else None, | |
| ) | |
| if capture is not None: | |
| ref_kv_cache["kv"] = capture | |
| ref_kv_cache["mask"] = ref_mask | |
| # (B, imglen, c*p*p) -> (B, c, h, w) | |
| velocity = rearrange( | |
| out, | |
| "b (h w) (c ph pw) -> b c (h ph) (w pw)", | |
| ph=patch, | |
| pw=patch, | |
| h=h // patch, | |
| w=w // patch, | |
| ) | |
| return velocity | |
| # --------------------------------------------------------------------------- | |
| # Resolution-aware flow-matching timestep schedule. | |
| # --------------------------------------------------------------------------- | |
| def timesteps( | |
| seq_len: int, | |
| steps: int, | |
| x1: float, | |
| x2: float, | |
| y1: float = 0.5, | |
| y2: float = 1.15, | |
| sigma: float = 1.0, | |
| mu: Optional[float] = None, | |
| ) -> List[float]: | |
| """Resolution-aware flow-matching timestep schedule (t: 1 -> 0). | |
| ``mu`` is interpolated linearly in image-sequence length between (x1, y1) and | |
| (x2, y2), then used to time-shift a uniform 1->0 grid. Pass an explicit ``mu`` | |
| to pin a constant shift regardless of resolution (the distilled turbo | |
| checkpoint was trained at a fixed mu=1.15). | |
| """ | |
| ts = torch.linspace(1, 0, steps + 1) | |
| if mu is None: | |
| slope = (y2 - y1) / (x2 - x1) | |
| mu = slope * seq_len + (y1 - slope * x1) | |
| ts = math.exp(mu) / (math.exp(mu) + (1.0 / ts - 1.0) ** sigma) | |
| return ts.tolist() | |
| # --------------------------------------------------------------------------- | |
| # Minimal sampling pipeline (for training previews). | |
| # --------------------------------------------------------------------------- | |
| class Krea2Pipeline: | |
| """Lightweight flow-matching sampler used by ai-toolkit's preview generation.""" | |
| def __init__(self, model): | |
| # ``model`` is the Krea2Model so we can reuse its encode/decode and config. | |
| self.model = model | |
| def device(self): | |
| return self.model.device_torch | |
| def to(self, *args, **kwargs): | |
| return self | |
| def set_progress_bar_config(self, **kwargs): | |
| pass | |
| def __call__( | |
| self, | |
| conditional_embeds, | |
| unconditional_embeds, | |
| height: int = 1024, | |
| width: int = 1024, | |
| num_inference_steps: int = 28, | |
| guidance_scale: float = 4.5, | |
| latents: Optional[torch.Tensor] = None, | |
| generator: Optional[torch.Generator] = None, | |
| ref_latents: Optional[List[List[torch.Tensor]]] = None, | |
| **kwargs, | |
| ) -> List[Image.Image]: | |
| model = self.model | |
| device = model.device_torch | |
| dtype = model.torch_dtype | |
| transformer: SingleStreamDiT = model.transformer | |
| patch = model.patch_size | |
| ae_scale = model.vae_scale_factor # 8 | |
| mkw = model.model_config.model_kwargs | |
| y1 = float(mkw.get("schedule_y1", 0.5)) | |
| y2 = float(mkw.get("schedule_y2", 1.15)) | |
| minres = int(mkw.get("schedule_min_res", 256)) | |
| maxres = int(mkw.get("schedule_max_res", 1280)) | |
| mu = mkw.get("schedule_mu", None) | |
| mu = float(mu) if mu is not None else None | |
| do_cfg = guidance_scale > 0 and unconditional_embeds is not None | |
| gh = height // (ae_scale * patch) | |
| gw = width // (ae_scale * patch) | |
| latent_channels = transformer.config.channels | |
| # Starting gaussian noise in the (B, C, h8, w8) latent layout. | |
| if latents is None: | |
| shape = (1, latent_channels, height // ae_scale, width // ae_scale) | |
| latents = randn_tensor( | |
| shape, generator=generator, device=device, dtype=torch.float32 | |
| ) | |
| latents = latents.to(device, dtype=torch.float32) | |
| cond_feats, cond_mask = pad_text_features( | |
| conditional_embeds.text_embeds, device, dtype | |
| ) | |
| if do_cfg: | |
| uncond_feats, uncond_mask = pad_text_features( | |
| unconditional_embeds.text_embeds, device, dtype | |
| ) | |
| # min_res / max_res define the (x1,y1)-(x2,y2) interpolation endpoints for mu. | |
| align = ae_scale * patch | |
| x1 = (minres // align) ** 2 | |
| x2 = (maxres // align) ** 2 | |
| ts = timesteps(gh * gw, num_inference_steps, x1, x2, y1=y1, y2=y2, mu=mu) | |
| # With the kv_cache model kwarg (isolated ref attention) the ref K/V | |
| # are step-invariant, so the very first model call doubles as the | |
| # precompute pass: it runs with the refs in the sequence and fills this | |
| # cache; every later call (including step 1's uncond pass) drops the | |
| # ref tokens and reuses it. | |
| isolate = model.kv_cache | |
| ref_cache = None | |
| if isolate and ref_latents is not None and any(len(r) > 0 for r in ref_latents): | |
| ref_cache = {"kv": None, "mask": None} | |
| # Euler integration of the flow ODE (with optional CFG). | |
| num_steps = len(ts) - 1 | |
| for i, (tcurr, tprev) in enumerate(zip(ts[:-1], ts[1:])): | |
| t = torch.full((latents.shape[0],), tcurr, dtype=dtype, device=device) | |
| v_cond = predict_velocity( | |
| transformer, | |
| latents.to(dtype), | |
| t, | |
| cond_feats, | |
| cond_mask, | |
| ref_latents=ref_latents, | |
| isolate_refs=isolate, | |
| ref_kv_cache=ref_cache, | |
| ) | |
| if do_cfg: | |
| v_uncond = predict_velocity( | |
| transformer, | |
| latents.to(dtype), | |
| t, | |
| uncond_feats, | |
| uncond_mask, | |
| ref_latents=ref_latents, | |
| isolate_refs=isolate, | |
| ref_kv_cache=ref_cache, | |
| ) | |
| v = v_cond + guidance_scale * (v_cond - v_uncond) | |
| else: | |
| v = v_cond | |
| v = v.to(torch.float32) | |
| # inference engine preview: x0 estimate for this step | |
| emit = getattr(model, "_emit_sample_step", None) | |
| if emit is not None and getattr(model, "sample_step_hook", None) is not None: | |
| emit(latents - float(tcurr) * v, i, num_steps) | |
| latents = latents + (tprev - tcurr) * v | |
| images = model.decode_latents(latents, device=device, dtype=dtype) | |
| images = images.float().clamp(-1.0, 1.0) | |
| images = ((images + 1.0) * 127.5).round().to(torch.uint8) | |
| images = images.permute(0, 2, 3, 1).cpu().numpy() | |
| return [Image.fromarray(arr) for arr in images] | |
Xet Storage Details
- Size:
- 15.8 kB
- Xet hash:
- f9af47bfcfacead893fc72ec9de174111cd73caea15425277fd96006fa5162f2
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.