Text-to-Image
Diffusers
Safetensors
diffusion-transformer
rectified-flow
from-scratch
tinydit-256 / pipeline_tinydit.py
ivanmikhnenkov's picture
Add diffusers format: TinyDiTPipeline + TinyDiTTransformer2DModel (remote code), bundled T5 and FLUX.2 VAE
1c18331 verified
Raw History Blame Contribute Delete
7.86 kB
"""Text-to-image pipeline for tinydit-256: frozen flan-t5-base text encoder, the tinydit transformer,
frozen FLUX.2 autoencoder. Sampling is the training-time sampler: Euler steps on the SD3 timestep shift
(2.8, in the scheduler config) with classifier-free guidance against the empty prompt.
pipe = DiffusionPipeline.from_pretrained("ivanmikhnenkov/tinydit-256", trust_remote_code=True)
image = pipe("a lighthouse on a rocky coast at sunset", width=320, height=208).images[0]
"""
from __future__ import annotations
from typing import Callable
import numpy as np
import torch
from diffusers import DiffusionPipeline
from diffusers.image_processor import VaeImageProcessor
from diffusers.pipelines.pipeline_utils import ImagePipelineOutput
from diffusers.utils.torch_utils import randn_tensor
class TinyDiTPipeline(DiffusionPipeline):
model_cpu_offload_seq = "text_encoder->transformer->vae"
_callback_tensor_inputs = ["latents"]
def __init__(self, tokenizer, text_encoder, transformer, vae, scheduler):
super().__init__()
self.register_modules(tokenizer=tokenizer, text_encoder=text_encoder, transformer=transformer,
vae=vae, scheduler=scheduler)
self.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1) if getattr(self, "vae", None) else 8
patch = self.transformer.config.patch_size if getattr(self, "transformer", None) else 2
self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae_scale_factor * patch)
def _embed(self, prompts: list[str], max_sequence_length: int, device):
"""-> T5 states (B,L,D), mask (B,L) bool, L = longest prompt in the batch."""
tok = self.tokenizer(prompts, padding="longest", max_length=max_sequence_length, truncation=True,
return_tensors="pt").to(device)
emb = self.text_encoder(input_ids=tok.input_ids, attention_mask=tok.attention_mask).last_hidden_state
return emb, tok.attention_mask.bool()
@staticmethod
def _pad(emb, mask, L):
"""Right-pad to length L with masked zeros (padding never reaches cross-attention)."""
B, l, D = emb.shape
if l == L:
return emb, mask
return (torch.cat([emb, emb.new_zeros(B, L - l, D)], 1),
torch.cat([mask, mask.new_zeros(B, L - l)], 1))
def encode_prompt(self, prompt, negative_prompt="", do_classifier_free_guidance=True,
num_images_per_prompt=1, device=None, max_sequence_length=128):
"""-> (prompt_embeds, prompt_mask, negative_embeds, negative_mask), all padded to one length.
The empty negative prompt is the null caption tinydit was trained with (10% caption dropout)."""
device = device or self._execution_device
prompt = [prompt] if isinstance(prompt, str) else list(prompt)
emb, mask = self._embed(prompt, max_sequence_length, device)
neg_emb = neg_mask = None
if do_classifier_free_guidance:
negative_prompt = negative_prompt or ""
negative_prompt = [negative_prompt] * len(prompt) if isinstance(negative_prompt, str) else list(negative_prompt)
if len(negative_prompt) != len(prompt):
raise ValueError(f"got {len(negative_prompt)} negative prompts for {len(prompt)} prompts")
neg_emb, neg_mask = self._embed(negative_prompt, max_sequence_length, device)
L = max(emb.shape[1], neg_emb.shape[1])
emb, mask = self._pad(emb, mask, L)
neg_emb, neg_mask = self._pad(neg_emb, neg_mask, L)
neg_emb = neg_emb.repeat_interleave(num_images_per_prompt, 0)
neg_mask = neg_mask.repeat_interleave(num_images_per_prompt, 0)
emb = emb.repeat_interleave(num_images_per_prompt, 0)
mask = mask.repeat_interleave(num_images_per_prompt, 0)
return emb, mask, neg_emb, neg_mask
@torch.no_grad()
def __call__(
self,
prompt: str | list[str],
negative_prompt: str | list[str] = "",
height: int = 256,
width: int = 256,
num_inference_steps: int = 20,
guidance_scale: float = 4.0,
num_images_per_prompt: int = 1,
generator: torch.Generator | list[torch.Generator] | None = None,
latents: torch.Tensor | None = None,
output_type: str = "pil",
return_dict: bool = True,
max_sequence_length: int = 128,
callback_on_step_end: Callable | None = None,
callback_on_step_end_tensor_inputs: list[str] = ["latents"],
):
"""Trained at ~256x256 pixels in five shapes: 256x256, 288x224, 224x288, 320x208, 208x320 (W x H).
Any multiple of 16 works (2D RoPE), quality drops away from that area. 20 steps is the training-time
sampler, 50 is a little better; the timestep shift matters more than the step count."""
div = self.vae_scale_factor * self.transformer.config.patch_size
if height % div or width % div:
raise ValueError(f"height and width must be multiples of {div}, got {height}x{width}")
device = self._execution_device
do_cfg = guidance_scale > 1.0
emb, mask, neg_emb, neg_mask = self.encode_prompt(prompt, negative_prompt, do_cfg, num_images_per_prompt,
device, max_sequence_length)
dtype = self.transformer.dtype
if do_cfg:
emb, mask = torch.cat([neg_emb, emb]), torch.cat([neg_mask, mask])
emb = emb.to(dtype)
batch = mask.shape[0] // (2 if do_cfg else 1)
shape = (batch, self.transformer.config.in_channels, height // self.vae_scale_factor, width // self.vae_scale_factor)
if latents is None: # the Euler state stays fp32 whatever the transformer dtype, as in training
latents = randn_tensor(shape, generator=generator, device=device, dtype=torch.float32)
else:
latents = latents.to(device=device, dtype=torch.float32)
sigmas = np.linspace(1.0, 1 / num_inference_steps, num_inference_steps)
self.scheduler.set_timesteps(sigmas=sigmas, device=device)
timesteps = self.scheduler.timesteps
self._num_timesteps = len(timesteps)
with self.progress_bar(total=num_inference_steps) as progress_bar:
for i, t in enumerate(timesteps):
x = torch.cat([latents] * 2) if do_cfg else latents
noise_pred = self.transformer(x.to(dtype), t.expand(x.shape[0]), emb, mask, return_dict=False)[0].float()
if do_cfg:
uncond, cond = noise_pred.chunk(2)
noise_pred = uncond + guidance_scale * (cond - uncond)
latents = self.scheduler.step(noise_pred, t, latents, return_dict=False)[0]
if callback_on_step_end is not None:
callback_kwargs = {}
for k in callback_on_step_end_tensor_inputs:
callback_kwargs[k] = locals()[k]
callback_outputs = callback_on_step_end(self, i, t, callback_kwargs)
latents = callback_outputs.pop("latents", latents)
progress_bar.update()
if output_type == "latent":
image = latents
else:
cfg = self.transformer.config
mean = torch.tensor(cfg.latents_mean, device=device).view(1, -1, 1, 1)
std = torch.tensor(cfg.latents_std, device=device).view(1, -1, 1, 1)
z = (latents * std + mean).to(self.vae.dtype)
image = self.vae.decode(z, return_dict=False)[0]
image = self.image_processor.postprocess(image, output_type=output_type)
self.maybe_free_model_hooks()
if not return_dict:
return (image,)
return ImagePipelineOutput(images=image)