Instructions to use ivanmikhnenkov/tinydit-256 with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Diffusers
How to use ivanmikhnenkov/tinydit-256 with Diffusers:
pip install -U diffusers transformers accelerate
import torch from diffusers import DiffusionPipeline # switch to "mps" for apple devices pipe = DiffusionPipeline.from_pretrained("ivanmikhnenkov/tinydit-256", dtype=torch.bfloat16, device_map="cuda") prompt = "Astronaut in a jungle, cold color palette, muted colors, detailed, 8k" image = pipe(prompt).images[0] - Notebooks
- Google Colab
- Kaggle
- Local Apps Settings
- Draw Things
- DiffusionBee
Add diffusers format: TinyDiTPipeline + TinyDiTTransformer2DModel (remote code), bundled T5 and FLUX.2 VAE
1c18331 verified Download pipeline_tinydit.py from ivanmikhnenkov/tinydit-256: direct link, hf CLI and curl.
- Browser
- Download file 7.86 kB
-
https://huggingface.co/ivanmikhnenkov/tinydit-256/resolve/main/pipeline_tinydit.py
- Command line
-
hf download hf://ivanmikhnenkov/tinydit-256/pipeline_tinydit.py
-
curl -L -o pipeline_tinydit.py https://huggingface.co/ivanmikhnenkov/tinydit-256/resolve/main/pipeline_tinydit.py
7.86 kB
| """Text-to-image pipeline for tinydit-256: frozen flan-t5-base text encoder, the tinydit transformer, | |
| frozen FLUX.2 autoencoder. Sampling is the training-time sampler: Euler steps on the SD3 timestep shift | |
| (2.8, in the scheduler config) with classifier-free guidance against the empty prompt. | |
| pipe = DiffusionPipeline.from_pretrained("ivanmikhnenkov/tinydit-256", trust_remote_code=True) | |
| image = pipe("a lighthouse on a rocky coast at sunset", width=320, height=208).images[0] | |
| """ | |
| from __future__ import annotations | |
| from typing import Callable | |
| import numpy as np | |
| import torch | |
| from diffusers import DiffusionPipeline | |
| from diffusers.image_processor import VaeImageProcessor | |
| from diffusers.pipelines.pipeline_utils import ImagePipelineOutput | |
| from diffusers.utils.torch_utils import randn_tensor | |
| class TinyDiTPipeline(DiffusionPipeline): | |
| model_cpu_offload_seq = "text_encoder->transformer->vae" | |
| _callback_tensor_inputs = ["latents"] | |
| def __init__(self, tokenizer, text_encoder, transformer, vae, scheduler): | |
| super().__init__() | |
| self.register_modules(tokenizer=tokenizer, text_encoder=text_encoder, transformer=transformer, | |
| vae=vae, scheduler=scheduler) | |
| self.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1) if getattr(self, "vae", None) else 8 | |
| patch = self.transformer.config.patch_size if getattr(self, "transformer", None) else 2 | |
| self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae_scale_factor * patch) | |
| def _embed(self, prompts: list[str], max_sequence_length: int, device): | |
| """-> T5 states (B,L,D), mask (B,L) bool, L = longest prompt in the batch.""" | |
| tok = self.tokenizer(prompts, padding="longest", max_length=max_sequence_length, truncation=True, | |
| return_tensors="pt").to(device) | |
| emb = self.text_encoder(input_ids=tok.input_ids, attention_mask=tok.attention_mask).last_hidden_state | |
| return emb, tok.attention_mask.bool() | |
| def _pad(emb, mask, L): | |
| """Right-pad to length L with masked zeros (padding never reaches cross-attention).""" | |
| B, l, D = emb.shape | |
| if l == L: | |
| return emb, mask | |
| return (torch.cat([emb, emb.new_zeros(B, L - l, D)], 1), | |
| torch.cat([mask, mask.new_zeros(B, L - l)], 1)) | |
| def encode_prompt(self, prompt, negative_prompt="", do_classifier_free_guidance=True, | |
| num_images_per_prompt=1, device=None, max_sequence_length=128): | |
| """-> (prompt_embeds, prompt_mask, negative_embeds, negative_mask), all padded to one length. | |
| The empty negative prompt is the null caption tinydit was trained with (10% caption dropout).""" | |
| device = device or self._execution_device | |
| prompt = [prompt] if isinstance(prompt, str) else list(prompt) | |
| emb, mask = self._embed(prompt, max_sequence_length, device) | |
| neg_emb = neg_mask = None | |
| if do_classifier_free_guidance: | |
| negative_prompt = negative_prompt or "" | |
| negative_prompt = [negative_prompt] * len(prompt) if isinstance(negative_prompt, str) else list(negative_prompt) | |
| if len(negative_prompt) != len(prompt): | |
| raise ValueError(f"got {len(negative_prompt)} negative prompts for {len(prompt)} prompts") | |
| neg_emb, neg_mask = self._embed(negative_prompt, max_sequence_length, device) | |
| L = max(emb.shape[1], neg_emb.shape[1]) | |
| emb, mask = self._pad(emb, mask, L) | |
| neg_emb, neg_mask = self._pad(neg_emb, neg_mask, L) | |
| neg_emb = neg_emb.repeat_interleave(num_images_per_prompt, 0) | |
| neg_mask = neg_mask.repeat_interleave(num_images_per_prompt, 0) | |
| emb = emb.repeat_interleave(num_images_per_prompt, 0) | |
| mask = mask.repeat_interleave(num_images_per_prompt, 0) | |
| return emb, mask, neg_emb, neg_mask | |
| def __call__( | |
| self, | |
| prompt: str | list[str], | |
| negative_prompt: str | list[str] = "", | |
| height: int = 256, | |
| width: int = 256, | |
| num_inference_steps: int = 20, | |
| guidance_scale: float = 4.0, | |
| num_images_per_prompt: int = 1, | |
| generator: torch.Generator | list[torch.Generator] | None = None, | |
| latents: torch.Tensor | None = None, | |
| output_type: str = "pil", | |
| return_dict: bool = True, | |
| max_sequence_length: int = 128, | |
| callback_on_step_end: Callable | None = None, | |
| callback_on_step_end_tensor_inputs: list[str] = ["latents"], | |
| ): | |
| """Trained at ~256x256 pixels in five shapes: 256x256, 288x224, 224x288, 320x208, 208x320 (W x H). | |
| Any multiple of 16 works (2D RoPE), quality drops away from that area. 20 steps is the training-time | |
| sampler, 50 is a little better; the timestep shift matters more than the step count.""" | |
| div = self.vae_scale_factor * self.transformer.config.patch_size | |
| if height % div or width % div: | |
| raise ValueError(f"height and width must be multiples of {div}, got {height}x{width}") | |
| device = self._execution_device | |
| do_cfg = guidance_scale > 1.0 | |
| emb, mask, neg_emb, neg_mask = self.encode_prompt(prompt, negative_prompt, do_cfg, num_images_per_prompt, | |
| device, max_sequence_length) | |
| dtype = self.transformer.dtype | |
| if do_cfg: | |
| emb, mask = torch.cat([neg_emb, emb]), torch.cat([neg_mask, mask]) | |
| emb = emb.to(dtype) | |
| batch = mask.shape[0] // (2 if do_cfg else 1) | |
| shape = (batch, self.transformer.config.in_channels, height // self.vae_scale_factor, width // self.vae_scale_factor) | |
| if latents is None: # the Euler state stays fp32 whatever the transformer dtype, as in training | |
| latents = randn_tensor(shape, generator=generator, device=device, dtype=torch.float32) | |
| else: | |
| latents = latents.to(device=device, dtype=torch.float32) | |
| sigmas = np.linspace(1.0, 1 / num_inference_steps, num_inference_steps) | |
| self.scheduler.set_timesteps(sigmas=sigmas, device=device) | |
| timesteps = self.scheduler.timesteps | |
| self._num_timesteps = len(timesteps) | |
| with self.progress_bar(total=num_inference_steps) as progress_bar: | |
| for i, t in enumerate(timesteps): | |
| x = torch.cat([latents] * 2) if do_cfg else latents | |
| noise_pred = self.transformer(x.to(dtype), t.expand(x.shape[0]), emb, mask, return_dict=False)[0].float() | |
| if do_cfg: | |
| uncond, cond = noise_pred.chunk(2) | |
| noise_pred = uncond + guidance_scale * (cond - uncond) | |
| latents = self.scheduler.step(noise_pred, t, latents, return_dict=False)[0] | |
| if callback_on_step_end is not None: | |
| callback_kwargs = {} | |
| for k in callback_on_step_end_tensor_inputs: | |
| callback_kwargs[k] = locals()[k] | |
| callback_outputs = callback_on_step_end(self, i, t, callback_kwargs) | |
| latents = callback_outputs.pop("latents", latents) | |
| progress_bar.update() | |
| if output_type == "latent": | |
| image = latents | |
| else: | |
| cfg = self.transformer.config | |
| mean = torch.tensor(cfg.latents_mean, device=device).view(1, -1, 1, 1) | |
| std = torch.tensor(cfg.latents_std, device=device).view(1, -1, 1, 1) | |
| z = (latents * std + mean).to(self.vae.dtype) | |
| image = self.vae.decode(z, return_dict=False)[0] | |
| image = self.image_processor.postprocess(image, output_type=output_type) | |
| self.maybe_free_model_hooks() | |
| if not return_dict: | |
| return (image,) | |
| return ImagePipelineOutput(images=image) | |