LiliPixel-V1-190M / pipeline.py
Pinkstack's picture
Upload 2 files
ccfd954 verified
Raw History Blame Contribute Delete
16.8 kB
import json
import os
import torch
from pathlib import Path
from PIL import Image
import math
import numpy as np
import torch
import torch.nn.functional as F
import torch.utils.checkpoint
from torch import nn
from PIL import Image
SIZES = (16, 32, 64)
def timestep_embedding(t, dim, max_period=10000):
half = dim // 2
freqs = torch.exp(-math.log(max_period) * torch.arange(half, device=t.device, dtype=torch.float32) / half)
args = t.float()[:, None] * freqs[None, :]
return torch.cat([torch.cos(args), torch.sin(args)], dim=-1)
class RMSNorm(nn.Module):
def __init__(self, dim, eps=1e-6):
super().__init__()
self.weight = nn.Parameter(torch.ones(dim))
self.eps = eps
def forward(self, x):
dtype = x.dtype
x = x.float()
x = x * torch.rsqrt(x.pow(2).mean(-1, keepdim=True) + self.eps)
return x.to(dtype) * self.weight
def _rope_angles(pos, n_freq, theta=10000.0):
inv = 1.0 / (theta ** (torch.arange(n_freq, device=pos.device, dtype=torch.float32) / n_freq))
return pos.float()[:, None] * inv[None, :]
def rotate_half(x):
a, b = x.chunk(2, dim=-1)
return torch.cat([-b, a], dim=-1)
_ROPE_CACHE = {}
def build_rope(n, text_len, head_dim, device, p=1):
# image token positions are in PIXEL coordinates (patch centres), so patch 1 and patch 2 share one coordinate system
key = (n, text_len, head_dim, str(device), p)
if key not in _ROPE_CACHE:
n_freq = head_dim // 4
coords = torch.arange(n, device=device, dtype=torch.float32) * p + (p - 1) / 2.0
ys, xs = torch.meshgrid(coords, coords, indexing="ij")
txt_y = -(torch.arange(text_len, device=device, dtype=torch.float32) + 1.0)
pos_y = torch.cat([txt_y, ys.reshape(-1)])
pos_x = torch.cat([torch.zeros(text_len, device=device), xs.reshape(-1)])
ang = torch.cat([_rope_angles(pos_y, n_freq), _rope_angles(pos_x, n_freq)], dim=-1)
_ROPE_CACHE[key] = (torch.cat([ang.cos(), ang.cos()], dim=-1),
torch.cat([ang.sin(), ang.sin()], dim=-1))
return _ROPE_CACHE[key]
class DiTBlock(nn.Module):
def __init__(self, dim, n_heads, mlp_hidden):
super().__init__()
self.n_heads = n_heads
self.head_dim = dim // n_heads
self.norm1 = RMSNorm(dim)
self.qkv = nn.Linear(dim, dim * 3, bias=False)
self.proj_out = nn.Linear(dim, dim, bias=False)
self.norm2 = RMSNorm(dim)
self.fc1 = nn.Linear(dim, mlp_hidden, bias=False)
self.fc3 = nn.Linear(dim, mlp_hidden, bias=False)
self.fc2 = nn.Linear(mlp_hidden, dim, bias=False)
self.table = nn.Parameter(torch.zeros(6, dim))
def forward(self, x, cos, sin, mod):
s1, sc1, g1, s2, sc2, g2 = [m + self.table[i].to(m.dtype) for i, m in enumerate(mod)]
h = self.norm1(x) * (1 + sc1) + s1
B, S, _ = h.shape
qkv = self.qkv(h).view(B, S, 3, self.n_heads, self.head_dim).permute(2, 0, 3, 1, 4)
q, k, v = qkv.unbind(0)
qf, kf = q.float(), k.float()
q = (qf * cos + rotate_half(qf) * sin).to(v.dtype)
k = (kf * cos + rotate_half(kf) * sin).to(v.dtype)
o = F.scaled_dot_product_attention(q, k, v)
o = o.transpose(1, 2).reshape(B, S, -1)
x = x + self.proj_out(o) * g1
h = self.norm2(x) * (1 + sc2) + s2
x = x + self.fc2(F.silu(self.fc1(h)) * self.fc3(h)) * g2
return x
class PixelArtDiT(nn.Module):
def __init__(self, dim=768, n_layers=18, n_heads=12, mlp_ratio=3.0,
patch_small=1, patch_large=2, large_from=64, in_channels=4,
text_len=64, enc_dim=2048):
super().__init__()
self.dim = dim
self.n_heads = n_heads
self.patch_small = patch_small
self.patch_large = patch_large
self.large_from = large_from
self.in_channels = in_channels
self.text_len = text_len
self.enc_dim = enc_dim
patch_set = sorted({patch_small, patch_large})
# one patch-in / patch-out layer per patch size; the transformer body is shared
self.x_projs = nn.ModuleDict({str(p): nn.Linear(in_channels * p * p, dim, bias=False) for p in patch_set})
self.outs = nn.ModuleDict({str(p): nn.Linear(dim, in_channels * p * p) for p in patch_set})
self.res_emb = nn.Embedding(len(SIZES), dim) # which resolution regime (16 / 32 / 64)
# frozen-encoder features -> model width
self.txt_norm = RMSNorm(enc_dim)
self.txt_proj = nn.Linear(enc_dim, dim, bias=False)
self.pad_emb = nn.Parameter(torch.randn(dim) * 0.02) # replaces padded positions
self.null_txt = nn.Parameter(torch.randn(text_len, dim) * 0.02) # learned "no caption" (CFG)
self.time_mlp = nn.Sequential(nn.Linear(dim, dim), nn.SiLU(), nn.Linear(dim, dim))
self.mod = nn.Linear(dim, 6 * dim)
self.blocks = nn.ModuleList(
[DiTBlock(dim, n_heads, int(dim * mlp_ratio)) for _ in range(n_layers)]
)
self.norm_out = RMSNorm(dim)
self.grad_ckpt = False
nn.init.normal_(self.res_emb.weight, std=0.02)
nn.init.zeros_(self.mod.weight)
nn.init.zeros_(self.mod.bias)
for o in self.outs.values():
nn.init.zeros_(o.weight)
nn.init.zeros_(o.bias)
def patch_for(self, size):
return self.patch_large if size >= self.large_from else self.patch_small
@staticmethod
def res_index(size):
if size not in SIZES:
raise ValueError(f"unsupported image size {size}; supported sizes: {SIZES}")
return SIZES.index(size)
def patchify(self, x, p):
B, C, H, W = x.shape
nh, nw = H // p, W // p
x = x.reshape(B, C, nh, p, nw, p).permute(0, 2, 4, 1, 3, 5)
return x.reshape(B, nh * nw, C * p * p), nh, nw
def unpatchify(self, x, nh, nw, p):
B = x.shape[0]
x = x.reshape(B, nh, nw, self.in_channels, p, p).permute(0, 3, 1, 4, 2, 5)
return x.reshape(B, self.in_channels, nh * p, nw * p)
def forward(self, x, t, txt, txt_mask, drop=None):
# txt: (B, text_len, enc_dim) encoder features | txt_mask: (B, text_len) bool
# drop: optional (B,) bool -> replace that sample's caption with the learned null caption
B, _, H, W = x.shape
p = self.patch_for(H)
proj = self.x_projs[str(p)]
img, nh, nw = self.patchify(x, p)
img = proj(img.to(proj.weight.dtype))
txt = self.txt_proj(self.txt_norm(txt.to(self.txt_proj.weight.dtype)))
mask = txt_mask.bool()
txt = torch.where(mask.unsqueeze(-1), txt, self.pad_emb)
if drop is not None:
d = drop.bool()
txt = torch.where(d.view(B, 1, 1), self.null_txt.unsqueeze(0), txt)
mask = torch.where(d.view(B, 1), torch.ones_like(mask), mask)
seq = torch.cat([txt, img], dim=1)
t_emb = timestep_embedding(t * 1000.0, self.dim).to(self.time_mlp[0].weight.dtype)
keep = mask.unsqueeze(-1).to(txt.dtype)
pooled = (txt * keep).sum(dim=1) / keep.sum(dim=1).clamp(min=1.0)
cond = self.time_mlp(t_emb) + pooled + self.res_emb.weight[self.res_index(H)]
mod = self.mod(F.silu(cond)).view(B, 6, 1, self.dim).unbind(1)
cos, sin = build_rope(nh, self.text_len, self.dim // self.n_heads, x.device, p)
h = seq
for blk in self.blocks:
if self.grad_ckpt and self.training:
h = torch.utils.checkpoint.checkpoint(blk, h, cos, sin, mod, use_reentrant=False)
else:
h = blk(h, cos, sin, mod)
h = self.norm_out(h)[:, self.text_len:]
return self.unpatchify(self.outs[str(p)](h), nh, nw, p)
def load_text_encoder(model_id, device="cuda", dtype=torch.bfloat16):
from transformers import AutoModel, AutoTokenizer
tokenizer = AutoTokenizer.from_pretrained(model_id)
if tokenizer.pad_token is None:
tokenizer.pad_token = tokenizer.eos_token
last_err = None
for kw in ({"dtype": dtype, "attn_implementation": "sdpa"},
{"torch_dtype": dtype, "attn_implementation": "sdpa"},
{"dtype": dtype},
{"torch_dtype": dtype}):
try:
enc = AutoModel.from_pretrained(model_id, trust_remote_code=True, **kw)
break
except (TypeError, ValueError, ImportError) as e:
last_err = e
else:
raise last_err
enc = enc.to(device).eval().requires_grad_(False)
return tokenizer, enc
@torch.no_grad()
def encode_prompts(tokenizer, text_encoder, prompts, text_len, device):
prompts = [p if isinstance(p, str) and p.strip() else "." for p in prompts]
tok = tokenizer(prompts, padding="max_length", truncation=True, max_length=text_len,
return_tensors="pt")
ids = tok["input_ids"].to(device)
mask = tok["attention_mask"].to(device)
out = text_encoder(input_ids=ids, attention_mask=mask)
feats = out.last_hidden_state if hasattr(out, "last_hidden_state") else out[0]
return feats, mask.bool()
def decode_latents(x):
x = x.float().clamp(-1, 1).add(1).mul(127.5).round().to(torch.uint8)
return [Image.fromarray(np.ascontiguousarray(arr.transpose(1, 2, 0))) for arr in x.cpu().numpy()]
def sprite_grid(images, scale=8, cols=None, bg=(40, 40, 40, 255)):
cols = cols or int(math.ceil(math.sqrt(len(images))))
rows = math.ceil(len(images) / cols)
w, h = images[0].size
grid = Image.new("RGBA", (cols * w, rows * h), bg)
for i, im in enumerate(images):
grid.alpha_composite(im.convert("RGBA"), dest=((i % cols) * w, (i // cols) * h))
return grid.resize((cols * w * scale, rows * h * scale), Image.NEAREST)
@torch.no_grad()
def generate(model, prompts, encode_fn, size=32, steps=30, guidance=3.0, seed=0,
device="cuda", num_per_prompt=1):
# encode_fn(list[str]) -> (features, bool mask)
was_training = model.training
model.eval()
dev = torch.device(device)
gen = torch.Generator(device=dev).manual_seed(int(seed))
if isinstance(prompts, str):
prompts = [prompts]
expanded = [p for p in prompts for _ in range(num_per_prompt)]
B = len(expanded)
feats, mask = encode_fn(expanded)
use_cfg = guidance != 1.0
if use_cfg:
feats2 = torch.cat([feats, feats])
mask2 = torch.cat([mask, mask])
drop2 = torch.cat([torch.zeros(B, dtype=torch.bool, device=dev),
torch.ones(B, dtype=torch.bool, device=dev)])
x = torch.randn(B, model.in_channels, size, size, device=dev, generator=gen)
ts = torch.linspace(1.0, 0.0, steps + 1, device=dev)
for i in range(steps):
if use_cfg:
xi, f, m, d, t = torch.cat([x, x]), feats2, mask2, drop2, ts[i].expand(2 * B)
else:
xi, f, m, d, t = x, feats, mask, None, ts[i].expand(B)
with torch.autocast(device_type=dev.type, dtype=torch.bfloat16, enabled=dev.type == "cuda"):
v = model(xi, t, f, m, d)
v = v.float()
if use_cfg:
vc, vu = v.chunk(2)
v = vu + guidance * (vc - vu)
x = x + (ts[i + 1] - ts[i]) * v
model.train(was_training)
return decode_latents(x)
import json
import os
from pathlib import Path
try:
from diffusers import DiffusionPipeline
from diffusers.configuration_utils import ConfigMixin, register_to_config
from diffusers.models.modeling_utils import ModelMixin
from diffusers.pipelines.pipeline_utils import ImagePipelineOutput
from diffusers.utils.torch_utils import randn_tensor
_HAS_DIFFUSERS = True
except ImportError:
_HAS_DIFFUSERS = False
if _HAS_DIFFUSERS:
class PixelArtDiTModel(ModelMixin, ConfigMixin):
config_name = "config.json"
@register_to_config
def __init__(self, dim=768, n_layers=18, n_heads=12, mlp_ratio=3.0,
patch_small=1, patch_large=2, large_from=64, in_channels=4,
text_len=64, enc_dim=2048, text_encoder_id="MongoDB/voyage-4-nano"):
super().__init__()
self.inner = PixelArtDiT(dim=dim, n_layers=n_layers, n_heads=n_heads,
mlp_ratio=mlp_ratio, patch_small=patch_small,
patch_large=patch_large, large_from=large_from,
in_channels=in_channels, text_len=text_len,
enc_dim=enc_dim)
def forward(self, x, t, txt, txt_mask, drop=None):
return self.inner(x, t, txt, txt_mask, drop)
@property
def text_len(self):
return self.inner.text_len
@property
def in_channels(self):
return self.inner.in_channels
class PixelArtDiTPipeline(DiffusionPipeline):
def __init__(self, transformer, scheduler):
super().__init__()
self.register_modules(transformer=transformer, scheduler=scheduler)
def _get_text_encoder(self, device):
# frozen text encoder is downloaded from the Hub id stored in transformer/config.json
if getattr(self, "_text_encoder", None) is None:
self._tokenizer, self._text_encoder = load_text_encoder(
self.transformer.config.text_encoder_id, device=device)
else:
self._text_encoder.to(device)
return self._tokenizer, self._text_encoder
@torch.no_grad()
def __call__(self, prompt, height=32, width=32, num_inference_steps=30,
guidance_scale=3.0, num_images_per_prompt=1, generator=None,
output_type="pil", return_dict=True):
if height != width or height not in (16, 32, 64):
raise ValueError("PixelArtDiT supports square images of size 16, 32 or 64.")
if isinstance(prompt, str):
prompt = [prompt]
prompts = [p for p in prompt for _ in range(num_images_per_prompt)]
B = len(prompts)
device = self._execution_device
model = self.transformer
dtype = model.dtype
tokenizer, text_encoder = self._get_text_encoder(device)
feats, mask = encode_prompts(tokenizer, text_encoder, prompts, model.text_len, device)
use_cfg = guidance_scale != 1.0
if use_cfg:
feats_in = torch.cat([feats, feats])
mask_in = torch.cat([mask, mask])
drop = torch.cat([torch.zeros(B, dtype=torch.bool, device=device),
torch.ones(B, dtype=torch.bool, device=device)])
else:
feats_in, mask_in, drop = feats, mask, None
self.scheduler.set_timesteps(num_inference_steps, device=device)
latents = randn_tensor((B, model.in_channels, height, width), generator=generator,
device=device, dtype=torch.float32)
for i, t in enumerate(self.scheduler.timesteps):
sigma = self.scheduler.sigmas[i].to(device=device, dtype=torch.float32)
x_in = torch.cat([latents, latents]) if use_cfg else latents
t_in = sigma.expand(x_in.shape[0])
v = model(x_in.to(dtype), t_in, feats_in, mask_in, drop).float()
if use_cfg:
vc, vu = v.chunk(2)
v = vu + guidance_scale * (vc - vu)
latents = self.scheduler.step(v, t, latents).prev_sample
if output_type == "latent":
out = latents
elif output_type == "np":
out = ((latents.clamp(-1, 1) + 1) / 2).permute(0, 2, 3, 1).cpu().numpy()
else:
out = decode_latents(latents)
if not return_dict:
return (out,)
return ImagePipelineOutput(images=out)
else:
PixelArtDiTModel = PixelArtDiT
def load_pixel_art_dit(repo_or_path, device="cuda", dtype=torch.bfloat16):
from safetensors.torch import load_file
if os.path.isdir(repo_or_path):
root = Path(repo_or_path)
else:
from huggingface_hub import snapshot_download
root = Path(snapshot_download(repo_or_path))
with open(root / "transformer" / "config.json") as f:
cfg = json.load(f)
cfg = {k: v for k, v in cfg.items() if not k.startswith("_") and k != "text_encoder_id"}
model = PixelArtDiT(**cfg)
sd = load_file(root / "transformer" / "diffusion_pytorch_model.safetensors")
sd = {k[len("inner."):] if k.startswith("inner.") else k: v for k, v in sd.items()}
model.load_state_dict(sd)
return model.to(device=device, dtype=dtype).eval()