Pinkstack commited on
Commit
ccfd954
·
verified ·
1 Parent(s): accd49b

Upload 2 files

Browse files
Files changed (2) hide show
  1. model_index.json +15 -0
  2. pipeline.py +399 -0
model_index.json ADDED
@@ -0,0 +1,15 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_class_name": [
3
+ "pipeline",
4
+ "PixelArtDiTPipeline"
5
+ ],
6
+ "_diffusers_version": "0.32.0",
7
+ "scheduler": [
8
+ "diffusers",
9
+ "FlowMatchEulerDiscreteScheduler"
10
+ ],
11
+ "transformer": [
12
+ "pipeline",
13
+ "PixelArtDiTModel"
14
+ ]
15
+ }
pipeline.py ADDED
@@ -0,0 +1,399 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import json
2
+ import os
3
+
4
+ import torch
5
+ from pathlib import Path
6
+ from PIL import Image
7
+
8
+
9
+ import math
10
+
11
+ import numpy as np
12
+ import torch
13
+ import torch.nn.functional as F
14
+ import torch.utils.checkpoint
15
+ from torch import nn
16
+ from PIL import Image
17
+
18
+ SIZES = (16, 32, 64)
19
+
20
+
21
+ def timestep_embedding(t, dim, max_period=10000):
22
+ half = dim // 2
23
+ freqs = torch.exp(-math.log(max_period) * torch.arange(half, device=t.device, dtype=torch.float32) / half)
24
+ args = t.float()[:, None] * freqs[None, :]
25
+ return torch.cat([torch.cos(args), torch.sin(args)], dim=-1)
26
+
27
+
28
+ class RMSNorm(nn.Module):
29
+ def __init__(self, dim, eps=1e-6):
30
+ super().__init__()
31
+ self.weight = nn.Parameter(torch.ones(dim))
32
+ self.eps = eps
33
+
34
+ def forward(self, x):
35
+ dtype = x.dtype
36
+ x = x.float()
37
+ x = x * torch.rsqrt(x.pow(2).mean(-1, keepdim=True) + self.eps)
38
+ return x.to(dtype) * self.weight
39
+
40
+
41
+ def _rope_angles(pos, n_freq, theta=10000.0):
42
+ inv = 1.0 / (theta ** (torch.arange(n_freq, device=pos.device, dtype=torch.float32) / n_freq))
43
+ return pos.float()[:, None] * inv[None, :]
44
+
45
+
46
+ def rotate_half(x):
47
+ a, b = x.chunk(2, dim=-1)
48
+ return torch.cat([-b, a], dim=-1)
49
+
50
+
51
+ _ROPE_CACHE = {}
52
+
53
+
54
+ def build_rope(n, text_len, head_dim, device, p=1):
55
+ # image token positions are in PIXEL coordinates (patch centres), so patch 1 and patch 2 share one coordinate system
56
+ key = (n, text_len, head_dim, str(device), p)
57
+ if key not in _ROPE_CACHE:
58
+ n_freq = head_dim // 4
59
+ coords = torch.arange(n, device=device, dtype=torch.float32) * p + (p - 1) / 2.0
60
+ ys, xs = torch.meshgrid(coords, coords, indexing="ij")
61
+ txt_y = -(torch.arange(text_len, device=device, dtype=torch.float32) + 1.0)
62
+ pos_y = torch.cat([txt_y, ys.reshape(-1)])
63
+ pos_x = torch.cat([torch.zeros(text_len, device=device), xs.reshape(-1)])
64
+ ang = torch.cat([_rope_angles(pos_y, n_freq), _rope_angles(pos_x, n_freq)], dim=-1)
65
+ _ROPE_CACHE[key] = (torch.cat([ang.cos(), ang.cos()], dim=-1),
66
+ torch.cat([ang.sin(), ang.sin()], dim=-1))
67
+ return _ROPE_CACHE[key]
68
+
69
+
70
+ class DiTBlock(nn.Module):
71
+ def __init__(self, dim, n_heads, mlp_hidden):
72
+ super().__init__()
73
+ self.n_heads = n_heads
74
+ self.head_dim = dim // n_heads
75
+ self.norm1 = RMSNorm(dim)
76
+ self.qkv = nn.Linear(dim, dim * 3, bias=False)
77
+ self.proj_out = nn.Linear(dim, dim, bias=False)
78
+ self.norm2 = RMSNorm(dim)
79
+ self.fc1 = nn.Linear(dim, mlp_hidden, bias=False)
80
+ self.fc3 = nn.Linear(dim, mlp_hidden, bias=False)
81
+ self.fc2 = nn.Linear(mlp_hidden, dim, bias=False)
82
+ self.table = nn.Parameter(torch.zeros(6, dim))
83
+
84
+ def forward(self, x, cos, sin, mod):
85
+ s1, sc1, g1, s2, sc2, g2 = [m + self.table[i].to(m.dtype) for i, m in enumerate(mod)]
86
+ h = self.norm1(x) * (1 + sc1) + s1
87
+ B, S, _ = h.shape
88
+ qkv = self.qkv(h).view(B, S, 3, self.n_heads, self.head_dim).permute(2, 0, 3, 1, 4)
89
+ q, k, v = qkv.unbind(0)
90
+ qf, kf = q.float(), k.float()
91
+ q = (qf * cos + rotate_half(qf) * sin).to(v.dtype)
92
+ k = (kf * cos + rotate_half(kf) * sin).to(v.dtype)
93
+ o = F.scaled_dot_product_attention(q, k, v)
94
+ o = o.transpose(1, 2).reshape(B, S, -1)
95
+ x = x + self.proj_out(o) * g1
96
+ h = self.norm2(x) * (1 + sc2) + s2
97
+ x = x + self.fc2(F.silu(self.fc1(h)) * self.fc3(h)) * g2
98
+ return x
99
+
100
+
101
+ class PixelArtDiT(nn.Module):
102
+ def __init__(self, dim=768, n_layers=18, n_heads=12, mlp_ratio=3.0,
103
+ patch_small=1, patch_large=2, large_from=64, in_channels=4,
104
+ text_len=64, enc_dim=2048):
105
+ super().__init__()
106
+ self.dim = dim
107
+ self.n_heads = n_heads
108
+ self.patch_small = patch_small
109
+ self.patch_large = patch_large
110
+ self.large_from = large_from
111
+ self.in_channels = in_channels
112
+ self.text_len = text_len
113
+ self.enc_dim = enc_dim
114
+ patch_set = sorted({patch_small, patch_large})
115
+ # one patch-in / patch-out layer per patch size; the transformer body is shared
116
+ self.x_projs = nn.ModuleDict({str(p): nn.Linear(in_channels * p * p, dim, bias=False) for p in patch_set})
117
+ self.outs = nn.ModuleDict({str(p): nn.Linear(dim, in_channels * p * p) for p in patch_set})
118
+ self.res_emb = nn.Embedding(len(SIZES), dim) # which resolution regime (16 / 32 / 64)
119
+ # frozen-encoder features -> model width
120
+ self.txt_norm = RMSNorm(enc_dim)
121
+ self.txt_proj = nn.Linear(enc_dim, dim, bias=False)
122
+ self.pad_emb = nn.Parameter(torch.randn(dim) * 0.02) # replaces padded positions
123
+ self.null_txt = nn.Parameter(torch.randn(text_len, dim) * 0.02) # learned "no caption" (CFG)
124
+ self.time_mlp = nn.Sequential(nn.Linear(dim, dim), nn.SiLU(), nn.Linear(dim, dim))
125
+ self.mod = nn.Linear(dim, 6 * dim)
126
+ self.blocks = nn.ModuleList(
127
+ [DiTBlock(dim, n_heads, int(dim * mlp_ratio)) for _ in range(n_layers)]
128
+ )
129
+ self.norm_out = RMSNorm(dim)
130
+ self.grad_ckpt = False
131
+ nn.init.normal_(self.res_emb.weight, std=0.02)
132
+ nn.init.zeros_(self.mod.weight)
133
+ nn.init.zeros_(self.mod.bias)
134
+ for o in self.outs.values():
135
+ nn.init.zeros_(o.weight)
136
+ nn.init.zeros_(o.bias)
137
+
138
+ def patch_for(self, size):
139
+ return self.patch_large if size >= self.large_from else self.patch_small
140
+
141
+ @staticmethod
142
+ def res_index(size):
143
+ if size not in SIZES:
144
+ raise ValueError(f"unsupported image size {size}; supported sizes: {SIZES}")
145
+ return SIZES.index(size)
146
+
147
+ def patchify(self, x, p):
148
+ B, C, H, W = x.shape
149
+ nh, nw = H // p, W // p
150
+ x = x.reshape(B, C, nh, p, nw, p).permute(0, 2, 4, 1, 3, 5)
151
+ return x.reshape(B, nh * nw, C * p * p), nh, nw
152
+
153
+ def unpatchify(self, x, nh, nw, p):
154
+ B = x.shape[0]
155
+ x = x.reshape(B, nh, nw, self.in_channels, p, p).permute(0, 3, 1, 4, 2, 5)
156
+ return x.reshape(B, self.in_channels, nh * p, nw * p)
157
+
158
+ def forward(self, x, t, txt, txt_mask, drop=None):
159
+ # txt: (B, text_len, enc_dim) encoder features | txt_mask: (B, text_len) bool
160
+ # drop: optional (B,) bool -> replace that sample's caption with the learned null caption
161
+ B, _, H, W = x.shape
162
+ p = self.patch_for(H)
163
+ proj = self.x_projs[str(p)]
164
+ img, nh, nw = self.patchify(x, p)
165
+ img = proj(img.to(proj.weight.dtype))
166
+
167
+ txt = self.txt_proj(self.txt_norm(txt.to(self.txt_proj.weight.dtype)))
168
+ mask = txt_mask.bool()
169
+ txt = torch.where(mask.unsqueeze(-1), txt, self.pad_emb)
170
+ if drop is not None:
171
+ d = drop.bool()
172
+ txt = torch.where(d.view(B, 1, 1), self.null_txt.unsqueeze(0), txt)
173
+ mask = torch.where(d.view(B, 1), torch.ones_like(mask), mask)
174
+
175
+ seq = torch.cat([txt, img], dim=1)
176
+ t_emb = timestep_embedding(t * 1000.0, self.dim).to(self.time_mlp[0].weight.dtype)
177
+ keep = mask.unsqueeze(-1).to(txt.dtype)
178
+ pooled = (txt * keep).sum(dim=1) / keep.sum(dim=1).clamp(min=1.0)
179
+ cond = self.time_mlp(t_emb) + pooled + self.res_emb.weight[self.res_index(H)]
180
+ mod = self.mod(F.silu(cond)).view(B, 6, 1, self.dim).unbind(1)
181
+ cos, sin = build_rope(nh, self.text_len, self.dim // self.n_heads, x.device, p)
182
+ h = seq
183
+ for blk in self.blocks:
184
+ if self.grad_ckpt and self.training:
185
+ h = torch.utils.checkpoint.checkpoint(blk, h, cos, sin, mod, use_reentrant=False)
186
+ else:
187
+ h = blk(h, cos, sin, mod)
188
+ h = self.norm_out(h)[:, self.text_len:]
189
+ return self.unpatchify(self.outs[str(p)](h), nh, nw, p)
190
+
191
+
192
+ def load_text_encoder(model_id, device="cuda", dtype=torch.bfloat16):
193
+ from transformers import AutoModel, AutoTokenizer
194
+ tokenizer = AutoTokenizer.from_pretrained(model_id)
195
+ if tokenizer.pad_token is None:
196
+ tokenizer.pad_token = tokenizer.eos_token
197
+ last_err = None
198
+ for kw in ({"dtype": dtype, "attn_implementation": "sdpa"},
199
+ {"torch_dtype": dtype, "attn_implementation": "sdpa"},
200
+ {"dtype": dtype},
201
+ {"torch_dtype": dtype}):
202
+ try:
203
+ enc = AutoModel.from_pretrained(model_id, trust_remote_code=True, **kw)
204
+ break
205
+ except (TypeError, ValueError, ImportError) as e:
206
+ last_err = e
207
+ else:
208
+ raise last_err
209
+ enc = enc.to(device).eval().requires_grad_(False)
210
+ return tokenizer, enc
211
+
212
+
213
+ @torch.no_grad()
214
+ def encode_prompts(tokenizer, text_encoder, prompts, text_len, device):
215
+ prompts = [p if isinstance(p, str) and p.strip() else "." for p in prompts]
216
+ tok = tokenizer(prompts, padding="max_length", truncation=True, max_length=text_len,
217
+ return_tensors="pt")
218
+ ids = tok["input_ids"].to(device)
219
+ mask = tok["attention_mask"].to(device)
220
+ out = text_encoder(input_ids=ids, attention_mask=mask)
221
+ feats = out.last_hidden_state if hasattr(out, "last_hidden_state") else out[0]
222
+ return feats, mask.bool()
223
+
224
+
225
+ def decode_latents(x):
226
+ x = x.float().clamp(-1, 1).add(1).mul(127.5).round().to(torch.uint8)
227
+ return [Image.fromarray(np.ascontiguousarray(arr.transpose(1, 2, 0))) for arr in x.cpu().numpy()]
228
+
229
+
230
+ def sprite_grid(images, scale=8, cols=None, bg=(40, 40, 40, 255)):
231
+ cols = cols or int(math.ceil(math.sqrt(len(images))))
232
+ rows = math.ceil(len(images) / cols)
233
+ w, h = images[0].size
234
+ grid = Image.new("RGBA", (cols * w, rows * h), bg)
235
+ for i, im in enumerate(images):
236
+ grid.alpha_composite(im.convert("RGBA"), dest=((i % cols) * w, (i // cols) * h))
237
+ return grid.resize((cols * w * scale, rows * h * scale), Image.NEAREST)
238
+
239
+
240
+ @torch.no_grad()
241
+ def generate(model, prompts, encode_fn, size=32, steps=30, guidance=3.0, seed=0,
242
+ device="cuda", num_per_prompt=1):
243
+ # encode_fn(list[str]) -> (features, bool mask)
244
+ was_training = model.training
245
+ model.eval()
246
+ dev = torch.device(device)
247
+ gen = torch.Generator(device=dev).manual_seed(int(seed))
248
+ if isinstance(prompts, str):
249
+ prompts = [prompts]
250
+ expanded = [p for p in prompts for _ in range(num_per_prompt)]
251
+ B = len(expanded)
252
+ feats, mask = encode_fn(expanded)
253
+ use_cfg = guidance != 1.0
254
+ if use_cfg:
255
+ feats2 = torch.cat([feats, feats])
256
+ mask2 = torch.cat([mask, mask])
257
+ drop2 = torch.cat([torch.zeros(B, dtype=torch.bool, device=dev),
258
+ torch.ones(B, dtype=torch.bool, device=dev)])
259
+ x = torch.randn(B, model.in_channels, size, size, device=dev, generator=gen)
260
+ ts = torch.linspace(1.0, 0.0, steps + 1, device=dev)
261
+ for i in range(steps):
262
+ if use_cfg:
263
+ xi, f, m, d, t = torch.cat([x, x]), feats2, mask2, drop2, ts[i].expand(2 * B)
264
+ else:
265
+ xi, f, m, d, t = x, feats, mask, None, ts[i].expand(B)
266
+ with torch.autocast(device_type=dev.type, dtype=torch.bfloat16, enabled=dev.type == "cuda"):
267
+ v = model(xi, t, f, m, d)
268
+ v = v.float()
269
+ if use_cfg:
270
+ vc, vu = v.chunk(2)
271
+ v = vu + guidance * (vc - vu)
272
+ x = x + (ts[i + 1] - ts[i]) * v
273
+ model.train(was_training)
274
+ return decode_latents(x)
275
+
276
+
277
+ import json
278
+ import os
279
+ from pathlib import Path
280
+
281
+ try:
282
+ from diffusers import DiffusionPipeline
283
+ from diffusers.configuration_utils import ConfigMixin, register_to_config
284
+ from diffusers.models.modeling_utils import ModelMixin
285
+ from diffusers.pipelines.pipeline_utils import ImagePipelineOutput
286
+ from diffusers.utils.torch_utils import randn_tensor
287
+ _HAS_DIFFUSERS = True
288
+ except ImportError:
289
+ _HAS_DIFFUSERS = False
290
+
291
+ if _HAS_DIFFUSERS:
292
+
293
+ class PixelArtDiTModel(ModelMixin, ConfigMixin):
294
+ config_name = "config.json"
295
+
296
+ @register_to_config
297
+ def __init__(self, dim=768, n_layers=18, n_heads=12, mlp_ratio=3.0,
298
+ patch_small=1, patch_large=2, large_from=64, in_channels=4,
299
+ text_len=64, enc_dim=2048, text_encoder_id="MongoDB/voyage-4-nano"):
300
+ super().__init__()
301
+ self.inner = PixelArtDiT(dim=dim, n_layers=n_layers, n_heads=n_heads,
302
+ mlp_ratio=mlp_ratio, patch_small=patch_small,
303
+ patch_large=patch_large, large_from=large_from,
304
+ in_channels=in_channels, text_len=text_len,
305
+ enc_dim=enc_dim)
306
+
307
+ def forward(self, x, t, txt, txt_mask, drop=None):
308
+ return self.inner(x, t, txt, txt_mask, drop)
309
+
310
+ @property
311
+ def text_len(self):
312
+ return self.inner.text_len
313
+
314
+ @property
315
+ def in_channels(self):
316
+ return self.inner.in_channels
317
+
318
+ class PixelArtDiTPipeline(DiffusionPipeline):
319
+ def __init__(self, transformer, scheduler):
320
+ super().__init__()
321
+ self.register_modules(transformer=transformer, scheduler=scheduler)
322
+
323
+ def _get_text_encoder(self, device):
324
+ # frozen text encoder is downloaded from the Hub id stored in transformer/config.json
325
+ if getattr(self, "_text_encoder", None) is None:
326
+ self._tokenizer, self._text_encoder = load_text_encoder(
327
+ self.transformer.config.text_encoder_id, device=device)
328
+ else:
329
+ self._text_encoder.to(device)
330
+ return self._tokenizer, self._text_encoder
331
+
332
+ @torch.no_grad()
333
+ def __call__(self, prompt, height=32, width=32, num_inference_steps=30,
334
+ guidance_scale=3.0, num_images_per_prompt=1, generator=None,
335
+ output_type="pil", return_dict=True):
336
+ if height != width or height not in (16, 32, 64):
337
+ raise ValueError("PixelArtDiT supports square images of size 16, 32 or 64.")
338
+ if isinstance(prompt, str):
339
+ prompt = [prompt]
340
+ prompts = [p for p in prompt for _ in range(num_images_per_prompt)]
341
+ B = len(prompts)
342
+ device = self._execution_device
343
+ model = self.transformer
344
+ dtype = model.dtype
345
+
346
+ tokenizer, text_encoder = self._get_text_encoder(device)
347
+ feats, mask = encode_prompts(tokenizer, text_encoder, prompts, model.text_len, device)
348
+ use_cfg = guidance_scale != 1.0
349
+ if use_cfg:
350
+ feats_in = torch.cat([feats, feats])
351
+ mask_in = torch.cat([mask, mask])
352
+ drop = torch.cat([torch.zeros(B, dtype=torch.bool, device=device),
353
+ torch.ones(B, dtype=torch.bool, device=device)])
354
+ else:
355
+ feats_in, mask_in, drop = feats, mask, None
356
+
357
+ self.scheduler.set_timesteps(num_inference_steps, device=device)
358
+ latents = randn_tensor((B, model.in_channels, height, width), generator=generator,
359
+ device=device, dtype=torch.float32)
360
+
361
+ for i, t in enumerate(self.scheduler.timesteps):
362
+ sigma = self.scheduler.sigmas[i].to(device=device, dtype=torch.float32)
363
+ x_in = torch.cat([latents, latents]) if use_cfg else latents
364
+ t_in = sigma.expand(x_in.shape[0])
365
+ v = model(x_in.to(dtype), t_in, feats_in, mask_in, drop).float()
366
+ if use_cfg:
367
+ vc, vu = v.chunk(2)
368
+ v = vu + guidance_scale * (vc - vu)
369
+ latents = self.scheduler.step(v, t, latents).prev_sample
370
+
371
+ if output_type == "latent":
372
+ out = latents
373
+ elif output_type == "np":
374
+ out = ((latents.clamp(-1, 1) + 1) / 2).permute(0, 2, 3, 1).cpu().numpy()
375
+ else:
376
+ out = decode_latents(latents)
377
+ if not return_dict:
378
+ return (out,)
379
+ return ImagePipelineOutput(images=out)
380
+
381
+ else:
382
+ PixelArtDiTModel = PixelArtDiT
383
+
384
+
385
+ def load_pixel_art_dit(repo_or_path, device="cuda", dtype=torch.bfloat16):
386
+ from safetensors.torch import load_file
387
+ if os.path.isdir(repo_or_path):
388
+ root = Path(repo_or_path)
389
+ else:
390
+ from huggingface_hub import snapshot_download
391
+ root = Path(snapshot_download(repo_or_path))
392
+ with open(root / "transformer" / "config.json") as f:
393
+ cfg = json.load(f)
394
+ cfg = {k: v for k, v in cfg.items() if not k.startswith("_") and k != "text_encoder_id"}
395
+ model = PixelArtDiT(**cfg)
396
+ sd = load_file(root / "transformer" / "diffusion_pytorch_model.safetensors")
397
+ sd = {k[len("inner."):] if k.startswith("inner.") else k: v for k, v in sd.items()}
398
+ model.load_state_dict(sd)
399
+ return model.to(device=device, dtype=dtype).eval()