Spaces:
Running on Zero
Running on Zero
Download sampling.py from hugging-apps/hiuni: direct link, hf CLI and curl.
- Browser
- Download file 8.91 kB
-
https://huggingface.co/spaces/hugging-apps/hiuni/resolve/main/sampling.py
- Command line
-
hf download hf://spaces/hugging-apps/hiuni/sampling.py
-
curl -L -o sampling.py https://huggingface.co/spaces/hugging-apps/hiuni/resolve/main/sampling.py
8.91 kB
| """Pixel layout, attention masks and flow matching used by HF image generation.""" | |
| import math | |
| import random | |
| import torch | |
| INSTRUCTIONS = [ | |
| "\u8bf7\u6839\u636e\u4ee5\u4e0b\u63cf\u8ff0\u751f\u6210\u4e00\u5f20\u56fe\u7247\uff1a", | |
| "\u6839\u636e\u4e0b\u9762\u7684\u63cf\u8ff0\u751f\u6210\u56fe\u7247\uff1a", | |
| "\u751f\u6210\u56fe\u7247\uff1a", "\u8bf7\u753b\u4e00\u5f20\u56fe\uff1a", | |
| "\u5e2e\u6211\u751f\u6210\u4e00\u5f20\u56fe\u7247\uff1a", | |
| "\u6839\u636e\u63cf\u8ff0\u521b\u5efa\u56fe\u50cf\uff1a", | |
| "\u8bf7\u6839\u636e\u6587\u5b57\u63cf\u8ff0\u7ed8\u5236\u56fe\u7247\uff1a", | |
| "\u6309\u7167\u4ee5\u4e0b\u5185\u5bb9\u751f\u6210\u56fe\u7247\uff1a", | |
| "\u751f\u6210\u4e00\u5f20\u7b26\u5408\u63cf\u8ff0\u7684\u56fe\u7247\uff1a", | |
| "\u8bf7\u4e3a\u4ee5\u4e0b\u5185\u5bb9\u521b\u4f5c\u4e00\u5f20\u56fe\u7247\uff1a", | |
| "\u6839\u636e\u63d0\u793a\u751f\u6210\u56fe\u50cf\uff1a", "\u56fe\u7247\u751f\u6210\uff1a", | |
| "Generate an image based on the following description: ", "Create an image: ", | |
| "Please generate an image: ", "Draw an image based on: ", "Create a picture: ", | |
| "Generate: ", "Please draw: ", "Create an image from the description: ", | |
| "Generate a picture: ", "Please create an image based on: ", | |
| ] | |
| def merge_patches(raw, height, width): | |
| """Pack raster patches as [patch_channel, block_row, block_column].""" | |
| dim = raw.shape[-1] | |
| return raw.reshape(height, 2, width, 2, dim).permute(0, 2, 4, 1, 3).reshape(-1, dim * 4) | |
| def unmerge_patches(merged, height, width): | |
| """Invert the checkpoint's 2x2 pixel packing.""" | |
| dim = merged.shape[-1] // 4 | |
| return merged.reshape(height, width, dim, 2, 2).permute(0, 3, 1, 4, 2).reshape(-1, dim) | |
| def vision_patches(raw, height, width, patch_size=16): | |
| """Convert raster RGB patches to Qwen's block-major, duplicated-frame layout.""" | |
| patches = raw.reshape(height, 2, width, 2, 3, patch_size, patch_size) | |
| patches = patches.permute(0, 2, 1, 3, 4, 5, 6) | |
| return patches.unsqueeze(5).expand(-1, -1, -1, -1, -1, 2, -1, -1).reshape( | |
| 4 * height * width, 6 * patch_size * patch_size | |
| ) | |
| def build_sequence(tokenizer, config, prompt, height, width, instruction="", chat=True, route=False): | |
| """Build the training chat template without retokenizing instruction boundaries.""" | |
| def encode(text): | |
| """Encode text without injecting tokenizer-level special tokens.""" | |
| return tokenizer.encode(text, add_special_tokens=False) | |
| image = [config.vision_start_token_id, config.time_token_id] | |
| image += [config.image_token_id] * (height * width) | |
| image += [config.vision_end_token_id, 151645] | |
| if not chat: | |
| return torch.tensor([image], dtype=torch.long) | |
| prefix = [151644] + encode("user") + encode("\n") | |
| prefix += encode(instruction) + encode(prompt) + [151645] + encode("\n") | |
| prefix += [151644] + encode("assistant") + encode("\n") | |
| if route: | |
| if encode("<|image_gate|>") != [config.image_gate_token_id]: | |
| raise ValueError("Tokenizer does not contain the configured image gate token") | |
| prefix.append(config.image_gate_token_id) | |
| return torch.tensor([prefix + image], dtype=torch.long) | |
| def sequence_layout(ids, config, height, width, dtype): | |
| """Return M-RoPE positions and causal attention with bidirectional image tokens.""" | |
| image_mask = ids.eq(config.image_token_id) | |
| indices = image_mask[0].nonzero(as_tuple=True)[0] | |
| if indices.numel() != height * width: | |
| raise ValueError("Prompt must not contain image special tokens") | |
| start, end = int(indices[0]), int(indices[-1]) + 1 | |
| if end - start != height * width or ids.eq(config.time_token_id).sum() != 1: | |
| raise ValueError("Expected one contiguous image block and one time token") | |
| length = ids.shape[1] | |
| positions = torch.arange(length, device=ids.device).view(1, 1, -1).repeat(3, 1, 1) | |
| positions[0, 0, start:end] = start | |
| positions[1, 0, start:end] = start + torch.arange(height, device=ids.device).repeat_interleave(width) | |
| positions[2, 0, start:end] = start + torch.arange(width, device=ids.device).repeat(height) | |
| positions[:, 0, end:] = start + max(height, width) + torch.arange(length - end, device=ids.device) | |
| allowed = torch.ones(length, length, dtype=torch.bool, device=ids.device).tril() | |
| allowed[start:end, start:end] = True | |
| # The Megatron startend mask has LTS=L-1, masking the final EOS row. | |
| # Its output is unused by the image head; keep that row valid for SDPA. | |
| mask = torch.zeros(length, length, device=ids.device, dtype=dtype) | |
| mask.masked_fill_(~allowed, torch.finfo(dtype).min) | |
| return positions, mask[None, None], image_mask | |
| def flow_sigmas(steps, shift, device): | |
| """Reproduce FlowMatchScheduler(extra_one_step=True), including its sigma floor.""" | |
| if steps <= 0 or not math.isfinite(shift) or shift <= 0: | |
| raise ValueError("steps and shift must be positive") | |
| sigma = torch.linspace(1.0, 0.003 / 1.002, steps + 1, device=device)[:-1] | |
| sigma = shift * sigma / (1 + (shift - 1) * sigma) | |
| return torch.cat((sigma, sigma.new_zeros(1))) | |
| def flow_step(prediction, latents, sigma, next_sigma, predict_x0): | |
| """Match Pixel-DiT Euler sampling, which disables the training loss sigma clamp.""" | |
| velocity = (latents - prediction) / sigma if predict_x0 else prediction | |
| return latents + velocity * (next_sigma - sigma) | |
| def generate_image(model, tokenizer, prompt, height=1024, width=1024, num_steps=50, | |
| guidance_scale=4.0, shift=4.0, seed=42, negative_prompt="", | |
| cfg_keep_chat_template=False, instruction=None, predict_x0=None, | |
| drop_dit_text=False, dynamic_noise_scale=None, dynamic_noise_ref_tokens=None, | |
| dynamic_noise_base_scale=None): | |
| """Generate one PIL image; dimensions are pixels and seeds are local to the call.""" | |
| from PIL import Image | |
| if height <= 0 or width <= 0 or height % 32 or width % 32: | |
| raise ValueError("height and width must be positive multiples of 32 pixels") | |
| if not math.isfinite(guidance_scale) or guidance_scale < 1: | |
| raise ValueError("guidance_scale must be finite and at least 1") | |
| model.eval() | |
| config = model.config | |
| device, dtype = model.device, model.dtype | |
| h, w = height // 32, width // 32 | |
| prefix = random.Random(seed).choice(INSTRUCTIONS) if instruction is None else instruction | |
| cond = build_sequence(tokenizer, config, prompt, h, w, prefix, route=config.predict_gate_mode).to(device) | |
| sequences = [cond] | |
| if guidance_scale > 1: | |
| sequences.append(build_sequence(tokenizer, config, negative_prompt, h, w, | |
| prefix if cfg_keep_chat_template else "", | |
| chat=cfg_keep_chat_template).to(device)) | |
| layouts = [sequence_layout(ids, config, h, w, dtype) for ids in sequences] | |
| max_length = config.text_config.max_position_embeddings | |
| if max(ids.shape[1] for ids in sequences) > max_length: | |
| raise ValueError(f"Prompt and image exceed max_position_embeddings={max_length}") | |
| generator = torch.Generator(device=device).manual_seed(seed) | |
| latents = torch.randn(4 * h * w, 768, generator=generator, device=device, dtype=torch.float32) | |
| dynamic_noise_scale = config.use_dynamic_noise_scale if dynamic_noise_scale is None else dynamic_noise_scale | |
| noise_ref = config.dynamic_noise_ref_tokens if dynamic_noise_ref_tokens is None else dynamic_noise_ref_tokens | |
| noise_base = config.dynamic_noise_base_scale if dynamic_noise_base_scale is None else dynamic_noise_base_scale | |
| if noise_ref <= 0 or not math.isfinite(noise_base) or noise_base <= 0: | |
| raise ValueError("Dynamic noise reference tokens and base scale must be positive and finite") | |
| if dynamic_noise_scale: | |
| latents *= math.sqrt(4 * h * w / noise_ref) * noise_base | |
| sigmas = flow_sigmas(num_steps, shift, device) | |
| predict_x0 = config.predict_x0 if predict_x0 is None else predict_x0 | |
| for sigma, next_sigma in zip(sigmas[:-1], sigmas[1:]): | |
| predictions = [model.predict_image(ids, latents, sigma.reshape(1) * 1000, | |
| h, w, *layout, drop_dit_text=drop_dit_text) | |
| for ids, layout in zip(sequences, layouts)] | |
| prediction = predictions[0] | |
| if len(predictions) == 2: | |
| prediction = predictions[1] + guidance_scale * (prediction - predictions[1]) | |
| latents = flow_step(prediction, latents, sigma, next_sigma, predict_x0) | |
| if not torch.isfinite(latents).all(): | |
| raise RuntimeError("Non-finite pixels produced during denoising") | |
| pixels = latents.reshape(2 * h, 2 * w, 3, 16, 16).permute(0, 3, 1, 4, 2) | |
| pixels = pixels.reshape(height, width, 3).add(1).div(2).clamp(0, 1) | |
| return Image.fromarray(pixels.mul(255).add(0.5).to(torch.uint8).cpu().numpy()) | |