models/hidream/pipeline.py

import torch
import einops
import numpy as np
import tqdm
from PIL import Image
import torchvision.transforms.v2 as transforms
# FlowUniPCMultistepScheduler generates more details than FlowMatchEulerDiscreteScheduler
from .fm_solvers_unipc import FlowUniPCMultistepScheduler  # noqa: E402

from .flash_scheduler import FlashFlowMatchEulerDiscreteScheduler
from .utils import resize_pilimage, calculate_dimensions, find_closest_resolution, get_rope_index_fix_point

TIMESTEP_TOKEN_NUM = 1
NOISE_SCALE = 8.0
T_EPS = 0.001
CONDITION_IMAGE_SIZE = 384
PATCH_SIZE = 32

TENSOR_TRANSFORM = transforms.Compose([
    transforms.ToImage(),
    transforms.ToDtype(torch.float32, scale=True),
    transforms.Normalize([0.5], [0.5]),
])

DEFAULT_TIMESTEPS = [
    999, 987, 974, 960, 945, 929, 913, 895, 877, 857, 836, 814, 790, 764, 737,
    707, 675, 640, 602, 560, 515, 464, 409, 347, 278, 199, 110, 8,
]


def resample_timesteps(timesteps, num_steps):
    if num_steps == len(timesteps):
        return list(timesteps)
    positions = np.linspace(0, len(timesteps) - 1, num_steps)
    values = np.interp(positions, np.arange(len(timesteps)), timesteps)
    return [int(round(value)) for value in values]


def build_t2i_text_sample(prompt, height, width, tokenizer, processor, model_config):
    image_token_id = model_config.image_token_id
    video_token_id = model_config.video_token_id
    vision_start_token_id = model_config.vision_start_token_id
    image_len = (height // PATCH_SIZE) * (width // PATCH_SIZE)

    boi_token = getattr(tokenizer, "boi_token", "<|boi_token|>")
    tms_token = getattr(tokenizer, "tms_token", "<|tms_token|>")

    messages = [{"role": "user", "content": prompt}]
    template_caption = (
            processor.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
            + boi_token
            + tms_token * TIMESTEP_TOKEN_NUM
    )
    input_ids = tokenizer.encode(template_caption, return_tensors="pt", add_special_tokens=False)

    image_grid_thw = torch.tensor(
        [1, height // PATCH_SIZE, width // PATCH_SIZE], dtype=torch.int64, device=input_ids.device
    ).unsqueeze(0)

    vision_tokens = torch.full((1, image_len), image_token_id, dtype=input_ids.dtype, device=input_ids.device)
    vision_tokens[0, 0] = vision_start_token_id
    input_ids_pad = torch.cat([input_ids, vision_tokens], dim=-1)

    position_ids, _ = get_rope_index_fix_point(
        1, image_token_id, video_token_id, vision_start_token_id,
        input_ids=input_ids_pad, image_grid_thw=image_grid_thw,
        video_grid_thw=None, attention_mask=None, skip_vision_start_token=[1],
    )

    txt_seq_len = input_ids.shape[-1]
    all_seq_len = position_ids.shape[-1]

    token_types = torch.zeros((1, all_seq_len), dtype=input_ids.dtype, device=input_ids.device)
    bgn = txt_seq_len - TIMESTEP_TOKEN_NUM
    token_types[0, bgn: bgn + image_len + TIMESTEP_TOKEN_NUM] = 1
    token_types[0, txt_seq_len - TIMESTEP_TOKEN_NUM: txt_seq_len] = 3

    vinput_mask = (token_types == 1)
    token_types_bin = (token_types > 0).to(token_types.dtype)

    return {
        'input_ids': input_ids,
        'position_ids': position_ids,
        'token_types': token_types_bin,
        'vinput_mask': vinput_mask,
    }

def build_scheduler(num_inference_steps, timesteps_list, shift, device, scheduler_name="default"):
    if timesteps_list is not None:
        num_inference_steps = len(timesteps_list)
    if scheduler_name == "flash":
        sched = FlashFlowMatchEulerDiscreteScheduler(
            num_train_timesteps=1000, shift=shift, use_dynamic_shifting=False)
    elif scheduler_name == "default":
        sched = FlowUniPCMultistepScheduler(use_dynamic_shifting=False, shift=shift)
    else:
        raise ValueError(f"Unknown scheduler_name={scheduler_name!r}")
    sched.set_timesteps(num_inference_steps, device=device)
    if timesteps_list is not None:
        sched.timesteps = torch.tensor(timesteps_list, device=device, dtype=torch.long)
        sigmas = [t.item() / 1000.0 for t in sched.timesteps]
        sigmas.append(0.0)
        sched.sigmas = torch.tensor(sigmas, device=device)
    return sched


def clamp_tensor(tensor, percentage = 0.1):
    lower_bound = torch.quantile(tensor.float(), percentage)
    upper_bound = torch.quantile(tensor.float(), 1 - percentage)
    src_dtype = tensor.dtype
    return torch.clamp(tensor.float(), min=lower_bound, max=upper_bound).to(src_dtype)


@torch.no_grad()
def generate_image(
        model,
        processor,
        prompt: str,
        ref_image_paths: list = None,
        ref_images: list = None,
        height: int = 1440,
        width: int = 2560,
        num_inference_steps: int = 50,
        guidance_scale: float = 5.0,
        shift: float = 3.0,
        timesteps_list=None,
        scheduler_name: str = "default",
        seed: int = 42,
        noise_scale_start: float = NOISE_SCALE,
        noise_scale_end: float = NOISE_SCALE,
        noise_clip_std: float = 0.0,
        keep_original_aspect: bool = False,
        batch_size: int = 1,
        joint_pass: bool = True,
        callback=None,
        abort_callback=None,
) -> Image.Image:
    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
    dtype = torch.bfloat16
    model_config = model.config
    tokenizer = processor.tokenizer if hasattr(processor, 'tokenizer') else processor
    ref_image_paths = [] if ref_image_paths is None else ref_image_paths
    ref_images = [] if ref_images is None else ref_images
    requested_width, requested_height = width, height
    resize_to_requested = False
    ref_count = len(ref_images) + len(ref_image_paths)
    batch_size = max(1, int(batch_size))

    # Single-reference edits are unstable on low/non-native canvases. Use the
    # upstream reference canvas internally, then resize to WanGP's requested size.
    preresized_ref_pil = None
    if ref_count == 1:
        pil_orig = (ref_images[0] if ref_images else Image.open(ref_image_paths[0])).convert("RGB")
        preresized_ref_pil = resize_pilimage(pil_orig, 2048, PATCH_SIZE)
        width, height = preresized_ref_pil.size
        resize_to_requested = not keep_original_aspect and (width != requested_width or height != requested_height)
        if keep_original_aspect:
            print(f"[info] keep_original_aspect: target size set to {width}x{height} from reference image")
    elif keep_original_aspect:
        print("[warning] keep_original_aspect requires exactly one reference image; using requested dimensions.")
    elif ref_count:
        native_width, native_height = find_closest_resolution(width, height)
        resize_to_requested = native_width != width or native_height != height
        width, height = native_width, native_height

    h_patches = height // PATCH_SIZE
    w_patches = width // PATCH_SIZE

    if not ref_images and not ref_image_paths:
        cond_sample = build_t2i_text_sample(prompt, height, width, tokenizer, processor, model_config)
        cond_sample["prediction_mask"] = cond_sample["vinput_mask"]
        uncond_sample = None
        if guidance_scale > 1.0:
            uncond_sample = build_t2i_text_sample(" ", height, width, tokenizer, processor, model_config)
            uncond_sample["prediction_mask"] = uncond_sample["vinput_mask"]
        
        def to_device(s):
            return {k: (v.to(device) if torch.is_tensor(v) else v) for k, v in s.items()}
        
        cond_sample = to_device(cond_sample)
        if uncond_sample is not None:
            uncond_sample = to_device(uncond_sample)
            
        ref_patches = None
        tgt_image_len = (height // PATCH_SIZE) * (width // PATCH_SIZE)
        samples = [cond_sample]
        if uncond_sample:
            samples.append(uncond_sample)
    else:
        image_token_id = model_config.image_token_id
        video_token_id = model_config.video_token_id
        vision_start_token_id = model_config.vision_start_token_id
        spatial_merge_size = model_config.vision_config.spatial_merge_size
        
        if preresized_ref_pil is not None:
            ref_pils = [preresized_ref_pil]
        else:
            ref_pils = [img.convert("RGB") for img in ref_images] + [Image.open(p).convert("RGB") for p in ref_image_paths]
        K = len(ref_pils)

        if K == 1: max_size = max(height, width)
        elif K == 2: max_size = max(height, width) * 48 // 64
        elif K <= 4: max_size = max(height, width) // 2
        elif K <= 8: max_size = max(height, width) * 24 // 64
        else: max_size = max(height, width) // 4

        ref_pils_resized, ref_images = [], []
        for pil in ref_pils:
            # Skip resizing when caller already produced a patch-aligned ref via
            # `keep_original_aspect` — re-running resize_pilimage on it would
            # upscale (since max_size == max(width, height) of the resized ref).
            if preresized_ref_pil is not None and pil is preresized_ref_pil:
                pil_r = pil
            else:
                pil_r = resize_pilimage(pil, max_size, PATCH_SIZE)
            ref_pils_resized.append(pil_r)
            x = TENSOR_TRANSFORM(pil_r)
            x = einops.rearrange(x, "C (H p1) (W p2) -> (H W) (C p1 p2)", p1=PATCH_SIZE, p2=PATCH_SIZE)
            ref_images.append(x)

        ref_image_lens = [img.shape[0] for img in ref_images]
        total_ref_len = sum(ref_image_lens)
        ref_patches = torch.cat(ref_images, dim=0).unsqueeze(0).to(device, dtype)

        tgt_image_len = (height // PATCH_SIZE) * (width // PATCH_SIZE)
        h_patches = height // PATCH_SIZE
        w_patches = width // PATCH_SIZE

        if K <= 4: cond_img_size = CONDITION_IMAGE_SIZE
        elif K <= 8: cond_img_size = CONDITION_IMAGE_SIZE * 48 // 64
        else: cond_img_size = CONDITION_IMAGE_SIZE // 2

        ref_pils_vlm = []
        for pil_r in ref_pils_resized:
            cond_w, cond_h = calculate_dimensions(cond_img_size, pil_r.width / pil_r.height)
            ref_pils_vlm.append(pil_r.resize((cond_w, cond_h), resample=Image.LANCZOS))

        image_grid_thw_tgt = torch.tensor([1, height // PATCH_SIZE, width // PATCH_SIZE], dtype=torch.int64, device="cpu").unsqueeze(0)
        image_grid_thw_ref = torch.zeros((K, 3), dtype=torch.int64, device="cpu")
        for i, pil_r in enumerate(ref_pils_resized):
            rw, rh = pil_r.size
            image_grid_thw_ref[i] = torch.tensor([1, rh // PATCH_SIZE, rw // PATCH_SIZE], dtype=torch.int64, device=image_grid_thw_ref.device)

        samples = []
        captions = [prompt]
        if guidance_scale > 1.0:
            captions.append(" ")
            
        for caption in captions:
            boi_token = getattr(tokenizer, "boi_token", "<|boi_token|>")
            tms_token = getattr(tokenizer, "tms_token", "<|tms_token|>")
            
            content = [{"type": "image"} for _ in range(K)]
            content.append({"type": "text", "text": caption})
            messages = [{"role": "user", "content": content}]
            template_caption = processor.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
            proc = processor(text=[template_caption], images=ref_pils_vlm, padding="longest", return_tensors="pt")
            proc_input_ids = proc.input_ids.to("cpu")
            proc_image_grid_thw = proc.image_grid_thw.to("cpu")
            proc_pixel_values = proc.pixel_values.to("cpu")
            input_ids_2 = tokenizer.encode(boi_token + tms_token * TIMESTEP_TOKEN_NUM, return_tensors="pt", add_special_tokens=False)
            input_ids = torch.cat([proc_input_ids, input_ids_2.to(proc_input_ids.device)], dim=-1)

            igthw_cond = proc_image_grid_thw.clone()
            for i in range(K):
                igthw_cond[i, 1] //= spatial_merge_size
                igthw_cond[i, 2] //= spatial_merge_size
            igthw_all = torch.cat([igthw_cond, image_grid_thw_tgt, image_grid_thw_ref], dim=0)

            vision_tokens_list = []
            vt_tgt = torch.full((1, tgt_image_len), image_token_id, dtype=input_ids.dtype, device=input_ids.device)
            vt_tgt[0, 0] = vision_start_token_id
            vision_tokens_list.append(vt_tgt)
            for rl in ref_image_lens:
                vt_ref = torch.full((1, rl), image_token_id, dtype=input_ids.dtype, device=input_ids.device)
                vt_ref[0, 0] = vision_start_token_id
                vision_tokens_list.append(vt_ref)
            vision_tokens = torch.cat(vision_tokens_list, dim=1)
            input_ids_pad = torch.cat([input_ids, vision_tokens], dim=-1)

            position_ids, _ = get_rope_index_fix_point(
                1, image_token_id, video_token_id, vision_start_token_id,
                input_ids=input_ids_pad, image_grid_thw=igthw_all,
                video_grid_thw=None, attention_mask=None,
                skip_vision_start_token=[0] * K + [1] + [1] * K,
            )
            txt_seq_len = input_ids.shape[-1]
            all_seq_len = position_ids.shape[-1]

            token_types_raw = torch.zeros((1, all_seq_len), dtype=input_ids.dtype, device=input_ids.device)
            bgn = txt_seq_len - TIMESTEP_TOKEN_NUM
            end = bgn + tgt_image_len + TIMESTEP_TOKEN_NUM
            token_types_raw[0, bgn:end] = 1
            token_types_raw[0, end: end + total_ref_len] = 2
            token_types_raw[0, txt_seq_len - TIMESTEP_TOKEN_NUM: txt_seq_len] = 3

            vinput_mask = torch.logical_or(token_types_raw == 1, token_types_raw == 2)
            token_types_bin = (token_types_raw > 0).to(token_types_raw.dtype)
            prediction_mask = token_types_raw == 1

            samples.append({
                "input_ids": input_ids.to(device),
                "position_ids": position_ids.to(device),
                "token_types": token_types_bin.to(device),
                "vinput_mask": vinput_mask.to(device),
                "prediction_mask": prediction_mask.to(device),
                "pixel_values_cpu": proc_pixel_values,
                "image_grid_thw": proc_image_grid_thw.to(device),
            })

        for sample in samples:
            with torch.autocast(device.type, dtype=dtype, cache_enabled=False):
                pixel_values = sample.pop("pixel_values_cpu").to(device, dtype)
                image_embeds, _ = model.get_image_features(pixel_values, sample["image_grid_thw"])
                if image_embeds is None:
                    return None
                sample["image_embeds"] = torch.cat(image_embeds, dim=0).to(device, dtype)
                del pixel_values, image_embeds

    noise = torch.empty((batch_size, 3, height, width), device="cpu", dtype=torch.float32)
    for batch_idx in range(batch_size):
        noise[batch_idx].normal_(generator=torch.Generator("cpu").manual_seed(seed + batch_idx + 1))
    noise = noise.mul_(noise_scale_start).to(device, dtype)
    z = einops.rearrange(noise, 'B C (H p1) (W p2) -> B (H W) (C p1 p2)', p1=PATCH_SIZE, p2=PATCH_SIZE)
    del noise

    sched = build_scheduler(num_inference_steps, timesteps_list, shift, device, scheduler_name)
    if callback is not None:
        callback(-1, None, True, override_num_inference_steps=len(sched.timesteps))

    num_steps = len(sched.timesteps)
    if num_steps > 1:
        noise_scale_schedule = [
            noise_scale_start + (noise_scale_end - noise_scale_start) * i / (num_steps - 1)
            for i in range(num_steps)
        ]
    else:
        noise_scale_schedule = [noise_scale_start]

    torch.manual_seed(seed + 1)
    if torch.cuda.is_available(): torch.cuda.manual_seed_all(seed + 1)

    def forward_once(sample, z_in, t_pixeldit):
        with torch.autocast(device.type, dtype=dtype, cache_enabled=False):
            kwargs = {
                "input_ids": sample['input_ids'],
                "position_ids": sample['position_ids'],
                "vinputs": z_in,
                "timestep": t_pixeldit.reshape(1).expand(z_in.shape[0]).to(device),
                "token_types": sample['token_types'],
            }
            if "pixel_values" in sample: kwargs["pixel_values"] = sample["pixel_values"]
            if "image_grid_thw" in sample: kwargs["image_grid_thw"] = sample["image_grid_thw"]
            if "image_embeds" in sample: kwargs["image_embeds"] = sample["image_embeds"]
            if "prediction_mask" in sample: kwargs["prediction_mask"] = sample["prediction_mask"]

            outputs = model(**kwargs)
            if outputs.x_pred is None:
                return None
            
        x_pred = outputs.x_pred
        del outputs
        # x_pred = clamp_tensor(x_pred, percentage = 0.01)
        if "prediction_mask" in sample:
            return x_pred
        if ref_patches is None:
            return x_pred[0, sample['vinput_mask'][0]].unsqueeze(0)
        else:
            return x_pred[0, sample['vinput_mask'][0]][:tgt_image_len].unsqueeze(0)

    def forward_samples(sample_list, z_in, t_pixeldit):
        if not (joint_pass and len(sample_list) > 1):
            return [forward_once(sample, z_in, t_pixeldit) for sample in sample_list]

        timestep = t_pixeldit.reshape(1).expand(z_in.shape[0]).to(device)
        with torch.autocast(device.type, dtype=dtype, cache_enabled=False):
            kwargs = {
                "input_ids": [sample["input_ids"] for sample in sample_list],
                "position_ids": [sample["position_ids"] for sample in sample_list],
                "vinputs": [z_in] * len(sample_list),
                "timestep": [timestep] * len(sample_list),
                "token_types": [sample["token_types"] for sample in sample_list],
            }
            if all("image_grid_thw" in sample for sample in sample_list):
                kwargs["image_grid_thw"] = [sample["image_grid_thw"] for sample in sample_list]
            if all("image_embeds" in sample for sample in sample_list):
                kwargs["image_embeds"] = [sample["image_embeds"] for sample in sample_list]
            if all("prediction_mask" in sample for sample in sample_list):
                kwargs["prediction_mask"] = [sample["prediction_mask"] for sample in sample_list]

            outputs = model(**kwargs)
            if outputs.x_pred is None:
                return [None] * len(sample_list)
            x_preds = list(outputs.x_pred)
            del outputs
            return x_preds

    def _preview_tensor(z_in):
        return einops.rearrange(
            z_in.detach(),
            'B (H W) (C p1 p2) -> C B (H p1) (W p2)',
            H=h_patches, W=w_patches, p1=PATCH_SIZE, p2=PATCH_SIZE,
        )

    for step_idx, step_t in enumerate(tqdm.tqdm(sched.timesteps, desc="Generating")):
        if abort_callback is not None and abort_callback():
            return None

        t_pixeldit = 1.0 - step_t.float() / 1000.0
        sigma = (step_t.float() / 1000.0).to(dtype=torch.float32).clamp_min(T_EPS)

        if ref_patches is None:
            z_float = z.to(dtype=torch.float32)
            x_preds = forward_samples(samples, z, t_pixeldit)
            x_pred_cond = x_preds[0]
            if x_pred_cond is None:
                return None
            v_cond = x_pred_cond.to(dtype=torch.float32)
            del x_pred_cond
            v_cond.sub_(z_float).div_(sigma)

            if len(samples) > 1:
                x_pred_uncond = x_preds[1]
                if x_pred_uncond is None:
                    return None
                v_uncond = x_pred_uncond.to(dtype=torch.float32)
                del x_pred_uncond
                v_uncond.sub_(z_float).div_(sigma)
                v_cond.sub_(v_uncond).mul_(guidance_scale).add_(v_uncond)
                del v_uncond
            else:
                pass
            x_preds.clear()
            v_guided = v_cond
        else:
            vinputs = torch.cat([z, ref_patches.expand(z.shape[0], -1, -1)], dim=1)
            x_vis_list = forward_samples(samples, vinputs, t_pixeldit)
            del vinputs
            if any(x is None for x in x_vis_list):
                return None
            z_float = z.to(dtype=torch.float32)
            v_cond = x_vis_list[0].to(dtype=torch.float32)
            v_cond.sub_(z_float).div_(sigma)
            if len(samples) > 1:
                v_uncond = x_vis_list[1].to(dtype=torch.float32)
                v_uncond.sub_(z_float).div_(sigma)
                v_cond.sub_(v_uncond).mul_(guidance_scale).add_(v_uncond)
                v_guided = v_cond
                del v_uncond
            else:
                v_guided = v_cond
            x_vis_list.clear()

        model_output = v_guided.neg_()
        del v_guided
        # model_output = clamp_tensor(model_output, percentage = 0.05)
        step_t_float = step_t.to(dtype=torch.float32)
        if scheduler_name == "flash":
            new_z = sched.step(model_output, step_t_float, z_float, s_noise=noise_scale_schedule[step_idx], noise_clip_std=noise_clip_std, return_dict=False)[0].to(dtype)
        else:
            new_z = sched.step(model_output, step_t_float, z_float, return_dict=False)[0].to(dtype)
        del model_output, z, z_float, step_t_float
        z = new_z
        del new_z

        if callback is not None:
            callback(step_idx, _preview_tensor(z), False)

    img = z.to(device="cpu", dtype=torch.float32)
    del z
    img.add_(1).div_(2)
    img = einops.rearrange(img, 'B (H W) (C p1 p2) -> B C (H p1) (W p2)', H=h_patches, W=w_patches, p1=PATCH_SIZE, p2=PATCH_SIZE)
    images = []
    for batch_idx in range(img.shape[0]):
        arr = np.round(np.clip(img[batch_idx].numpy().transpose(1, 2, 0) * 255, 0, 255)).astype(np.uint8)
        image = Image.fromarray(arr).convert("RGB")
        if resize_to_requested:
            image = image.resize((requested_width, requested_height), resample=Image.LANCZOS)
        images.append(image)
    del img
    return images[0] if batch_size == 1 else images