utils/editor.py

# utils/editor.py
import os
import io
import math
from typing import Tuple, Dict, Any
from PIL import Image, ImageOps
import numpy as np

import torch
from diffusers import StableDiffusionControlNetPipeline, ControlNetModel, UniPCMultistepScheduler
from transformers import logging as hf_logging
hf_logging.set_verbosity_error()

# detector auxiliar para gerar mapa de pose OpenPose-like
from controlnet_aux import OpenposeDetector

# para remoção de fundo da peça (extrair RGBA)
from rembg import remove

# parâmetros padrão (você pode ajustar)
MODEL_ID = "runwayml/stable-diffusion-v1-5"  # base SD v1.5
CONTROLNET_ID = "lllyasviel/sd-controlnet-openpose"  # controlnet openpose
DEVICE = "cuda" if torch.cuda.is_available() else "cpu"

# pipeline cache globals
_PIPELINE = None
_OP_DETECTOR = None

def get_openpose_detector():
    global _OP_DETECTOR
    if _OP_DETECTOR is None:
        _OP_DETECTOR = OpenposeDetector()
    return _OP_DETECTOR

def load_pipeline():
    """
    Carrega o pipeline ControlNet + Stable Diffusion (com half precision quando possível).
    """
    global _PIPELINE
    if _PIPELINE is not None:
        return _PIPELINE

    # Carregar ControlNet
    controlnet = ControlNetModel.from_pretrained(CONTROLNET_ID, torch_dtype=torch.float16 if DEVICE=="cuda" else torch.float32)
    # Carregar pipeline SD + ControlNet
    pipe = StableDiffusionControlNetPipeline.from_pretrained(
        MODEL_ID,
        controlnet=controlnet,
        safety_checker=None,
        torch_dtype=torch.float16 if DEVICE=="cuda" else torch.float32,
    )
    # usar UniPC scheduler — melhora velocidade/qualidade
    pipe.scheduler = UniPCMultistepScheduler.from_config(pipe.scheduler.config)
    if DEVICE == "cuda":
        pipe.enable_attention_slicing()  # economiza VRAM
        pipe.to("cuda")
    else:
        pipe.to("cpu")

    # reduzir torch_autocast config handled later in inference
    _PIPELINE = pipe
    return _PIPELINE

def remove_background(pil_img: Image.Image) -> Image.Image:
    """
    Remove fundo da imagem da peça usando rembg (retorna RGBA com alpha).
    """
    # rembg expects bytes
    img_bytes = io.BytesIO()
    pil_img.convert("RGBA").save(img_bytes, format="PNG")
    img_bytes = img_bytes.getvalue()
    out = remove(img_bytes)
    # out is bytes of PNG with alpha
    out_img = Image.open(io.BytesIO(out)).convert("RGBA")
    return out_img

def simple_align_garment_to_model(model_img: Image.Image, garment_rgba: Image.Image, pose_keypoints=None) -> Image.Image:
    """
    Faz um alinhamento simples: escala a peça pela distância entre ombros (estimada)
    e cola-a sobre a modelo aproximadamente no torso. Retorna imagem RGBA (com a modelo).
    Isso é só a iniciação — o SD+ControlNet fará o refinamento.
    """
    model = model_img.convert("RGBA")
    g = garment_rgba

    Wm, Hm = model.size
    Wg, Hg = g.size

    # fallback: centragem se não houver keypoints
    if pose_keypoints is None:
        # escala para metade da largura do modelo
        target_w = int(Wm * 0.5)
        scale = target_w / Wg
        new_size = (max(1, int(Wg * scale)), max(1, int(Hg * scale)))
        g_resized = g.resize(new_size, resample=Image.LANCZOS)
        pos = ((Wm - new_size[0]) // 2, int(Hm * 0.28))  # 28% from top as rough torso position
        canvas = model.copy()
        canvas.paste(g_resized, pos, g_resized)
        return canvas

    # se houver keypoints, tentamos usar ombros para dimensionar
    try:
        # keypoints: dict with names->(x,y) in pixel coords (as returned below)
        ls = pose_keypoints.get("left_shoulder")
        rs = pose_keypoints.get("right_shoulder")
        if ls and rs:
            shoulder_dist = math.hypot(rs[0]-ls[0], rs[1]-ls[1])
            # queremos que a peça cubra ~1.4x a largura dos ombros (ajustar conforme peça)
            target_w = int(shoulder_dist * 1.4)
            scale = max(0.1, target_w / Wg)
            new_size = (max(1, int(Wg * scale)), max(1, int(Hg * scale)))
            g_resized = g.resize(new_size, resample=Image.LANCZOS)
            # center position between shoulders, and slightly below
            center_x = int((ls[0] + rs[0]) / 2)
            top_y = int((ls[1] + rs[1]) / 1.8)  # move slightly up/down
            pos = (max(0, center_x - new_size[0]//2), max(0, top_y - new_size[1]//6))
            canvas = model.copy()
            canvas.paste(g_resized, pos, g_resized)
            return canvas
    except Exception:
        pass

    # fallback
    return simple_align_garment_to_model(model_img, garment_rgba, pose_keypoints=None)

def extract_pose_and_keypoints(model_img: Image.Image) -> Tuple[Image.Image, Dict[str, Tuple[int,int]]]:
    """
    Usa controlnet_aux.OpenposeDetector para gerar a pose map (imagem) e tenta retornar
    keypoints úteis (ombros). keypoints dict = {"left_shoulder":(x,y), ...}
    """
    detector = get_openpose_detector()
    # detect returns a PIL image of the pose map; but also returns 'keypoints' structure if requested
    # controlnet_aux OpenposeDetector has method detect which returns images; to get keypoints we call detect_and_return_info
    # We'll attempt to call 'detect' and fallback if not available
    try:
        detected = detector.detect(model_img)
        # detector.detect returns a pose image (PIL)
        pose_image = detected
        # try to get keypoints via internal method if present (may vary by version)
        try:
            info = detector.get_pose(model_img)  # some versions provide get_pose
            # info parsing: try to find shoulders - adapt defensively
            keypoints = {}
            for person in info:
                # each person: list of points or dict depending implementation
                # attempt to parse common formats
                if isinstance(person, dict):
                    if "left_shoulder" in person and "right_shoulder" in person:
                        keypoints["left_shoulder"] = tuple(person["left_shoulder"])
                        keypoints["right_shoulder"] = tuple(person["right_shoulder"])
                        break
                elif isinstance(person, list) or isinstance(person, tuple):
                    # fallback: OpenPose ordering often uses indices:
                    # 2 = right shoulder, 5 = left shoulder OR vice-versa depending on lib.
                    # We'll try both orders defensively
                    try:
                        p2 = person[2]
                        p5 = person[5]
                        # p2/p5 are (x,y,confidence) or similar
                        keypoints["right_shoulder"] = (int(p2[0]), int(p2[1]))
                        keypoints["left_shoulder"] = (int(p5[0]), int(p5[1]))
                        break
                    except Exception:
                        continue
            return pose_image.convert("RGB"), keypoints
        except Exception:
            # if we can't get structured keypoints, just return pose image and empty dict
            return pose_image.convert("RGB"), {}
    except Exception as e:
        # last fallback: return blank pose (grayscale) and empty keypoints
        blank = Image.new("RGB", model_img.size, (255,255,255))
        return blank, {}

def run_pipeline(model_image: Image.Image, garment_image: Image.Image, prompt_extra: str = "") -> Tuple[Image.Image, Dict[str,Any]]:
    """
    Função principal que:
    1) extrai pose (pose_map)
    2) remove fundo da peça (garment) e alinha simplisticamente
    3) monta uma imagem inicial (init_image) com a peça sobre a modelo (RGBA)
    4) chama Stable Diffusion + ControlNet (image2image) usando pose_map como conditioning image
    Retorna: pil_image_result, info_dict
    """
    # Convert PIL to consistent size (we'll resize to 768 on larger side to balance quality/VRAM)
    max_side = 768
    model_img = model_image.convert("RGB")
    W, H = model_img.size
    scale = max_side / max(W, H) if max(W, H) > max_side else 1.0
    if scale != 1.0:
        model_img = model_img.resize((int(W*scale), int(H*scale)), Image.LANCZOS)

    # garment: remove background to get alpha
    garment_rgba = remove_background(garment_image)

    # get pose map and shoulder keypoints
    pose_map, keypoints = extract_pose_and_keypoints(model_img)

    # align garment roughly
    init_composite = simple_align_garment_to_model(model_img, garment_rgba, pose_keypoints=keypoints)

    # prepare pipeline and control image
    pipe = load_pipeline()

    # create prompt: combine prompt_extra with description of garment (basic default)
    prompt = ("photo-realistic fashion try-on, ultra detailed, high resolution, realistic lighting. "
              + (prompt_extra or "garment applied on person, preserve texture and zippers, realistic folds."))

    # convert images to correct formats
    init_image = init_composite.convert("RGB")
    control_image = pose_map.convert("RGB")

    # inference parameters (tune if OOM)
    num_inference_steps = 20
    guidance_scale = 7.5
    strength = 0.75  # image2image strength (how much to change)

    # Run in autocast for fp16 if GPU is available
    generator = torch.Generator(device=DEVICE).manual_seed(torch.randint(0, 2**31 - 1, (1,)).item())

    # Note: Some versions of diffusers expect 'image' and 'control_image' keyword arguments
    # We'll call the pipeline defensively.
    device = DEVICE
    pipe.to(device)

    try:
        # The StableDiffusionControlNetPipeline supports image2image by passing 'image' and 'control_image'
        with torch.autocast(device_type="cuda") if device == "cuda" else torch.cpu.amp.autocast(enabled=False):
            out = pipe(
                prompt=prompt,
                image=init_image,
                control_image=control_image,
                num_inference_steps=num_inference_steps,
                guidance_scale=guidance_scale,
                strength=strength,
                generator=generator
            )
            # out.images is a list
            result_img = out.images[0]
    except TypeError:
        # Some diffusers versions use different signature; try alternate call
        out = pipe(
            prompt=prompt,
            init_image=init_image,
            controlnet_conditioning_image=control_image,
            num_inference_steps=num_inference_steps,
            guidance_scale=guidance_scale,
            strength=strength,
            generator=generator
        )
        result_img = out.images[0]

    info = {
        "model_id": MODEL_ID,
        "controlnet_id": CONTROLNET_ID,
        "steps": num_inference_steps,
        "guidance_scale": guidance_scale,
        "strength": strength
    }
    return result_img, info