app.py

import os
import math
import gradio as gr
import numpy as np
import torch
import safetensors.torch as sf
from datetime import datetime

# Import spaces for GPU decorator
try:
    import spaces
    HF_SPACES_GPU = True
except ImportError:
    HF_SPACES_GPU = False
    # Create a dummy decorator if spaces is not available
    class spaces:
        @staticmethod
        def GPU(func):
            return func

from PIL import Image
from diffusers import StableDiffusionPipeline, StableDiffusionImg2ImgPipeline
from diffusers import AutoencoderKL, UNet2DConditionModel, DDIMScheduler, EulerAncestralDiscreteScheduler, DPMSolverMultistepScheduler
from transformers import CLIPTextModel, CLIPTokenizer
from enum import Enum
import torch.nn as nn
import torch.nn.functional as F
from huggingface_hub import PyTorchModelHubMixin

# Try to import RMBG, fallback to local implementation
try:
    from transformers import pipeline
    rmbg_pipeline = pipeline("image-segmentation", model="briaai/RMBG-1.4", trust_remote_code=True)
    USE_RMBG_PIPELINE = True
except Exception as e:
    print(f"Failed to load RMBG pipeline: {e}")
    USE_RMBG_PIPELINE = False
    try:
        from briarmbg import BriaRMBG, simple_background_removal
    except:
        # Inline simple background removal
        def simple_background_removal(image):
            if isinstance(image, np.ndarray):
                img = image
            else:
                img = np.array(image)
            
            # Simple fallback - return full mask
            gray = np.mean(img, axis=2)
            mask = np.ones_like(gray)
            return mask

# Model setup
sd15_name = 'stablediffusionapi/realistic-vision-v51'

# Better CUDA detection and debugging
print("===== Application Startup at", datetime.now().strftime("%Y-%m-%d %H:%M:%S"), "=====")
print()
print("=== GPU Detection Debug ===")
print(f"PyTorch version: {torch.__version__}")
print(f"Hugging Face Spaces GPU support: {HF_SPACES_GPU}")
print(f"CUDA available: {torch.cuda.is_available()}")
if torch.cuda.is_available():
    print(f"CUDA version: {torch.version.cuda}")
    print(f"GPU count: {torch.cuda.device_count()}")
    print(f"Current GPU: {torch.cuda.current_device()}")
    print(f"GPU name: {torch.cuda.get_device_name()}")
    print("✅ GPU detected and available!")
else:
    print("❌ CUDA not available - checking reasons...")
    try:
        import subprocess
        result = subprocess.run(['nvidia-smi'], capture_output=True, text=True)
        if result.returncode == 0:
            print("nvidia-smi works, GPU hardware detected")
            print("Issue might be with PyTorch CUDA installation")
        else:
            print("nvidia-smi failed, no GPU hardware detected")
    except:
        print("nvidia-smi command not found")
    
    if HF_SPACES_GPU:
        print("🔄 Running on Hugging Face Spaces with @spaces.GPU decorator")
        print("   GPU will be allocated when GPU-decorated functions are called")
    else:
        print()
        print("🚨 WARNING: This application requires GPU to run properly!")
        print("📋 To fix this issue:")
        print("   1. Go to your Space settings: https://huggingface.co/spaces/GreenGoat/IClight-demo/settings")
        print("   2. In the Hardware section, select 'GPU basic' or higher")
        print("   3. Make sure your Hugging Face account is verified")
        print("   4. Check if you have available GPU quota")
    print()

device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
print(f"Selected device: {device}")
print("=== End GPU Debug ===")

print(f"Using device: {device}")
print("Loading models...")

# Initialize models
tokenizer = CLIPTokenizer.from_pretrained(sd15_name, subfolder="tokenizer")
text_encoder = CLIPTextModel.from_pretrained(sd15_name, subfolder="text_encoder")
vae = AutoencoderKL.from_pretrained(sd15_name, subfolder="vae")
unet = UNet2DConditionModel.from_pretrained(sd15_name, subfolder="unet")

# Modify UNet for IC-Light
with torch.no_grad():
    new_conv_in = torch.nn.Conv2d(12, unet.conv_in.out_channels, unet.conv_in.kernel_size, unet.conv_in.stride, unet.conv_in.padding)
    new_conv_in.weight.zero_()
    new_conv_in.weight[:, :4, :, :].copy_(unet.conv_in.weight)
    new_conv_in.bias = unet.conv_in.bias
    unet.conv_in = new_conv_in

unet_original_forward = unet.forward

def hooked_unet_forward(sample, timestep, encoder_hidden_states, **kwargs):
    c_concat = kwargs['cross_attention_kwargs']['concat_conds'].to(sample)
    c_concat = torch.cat([c_concat] * (sample.shape[0] // c_concat.shape[0]), dim=0)
    new_sample = torch.cat([sample, c_concat], dim=1)
    kwargs['cross_attention_kwargs'] = {}
    return unet_original_forward(new_sample, timestep, encoder_hidden_states, **kwargs)

unet.forward = hooked_unet_forward

# Load IC-Light weights
model_path = './iclight_sd15_fbc.safetensors'
if not os.path.exists(model_path):
    print("Downloading IC-Light model...")
    try:
        from huggingface_hub import hf_hub_download
        model_path = hf_hub_download(
            repo_id="lllyasviel/ic-light",
            filename="iclight_sd15_fbc.safetensors"
        )
    except Exception as e:
        print(f"Failed to download with hf_hub_download: {e}")
        # Fallback to torch.hub
        from torch.hub import download_url_to_file
        download_url_to_file(url='https://huggingface.co/lllyasviel/ic-light/resolve/main/iclight_sd15_fbc.safetensors', dst=model_path)

sd_offset = sf.load_file(model_path)
sd_origin = unet.state_dict()
sd_merged = {k: sd_origin[k] + sd_offset[k] for k in sd_origin.keys()}
unet.load_state_dict(sd_merged, strict=True)
del sd_offset, sd_origin, sd_merged

# Move models to device
text_encoder = text_encoder.to(device=device, dtype=torch.float16)
vae = vae.to(device=device, dtype=torch.bfloat16)
unet = unet.to(device=device, dtype=torch.float16)

# Scheduler
scheduler = DPMSolverMultistepScheduler(
    num_train_timesteps=1000,
    beta_start=0.00085,
    beta_end=0.012,
    algorithm_type="sde-dpmsolver++",
    use_karras_sigmas=True,
    steps_offset=1
)

# Pipelines
t2i_pipe = StableDiffusionPipeline(
    vae=vae,
    text_encoder=text_encoder,
    tokenizer=tokenizer,
    unet=unet,
    scheduler=scheduler,
    safety_checker=None,
    requires_safety_checker=False,
    feature_extractor=None
)

i2i_pipe = StableDiffusionImg2ImgPipeline(
    vae=vae,
    text_encoder=text_encoder,
    tokenizer=tokenizer,
    unet=unet,
    scheduler=scheduler,
    safety_checker=None,
    requires_safety_checker=False,
    feature_extractor=None
)

print("Models loaded successfully!")

@torch.inference_mode()
def encode_prompt_inner(txt: str):
    max_length = tokenizer.model_max_length
    chunk_length = tokenizer.model_max_length - 2
    id_start = tokenizer.bos_token_id
    id_end = tokenizer.eos_token_id
    id_pad = id_end

    def pad(x, p, i):
        return x[:i] if len(x) >= i else x + [p] * (i - len(x))

    tokens = tokenizer(txt, truncation=False, add_special_tokens=False)["input_ids"]
    chunks = [[id_start] + tokens[i: i + chunk_length] + [id_end] for i in range(0, len(tokens), chunk_length)]
    chunks = [pad(ck, id_pad, max_length) for ck in chunks]

    token_ids = torch.tensor(chunks).to(device=device, dtype=torch.int64)
    conds = text_encoder(token_ids).last_hidden_state

    return conds

@torch.inference_mode()
def encode_prompt_pair(positive_prompt, negative_prompt):
    c = encode_prompt_inner(positive_prompt)
    uc = encode_prompt_inner(negative_prompt)

    c_len = float(len(c))
    uc_len = float(len(uc))
    max_count = max(c_len, uc_len)
    c_repeat = int(math.ceil(max_count / c_len))
    uc_repeat = int(math.ceil(max_count / uc_len))
    max_chunk = max(len(c), len(uc))

    c = torch.cat([c] * c_repeat, dim=0)[:max_chunk]
    uc = torch.cat([uc] * uc_repeat, dim=0)[:max_chunk]

    c = torch.cat([p[None, ...] for p in c], dim=1)
    uc = torch.cat([p[None, ...] for p in uc], dim=1)

    return c, uc

@torch.inference_mode()
def pytorch2numpy(imgs, quant=True):
    results = []
    for x in imgs:
        y = x.movedim(0, -1)
        if quant:
            y = y * 127.5 + 127.5
            y = y.detach().float().cpu().numpy().clip(0, 255).astype(np.uint8)
        else:
            y = y * 0.5 + 0.5
            y = y.detach().float().cpu().numpy().clip(0, 1).astype(np.float32)
        results.append(y)
    return results

@torch.inference_mode()
def numpy2pytorch(imgs):
    h = torch.from_numpy(np.stack(imgs, axis=0)).float() / 127.0 - 1.0
    h = h.movedim(-1, 1)
    return h

def resize_and_center_crop(image, target_width, target_height):
    pil_image = Image.fromarray(image)
    original_width, original_height = pil_image.size
    scale_factor = max(target_width / original_width, target_height / original_height)
    new_width = int(original_width * scale_factor)
    new_height = int(original_height * scale_factor)
    pil_image = pil_image.resize((new_width, new_height), Image.LANCZOS)
    left = (new_width - target_width) / 2
    top = (new_height - target_height) / 2
    right = (new_width + target_width) / 2
    bottom = (new_height + target_height) / 2
    pil_image = pil_image.crop((left, top, right, bottom))
    return np.array(pil_image)

def resize_without_crop(image, target_width, target_height):
    pil_image = Image.fromarray(image)
    pil_image = pil_image.resize((target_width, target_height), Image.LANCZOS)
    return np.array(pil_image)

@spaces.GPU
@torch.inference_mode()
def run_rmbg(img, sigma=0.0):
    # Simplified background removal
    if USE_RMBG_PIPELINE:
        # Using transformers pipeline
        try:
            result = rmbg_pipeline(Image.fromarray(img))
            mask = np.array(result['mask'])
            if len(mask.shape) == 3:
                mask = mask[:, :, 0]
            mask = mask.astype(np.float32) / 255.0
        except Exception as e:
            print(f"RMBG pipeline failed: {e}, using fallback")
            mask = simple_background_removal(img)
    else:
        # Using simple background removal
        mask = simple_background_removal(img)
    
    # Apply sigma smoothing
    if sigma > 0:
        try:
            from scipy import ndimage
            mask = ndimage.gaussian_filter(mask, sigma=sigma)
        except ImportError:
            # Fallback if scipy is not available
            pass
    
    # Create RGBA output
    result = np.dstack((img, (mask * 255).astype(np.uint8)))
    return img, mask

class BGSource(Enum):
    UPLOAD = "Use Background Image"
    UPLOAD_FLIP = "Use Flipped Background Image"
    LEFT = "Left Light"
    RIGHT = "Right Light"
    TOP = "Top Light"
    BOTTOM = "Bottom Light"
    GREY = "Ambient"

@spaces.GPU
@torch.inference_mode()
def process(input_fg, input_bg, prompt, image_width, image_height, num_samples, seed, steps, a_prompt, n_prompt, cfg, highres_scale, highres_denoise, bg_source):
    bg_source = BGSource(bg_source)

    if bg_source == BGSource.UPLOAD:
        pass
    elif bg_source == BGSource.UPLOAD_FLIP:
        input_bg = np.fliplr(input_bg)
    elif bg_source == BGSource.GREY:
        input_bg = np.zeros(shape=(image_height, image_width, 3), dtype=np.uint8) + 64
    elif bg_source == BGSource.LEFT:
        gradient = np.linspace(224, 32, image_width)
        image = np.tile(gradient, (image_height, 1))
        input_bg = np.stack((image,) * 3, axis=-1).astype(np.uint8)
    elif bg_source == BGSource.RIGHT:
        gradient = np.linspace(32, 224, image_width)
        image = np.tile(gradient, (image_height, 1))
        input_bg = np.stack((image,) * 3, axis=-1).astype(np.uint8)
    elif bg_source == BGSource.TOP:
        gradient = np.linspace(224, 32, image_height)[:, None]
        image = np.tile(gradient, (1, image_width))
        input_bg = np.stack((image,) * 3, axis=-1).astype(np.uint8)
    elif bg_source == BGSource.BOTTOM:
        gradient = np.linspace(32, 224, image_height)[:, None]
        image = np.tile(gradient, (1, image_width))
        input_bg = np.stack((image,) * 3, axis=-1).astype(np.uint8)
    else:
        raise ValueError('Wrong background source!')

    rng = torch.Generator(device=device).manual_seed(seed)

    fg = resize_and_center_crop(input_fg, image_width, image_height)
    bg = resize_and_center_crop(input_bg, image_width, image_height)
    concat_conds = numpy2pytorch([fg, bg]).to(device=vae.device, dtype=vae.dtype)
    concat_conds = vae.encode(concat_conds).latent_dist.mode() * vae.config.scaling_factor
    concat_conds = torch.cat([c[None, ...] for c in concat_conds], dim=1)

    conds, unconds = encode_prompt_pair(positive_prompt=prompt + ', ' + a_prompt, negative_prompt=n_prompt)

    latents = t2i_pipe(
        prompt_embeds=conds,
        negative_prompt_embeds=unconds,
        width=image_width,
        height=image_height,
        num_inference_steps=steps,
        num_images_per_prompt=num_samples,
        generator=rng,
        output_type='latent',
        guidance_scale=cfg,
        cross_attention_kwargs={'concat_conds': concat_conds},
    ).images.to(vae.dtype) / vae.config.scaling_factor

    pixels = vae.decode(latents).sample
    pixels = pytorch2numpy(pixels)  # Use default quant=True for first pass
    
    # Always perform highres processing like the original code
    pixels = [resize_without_crop(
        image=p,
        target_width=int(round(image_width * highres_scale / 64.0) * 64),
        target_height=int(round(image_height * highres_scale / 64.0) * 64))
    for p in pixels]

    pixels = numpy2pytorch(pixels).to(device=vae.device, dtype=vae.dtype)
    latents = vae.encode(pixels).latent_dist.mode() * vae.config.scaling_factor
    latents = latents.to(device=unet.device, dtype=unet.dtype)

    image_height, image_width = latents.shape[2] * 8, latents.shape[3] * 8
    fg = resize_and_center_crop(input_fg, image_width, image_height)
    bg = resize_and_center_crop(input_bg, image_width, image_height)
    concat_conds = numpy2pytorch([fg, bg]).to(device=vae.device, dtype=vae.dtype)
    concat_conds = vae.encode(concat_conds).latent_dist.mode() * vae.config.scaling_factor
    concat_conds = torch.cat([c[None, ...] for c in concat_conds], dim=1)

    latents = i2i_pipe(
        image=latents,
        strength=highres_denoise,
        prompt_embeds=conds,
        negative_prompt_embeds=unconds,
        width=image_width,
        height=image_height,
        num_inference_steps=int(round(steps / highres_denoise)),
        num_images_per_prompt=num_samples,
        generator=rng,
        output_type='latent',
        guidance_scale=cfg,
        cross_attention_kwargs={'concat_conds': concat_conds},
    ).images.to(vae.dtype) / vae.config.scaling_factor

    pixels = vae.decode(latents).sample
    pixels = pytorch2numpy(pixels, quant=False)  # Return 0-1 range floats for final result

    return pixels, [fg, bg]

@spaces.GPU
@torch.inference_mode()
def process_relight(input_fg, input_bg, prompt, image_width, image_height, num_samples, seed, steps, a_prompt, n_prompt, cfg, highres_scale, highres_denoise, bg_source):
    try:
        # Input validation
        if input_fg is None:
            error_msg = "❌ Please upload a foreground image"
            print(error_msg)
            raise gr.Error(error_msg)
        if input_bg is None and bg_source == "Use Background Image":
            error_msg = "❌ Please upload a background image or choose a lighting direction"
            print(error_msg)
            raise gr.Error(error_msg)
        
        # Handle empty prompt - provide default when using background image
        if not prompt.strip():
            if bg_source == "Use Background Image" or bg_source == "Use Flipped Background Image":
                # When using background image as light source, use a generic default prompt
                prompt = "best quality, detailed"
                print(f"Using default prompt for background lighting: {prompt}")
            else:
                error_msg = "❌ Please enter a prompt"
                print(error_msg)
                raise gr.Error(error_msg)
        
        print(f"Processing with device: {device}")
        print(f"Input shapes - FG: {input_fg.shape}, BG: {input_bg.shape if input_bg is not None else 'None'}")
        
        # Optimize for Hugging Face free GPU (limited memory)
        if device.type == 'cuda':
            # Limit image size for free GPU tier
            max_size = 768  # Increased for GPU but still conservative
            if image_width > max_size or image_height > max_size:
                scale = min(max_size / image_width, max_size / image_height)
                image_width = int(image_width * scale // 64) * 64  # Keep multiple of 64
                image_height = int(image_height * scale // 64) * 64
                print(f"Reduced image size for GPU memory: {image_width}x{image_height}")
            
            # Disable highres for free tier to save memory
            if highres_scale > 1.0:
                highres_scale = 1.0
                print("Disabled highres scaling to save GPU memory")
                
        elif device.type == 'cpu':
            # Limit image size for CPU processing
            max_size = 512
            if image_width > max_size or image_height > max_size:
                image_width = min(image_width, max_size)
                image_height = min(image_height, max_size)
                print(f"Reduced image size for CPU: {image_width}x{image_height}")
            
            # Limit number of samples for CPU
            if num_samples > 1:
                num_samples = 1
                print("Reduced num_samples to 1 for CPU processing")
        
        print("Running background removal...")
        try:
            input_fg, matting = run_rmbg(input_fg)
            print("Background removal completed successfully")
        except Exception as e:
            print(f"Background removal failed: {e}")
            # Continue without background removal
            matting = np.ones((input_fg.shape[0], input_fg.shape[1]), dtype=np.float32)
        
        print("Starting main processing...")
        try:
            results, extra_images = process(input_fg, input_bg, prompt, image_width, image_height, num_samples, seed, steps, a_prompt, n_prompt, cfg, highres_scale, highres_denoise, bg_source)
            print("Main processing completed successfully")
        except Exception as e:
            error_msg = f"❌ Processing failed: {str(e)}"
            print(error_msg)
            import traceback
            traceback.print_exc()
            raise gr.Error(error_msg)
        
        print("Converting results...")
        try:
            results = [(x * 255.0).clip(0, 255).astype(np.uint8) for x in results]
            print("Results converted successfully")
        except Exception as e:
            error_msg = f"❌ Result conversion failed: {str(e)}"
            print(error_msg)
            raise gr.Error(error_msg)
        
        print("Processing completed successfully!")
        return results + extra_images
        
    except gr.Error:
        # Re-raise Gradio errors to show them in the UI
        raise
    except Exception as e:
        error_msg = f"❌ Unexpected error: {str(e)}"
        print(error_msg)
        import traceback
        traceback.print_exc()
        raise gr.Error(error_msg)

# Quick prompts for easy testing
quick_prompts = [
    'beautiful woman, cinematic lighting',
    'handsome man, cinematic lighting',
    'beautiful woman, natural lighting',
    'handsome man, natural lighting',
    'beautiful woman, neo punk lighting, cyberpunk',
    'handsome man, neo punk lighting, cyberpunk',
]
quick_prompts = [[x] for x in quick_prompts]

# Gradio Interface
def create_demo():
    with gr.Blocks(title="IC-Light Background Conditional Relighting") as demo:
        gr.Markdown("## IC-Light: Relighting with Foreground and Background Condition")
        gr.Markdown("Upload a foreground image and background image (or choose lighting direction) to perform relighting.")
        
        with gr.Row():
            with gr.Column():
                with gr.Row():
                    input_fg = gr.Image(label="Foreground Image", height=400, type="numpy")
                    input_bg = gr.Image(label="Background Image", height=400, type="numpy")
                
                prompt = gr.Textbox(label="Prompt", value="beautiful woman, cinematic lighting")
                bg_source = gr.Radio(
                    choices=[e.value for e in BGSource],
                    value=BGSource.UPLOAD.value,
                    label="Background Source"
                )
                
                example_prompts = gr.Dataset(
                    samples=quick_prompts, 
                    label='Quick Prompts', 
                    components=[prompt]
                )
                
                relight_button = gr.Button(value="✨ Relight Image", variant="primary")
                
                with gr.Accordion("Advanced Settings", open=False):
                    with gr.Row():
                        num_samples = gr.Slider(label="Number of Images", minimum=1, maximum=4, value=1, step=1)
                        seed = gr.Number(label="Seed", value=12345, precision=0)
                    with gr.Row():
                        image_width = gr.Slider(label="Width", minimum=256, maximum=1024, value=512, step=64)
                        image_height = gr.Slider(label="Height", minimum=256, maximum=1024, value=640, step=64)
                    with gr.Row():
                        steps = gr.Slider(label="Steps", minimum=1, maximum=50, value=20, step=1)
                        cfg = gr.Slider(label="CFG Scale", minimum=1.0, maximum=20.0, value=7.0, step=0.1)
                    with gr.Row():
                        highres_scale = gr.Slider(label="Highres Scale", minimum=1.0, maximum=2.0, value=1.5, step=0.1)
                        highres_denoise = gr.Slider(label="Highres Denoise", minimum=0.1, maximum=0.9, value=0.5, step=0.1)
                    
                    a_prompt = gr.Textbox(label="Additional Prompt", value='best quality')
                    n_prompt = gr.Textbox(label="Negative Prompt", value='lowres, bad anatomy, bad hands, cropped, worst quality')
            
            with gr.Column():
                result_gallery = gr.Gallery(label='Results', height=600, columns=2, rows=2)
        
        # Event handlers
        inputs = [input_fg, input_bg, prompt, image_width, image_height, num_samples, seed, steps, a_prompt, n_prompt, cfg, highres_scale, highres_denoise, bg_source]
        relight_button.click(
            fn=process_relight, 
            inputs=inputs, 
            outputs=[result_gallery],
            show_progress=True
        )
        example_prompts.click(lambda x: x[0], inputs=example_prompts, outputs=prompt, show_progress=False)
        
        # Examples - temporarily disabled due to missing image files
        # gr.Examples(
        #     examples=[
        #         ["examples/person1.jpg", "examples/bg1.jpg", "beautiful woman, cinematic lighting", "Use Background Image"],
        #         ["examples/person2.jpg", None, "handsome man, dramatic lighting", "Left Light"],
        #     ],
        #     inputs=[input_fg, input_bg, prompt, bg_source],
        #     outputs=[result_gallery],
        #     fn=process_relight,
        #     cache_examples=False,
        # )
    
    return demo

if __name__ == "__main__":
    demo = create_demo()
    demo.queue(max_size=20)
    demo.launch(
        server_name='0.0.0.0', 
        server_port=7860,
        show_error=True,
        share=False
    )