import base64
from io import BytesIO
import spaces
import gradio as gr
import torch
from PIL import Image, ImageChops
from diffusers import Flux2KleinPipeline
from utils import process_source, process_reference, paste_back, binarize_mask
pipe = Flux2KleinPipeline.from_pretrained(
"black-forest-labs/FLUX.2-klein-base-4B", torch_dtype=torch.bfloat16
)
pipe.to("cuda")
pipe.load_lora_weights("LiXiY/Easy-Insert")
PROMPT = "Replace the white mask of image1 with the content in image2. Preserving the background, lighting, and surrounding elements, maintain a seamless and natural result."
# ===================== Example Data =====================
CANVAS_W, CANVAS_H = 1024, 1024
EXAMPLES = [
("examples/background_image/1.png", "examples/insert_mask/1.png", "examples/ref_image/1.png", "examples/ref_mask/1.png"),
("examples/background_image/2.png", "examples/insert_mask/2.png", "examples/ref_image/2.png", "examples/ref_mask/2.png"),
("examples/background_image/3.png", "examples/insert_mask/3.png", "examples/ref_image/3.png", "examples/ref_mask/3.png"),
("examples/background_image/4.png", "examples/insert_mask/4.png", "examples/ref_image/4.png", "examples/ref_mask/4.png"),
]
# ===================== Thumbnail HTML Generation =====================
def img_to_b64(path, size=(240, 240)):
img = Image.open(path).convert("RGB")
# Preserve the original aspect ratio; fit inside `size`. 2x the display
# width (110 CSS px) so thumbnails stay sharp on HiDPI screens.
img.thumbnail(size, Image.LANCZOS)
buf = BytesIO()
img.save(buf, format="PNG")
return base64.b64encode(buf.getvalue()).decode()
def build_examples_html():
row_pairs = [EXAMPLES[0:2], EXAMPLES[2:4]]
html = '
'
for row_idx, row_examples in enumerate(row_pairs):
html += '
'
for col_idx, (bg, mask, ref, ref_mask) in enumerate(row_examples):
i = row_idx * 2 + col_idx
bg_b64 = img_to_b64(bg)
mask_b64 = img_to_b64(mask)
ref_b64 = img_to_b64(ref)
ref_mask_b64 = img_to_b64(ref_mask)
html += f'''
Example {i + 1}
Background
Mask
Reference
Ref Mask
'''
html += '
'
html += '
'
return html
# ===================== Utility Functions =====================
def extract_mask_from_layers(layers, target_size):
mask = Image.new("L", target_size, 0)
for layer in layers:
if layer is not None:
layer_rgba = layer.convert("RGBA").resize(target_size)
alpha = layer_rgba.split()[3]
alpha_binary = alpha.point(lambda x: 255 if x > 0 else 0)
mask = Image.composite(Image.new("L", target_size, 255), mask, alpha_binary)
return mask
EDITOR_SIZE = 800
# Matches Gradio light-mode image editor background so padded examples blend in.
EDITOR_BG_COLOR = (243, 244, 246)
def fit_transform(img, editor_size=EDITOR_SIZE):
"""Display transform the fixed-canvas editor applies to `img`: scale to fit
inside the square canvas, then center-pad. Returns (disp_size, pad)."""
scale = min(editor_size / img.width, editor_size / img.height, 1.0)
disp_size = (
max(1, round(img.width * scale)),
max(1, round(img.height * scale)),
)
pad = (
(editor_size - disp_size[0]) // 2,
(editor_size - disp_size[1]) // 2,
)
return disp_size, pad
def build_display_bg(bg, editor_size=EDITOR_SIZE):
"""Rebuild the padded square background the editor shows for `bg`."""
disp_size, pad = fit_transform(bg, editor_size)
canvas = Image.new("RGB", (editor_size, editor_size), EDITOR_BG_COLOR)
canvas.paste(bg.convert("RGB").resize(disp_size, Image.LANCZOS), pad)
return canvas
def images_close(a, b, tol_frac=0.001):
"""True if two RGB images are (nearly) pixel-identical."""
if a.size != b.size:
return False
diff = ImageChops.difference(a.convert("RGB"), b.convert("RGB"))
changed = sum(diff.histogram()[1:])
return changed <= tol_frac * a.size[0] * a.size[1] * 3
def make_editor_value(bg_pil, mask_pil=None, editor_size=EDITOR_SIZE):
"""Build an ImageEditor value on a fixed square canvas.
The background is downscaled to fit inside `editor_size` and padded to a
square; the mask is padded the same way and sent as a real layer. This
keeps the mask and background perfectly aligned in the frontend (both are
800x800 and the padding is identical), while the full-resolution image and
exact mask are kept in gr.State for inference.
Returns (editor_value, sent_mask, transform).
"""
bg = bg_pil.convert("RGB")
disp_size, pad = fit_transform(bg, editor_size)
bg_sq = build_display_bg(bg, editor_size)
transform = {
"editor_size": editor_size,
"disp_size": disp_size,
"pad": pad,
}
if mask_pil is None:
# composite=None avoids saving a second, redundant image file. The editor
# can render the background directly; if it needs a composite it will build
# it from background + (empty) layers.
return {"background": bg_sq, "layers": [], "composite": None}, None, transform
mask = mask_pil.convert("L").resize(disp_size, Image.NEAREST)
mask_sq = Image.new("L", (editor_size, editor_size), 0)
mask_sq.paste(mask, pad)
transparent = Image.new("RGBA", (editor_size, editor_size), (0, 0, 0, 0))
# Semi-transparent white (alpha 153 ≈ 60%) so the background shows through
# the loaded mask. Mask extraction binarizes alpha > 0, so this still
# resolves to a solid mask at inference time.
white_solid = Image.new("RGBA", (editor_size, editor_size), (255, 255, 255, 153))
mask_layer = Image.composite(white_solid, transparent, mask_sq)
return {"background": bg_sq, "layers": [mask_layer], "composite": None}, mask_sq, transform
def masks_close(a, b, tol_frac=0.0):
"""True if two binarized masks are pixel-identical (within tol_frac).
The fixed-canvas PNG round-trip is exact, so we treat any non-zero
difference as a real user edit."""
if a.size != b.size:
b = b.resize(a.size, Image.NEAREST)
diff = ImageChops.difference(binarize_mask(a), binarize_mask(b))
changed = sum(diff.histogram()[1:])
return changed <= tol_frac * a.size[0] * a.size[1]
def load_example(idx):
bg_path, mask_path, ref_path, ref_mask_path = EXAMPLES[idx]
bg_img = Image.open(bg_path).convert("RGB")
ref_img = Image.open(ref_path).convert("RGB")
base_val, base_sent, base_transform = make_editor_value(
bg_img, Image.open(mask_path)
)
ref_val, ref_sent, ref_transform = make_editor_value(
ref_img, Image.open(ref_mask_path)
)
# Keep the pristine full-res image + exact mask in State. The editor value
# is only for display; its small size avoids slow frontend re-uploads.
base_state = (bg_img, Image.open(mask_path).convert("L"), base_sent, base_transform)
ref_state = (ref_img, Image.open(ref_mask_path).convert("L"), ref_sent, ref_transform)
return base_val, ref_val, base_state, ref_state
def load_ex1():
return load_example(0)
def load_ex2():
return load_example(1)
def load_ex3():
return load_example(2)
def load_ex4():
return load_example(3)
# ===================== Generation Function =====================
def resolve_source(editor_value, state):
"""Return (full_res_image, full_res_mask) for one editor.
`state` is (image, mask, sent_mask, transform) captured when an example was
loaded. The editor value's background is display-only, so we don't trust it
blindly: rebuild the padded display image from the state's pristine image
and compare it with what the editor currently shows. If they match, the
example is still loaded and the state's full-res image (+ exact mask) is
used. If they differ, the user uploaded a different image - the editor
background is then the full-res original, and the brush strokes are mapped
back through the fixed-canvas transform.
"""
bg_ed = editor_value.get("background")
if bg_ed is None:
return None, None
img_ed = bg_ed.convert("RGB")
ed_mask = extract_mask_from_layers(
editor_value.get("layers", []), (EDITOR_SIZE, EDITOR_SIZE)
)
if state is not None:
img, full_mask, sent_mask, transform = state
img = img.convert("RGB")
if images_close(img_ed, build_display_bg(img)):
if full_mask is not None:
# Example flow: exact full-res mask is in state. If the editor's
# layer mask still matches what we sent (user hasn't brushed or
# erased), use the state mask directly - bit-exact with inference.py.
if sent_mask is not None and masks_close(ed_mask, sent_mask):
return img, binarize_mask(full_mask)
# User edited the example's mask (erased and/or brushed): crop the
# padding and map the edited mask back to the original resolution.
if transform:
ed_mask_bin = binarize_mask(ed_mask)
px, py = transform["pad"]
dw, dh = transform["disp_size"]
cropped = ed_mask_bin.crop((px, py, px + dw, py + dh))
return img, binarize_mask(cropped.resize(img.size, Image.NEAREST))
# Upload flow: the editor background is the user's full-res original.
disp_size, pad = fit_transform(img_ed)
ed_mask_bin = binarize_mask(ed_mask)
cropped = ed_mask_bin.crop((pad[0], pad[1], pad[0] + disp_size[0], pad[1] + disp_size[1]))
return img_ed, binarize_mask(cropped.resize(img_ed.size, Image.NEAREST))
@spaces.GPU
def run_local(base, ref, base_state, ref_state, seed, num_inference_steps, cfg_scale):
if base is None or ref is None or not isinstance(base, dict) or not isinstance(ref, dict):
return None, gr.update(visible=False)
pil_bg, pil_mask = resolve_source(base, base_state)
pil_ref, pil_ref_mask = resolve_source(ref, ref_state)
if pil_bg is None or pil_ref is None:
return None, gr.update(visible=False)
if pil_mask.getextrema() == (0, 0) or pil_ref_mask.getextrema() == (0, 0):
error_html = """
⚠️ Please draw the mask on BOTH the background image and the reference image first, or click an example!