Spaces:
Runtime error
Runtime error
Update generator.py
Browse files- generator.py +787 -252
generator.py
CHANGED
|
@@ -1,30 +1,36 @@
|
|
| 1 |
"""
|
| 2 |
-
Generation logic for Pixagram AI Pixel Art Generator
|
| 3 |
-
UPDATED VERSION with InstantID pipeline integration
|
| 4 |
"""
|
|
|
|
| 5 |
import torch
|
| 6 |
import numpy as np
|
| 7 |
import cv2
|
| 8 |
from PIL import Image
|
| 9 |
-
import
|
|
|
|
|
|
|
| 10 |
|
| 11 |
from config import (
|
| 12 |
-
device, dtype, TRIGGER_WORD,
|
| 13 |
-
ADAPTIVE_THRESHOLDS, ADAPTIVE_PARAMS, CAPTION_CONFIG
|
| 14 |
)
|
| 15 |
from utils import (
|
| 16 |
-
sanitize_text, enhanced_color_match, color_match,
|
| 17 |
-
get_demographic_description, calculate_optimal_size,
|
| 18 |
)
|
| 19 |
from models import (
|
| 20 |
-
load_face_analysis, load_depth_detector, load_controlnets,
|
| 21 |
-
load_sdxl_pipeline,
|
| 22 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 23 |
)
|
| 24 |
|
| 25 |
|
| 26 |
class RetroArtConverter:
|
| 27 |
-
"""Main class for retro art generation
|
| 28 |
|
| 29 |
def __init__(self):
|
| 30 |
self.device = device
|
|
@@ -33,72 +39,186 @@ class RetroArtConverter:
|
|
| 33 |
'custom_checkpoint': False,
|
| 34 |
'lora': False,
|
| 35 |
'instantid': False,
|
| 36 |
-
'
|
|
|
|
|
|
|
|
|
|
|
|
|
| 37 |
}
|
|
|
|
| 38 |
|
| 39 |
-
#
|
| 40 |
self.face_app, self.face_detection_enabled = load_face_analysis()
|
| 41 |
|
| 42 |
-
# Load
|
| 43 |
-
self.
|
| 44 |
-
self.models_loaded['
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 45 |
|
| 46 |
-
#
|
| 47 |
-
|
| 48 |
-
controlnets = [controlnet_instantid, controlnet_depth]
|
| 49 |
-
self.models_loaded['instantid'] = True
|
| 50 |
|
| 51 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 52 |
|
| 53 |
-
# Load SDXL pipeline with InstantID (handles IP-Adapter internally)
|
| 54 |
-
self.pipe, checkpoint_success = load_sdxl_pipeline(controlnets)
|
| 55 |
self.models_loaded['custom_checkpoint'] = checkpoint_success
|
| 56 |
|
| 57 |
-
# Load
|
| 58 |
-
lora_success =
|
| 59 |
self.models_loaded['lora'] = lora_success
|
| 60 |
|
| 61 |
-
# Setup
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 62 |
self.compel, self.use_compel = setup_compel(self.pipe)
|
|
|
|
| 63 |
|
| 64 |
-
# Setup scheduler
|
| 65 |
setup_scheduler(self.pipe)
|
| 66 |
|
| 67 |
-
# Optimize
|
| 68 |
optimize_pipeline(self.pipe)
|
| 69 |
|
| 70 |
# Load caption model
|
| 71 |
self.caption_processor, self.caption_model, self.caption_enabled, self.caption_model_type = load_caption_model()
|
| 72 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 73 |
# Set CLIP skip
|
| 74 |
set_clip_skip(self.pipe)
|
| 75 |
|
| 76 |
-
#
|
|
|
|
|
|
|
|
|
|
|
|
|
| 77 |
self._print_status()
|
| 78 |
|
| 79 |
-
print(" [OK]
|
| 80 |
|
| 81 |
def _print_status(self):
|
| 82 |
"""Print model loading status"""
|
| 83 |
print("\n=== MODEL STATUS ===")
|
| 84 |
for model, loaded in self.models_loaded.items():
|
| 85 |
-
|
| 86 |
-
|
| 87 |
-
|
| 88 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 89 |
print("===================\n")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 90 |
|
| 91 |
def get_depth_map(self, image):
|
| 92 |
-
"""
|
| 93 |
-
|
|
|
|
|
|
|
|
|
|
| 94 |
try:
|
| 95 |
if image.mode != 'RGB':
|
| 96 |
image = image.convert('RGB')
|
| 97 |
|
| 98 |
-
|
| 99 |
-
orig_width
|
|
|
|
| 100 |
|
| 101 |
-
# Use multiples of 64
|
| 102 |
target_width = int((orig_width // 64) * 64)
|
| 103 |
target_height = int((orig_height // 64) * 64)
|
| 104 |
|
|
@@ -108,39 +228,110 @@ class RetroArtConverter:
|
|
| 108 |
size_for_depth = (int(target_width), int(target_height))
|
| 109 |
image_for_depth = image.resize(size_for_depth, Image.LANCZOS)
|
| 110 |
|
| 111 |
-
|
| 112 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 113 |
|
| 114 |
-
|
| 115 |
-
|
| 116 |
-
depth_image = depth_image.resize(safe_image_size(image), Image.LANCZOS)
|
| 117 |
-
# --- END FIX ---
|
| 118 |
|
| 119 |
-
print(f"[DEPTH] Generated depth map: {depth_image.size}")
|
| 120 |
-
return depth_image, depth_array
|
| 121 |
except Exception as e:
|
| 122 |
-
print(f"[DEPTH]
|
| 123 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 124 |
else:
|
| 125 |
-
print("[DEPTH]
|
| 126 |
-
|
|
|
|
|
|
|
|
|
|
| 127 |
|
| 128 |
-
|
|
|
|
| 129 |
"""Add trigger word to prompt if not present"""
|
| 130 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 131 |
if not prompt or not prompt.strip():
|
| 132 |
-
return
|
| 133 |
-
|
|
|
|
| 134 |
return prompt
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 135 |
|
| 136 |
def detect_face_quality(self, face):
|
| 137 |
-
"""
|
|
|
|
|
|
|
|
|
|
| 138 |
try:
|
| 139 |
bbox = face.bbox
|
| 140 |
face_size = (bbox[2] - bbox[0]) * (bbox[3] - bbox[1])
|
| 141 |
det_score = float(face.det_score) if hasattr(face, 'det_score') else 1.0
|
| 142 |
|
| 143 |
-
# Small face -> boost preservation
|
| 144 |
if face_size < ADAPTIVE_THRESHOLDS['small_face_size']:
|
| 145 |
return ADAPTIVE_PARAMS['small_face'].copy()
|
| 146 |
|
|
@@ -148,7 +339,7 @@ class RetroArtConverter:
|
|
| 148 |
elif det_score < ADAPTIVE_THRESHOLDS['low_confidence']:
|
| 149 |
return ADAPTIVE_PARAMS['low_confidence'].copy()
|
| 150 |
|
| 151 |
-
# Check for profile view
|
| 152 |
elif hasattr(face, 'pose') and len(face.pose) > 1:
|
| 153 |
try:
|
| 154 |
yaw = float(face.pose[1])
|
|
@@ -157,256 +348,600 @@ class RetroArtConverter:
|
|
| 157 |
except (ValueError, TypeError, IndexError):
|
| 158 |
pass
|
| 159 |
|
|
|
|
| 160 |
return None
|
| 161 |
|
| 162 |
except Exception as e:
|
| 163 |
print(f"[ADAPTIVE] Quality detection failed: {e}")
|
| 164 |
return None
|
| 165 |
|
| 166 |
-
def
|
| 167 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 168 |
if not self.caption_enabled or self.caption_model is None:
|
| 169 |
return None
|
| 170 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 171 |
try:
|
| 172 |
-
|
| 173 |
-
|
| 174 |
-
|
| 175 |
-
|
| 176 |
-
|
| 177 |
-
inputs = self.caption_processor(image, return_tensors="pt").to(self.device)
|
| 178 |
-
|
| 179 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 180 |
else:
|
| 181 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 182 |
|
| 183 |
-
|
|
|
|
|
|
|
| 184 |
except Exception as e:
|
| 185 |
-
print(f"
|
|
|
|
| 186 |
return None
|
| 187 |
|
| 188 |
def generate_retro_art(
|
| 189 |
self,
|
| 190 |
input_image,
|
| 191 |
-
prompt=" ",
|
| 192 |
-
negative_prompt=" ",
|
| 193 |
num_inference_steps=12,
|
| 194 |
-
guidance_scale=1.
|
| 195 |
-
depth_control_scale=0.
|
| 196 |
identity_control_scale=0.85,
|
|
|
|
|
|
|
| 197 |
lora_scale=1.0,
|
| 198 |
-
identity_preservation=
|
| 199 |
-
strength=0.
|
| 200 |
enable_color_matching=False,
|
| 201 |
consistency_mode=True,
|
| 202 |
seed=-1
|
| 203 |
):
|
| 204 |
-
"""Generate retro art with
|
| 205 |
|
| 206 |
-
|
| 207 |
-
|
| 208 |
-
|
| 209 |
-
|
| 210 |
-
|
| 211 |
-
|
| 212 |
-
|
| 213 |
-
|
| 214 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 215 |
|
| 216 |
-
|
| 217 |
-
|
| 218 |
-
|
| 219 |
-
|
| 220 |
-
|
| 221 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 222 |
|
| 223 |
-
if self.
|
|
|
|
|
|
|
|
|
|
| 224 |
try:
|
| 225 |
-
|
| 226 |
-
|
| 227 |
-
image_array_orig = cv2.cvtColor(np.array(input_image), cv2.COLOR_RGB2BGR)
|
| 228 |
-
faces = self.face_app.get(image_array_orig)
|
| 229 |
|
| 230 |
if len(faces) > 0:
|
|
|
|
| 231 |
has_detected_faces = True
|
| 232 |
-
|
| 233 |
-
|
| 234 |
-
# Get face embeddings (512D array)
|
| 235 |
-
face_embeddings = face.normed_embedding
|
| 236 |
|
| 237 |
-
#
|
| 238 |
-
|
| 239 |
-
face_kps_image_orig = draw_kps(input_image, face.kps) # Draw on original
|
| 240 |
|
| 241 |
-
#
|
| 242 |
-
face_bbox_original = face.bbox
|
| 243 |
-
|
| 244 |
-
# Adaptive parameter adjustment
|
| 245 |
adaptive_params = self.detect_face_quality(face)
|
| 246 |
-
if adaptive_params:
|
| 247 |
print(f"[ADAPTIVE] {adaptive_params['reason']}")
|
| 248 |
-
identity_preservation = adaptive_params
|
| 249 |
-
identity_control_scale = adaptive_params
|
| 250 |
-
guidance_scale = adaptive_params
|
| 251 |
-
lora_scale = adaptive_params
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 252 |
|
| 253 |
-
|
| 254 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 255 |
else:
|
| 256 |
-
print("
|
| 257 |
-
|
| 258 |
-
except Exception as e:
|
| 259 |
-
print(f"[FACE] Detection failed: {e}")
|
| 260 |
-
|
| 261 |
-
# 2. Calculate optimal size for generation
|
| 262 |
-
orig_width, orig_height = safe_image_size(input_image)
|
| 263 |
-
optimal_width, optimal_height = calculate_optimal_size(orig_width, orig_height)
|
| 264 |
-
|
| 265 |
-
# 3. Resize main image for pipeline
|
| 266 |
-
resized_image = input_image.resize((optimal_width, optimal_height), Image.LANCZOS)
|
| 267 |
-
print(f"[SIZE] Resized to {optimal_width}x{optimal_height}")
|
| 268 |
-
|
| 269 |
-
# 4. Resize KPS image (if one was created) to match generation size
|
| 270 |
-
if face_kps_image_orig is not None:
|
| 271 |
-
face_kps_image = face_kps_image_orig.resize((optimal_width, optimal_height), Image.LANCZOS)
|
| 272 |
-
|
| 273 |
-
# 5. Generate depth map from the (now correctly sized) resized_image
|
| 274 |
-
depth_image, depth_array = self.get_depth_map(resized_image)
|
| 275 |
-
|
| 276 |
-
# --- END FIX ---
|
| 277 |
-
|
| 278 |
-
# Set LORA scale
|
| 279 |
-
if hasattr(self.pipe, 'set_adapters') and self.models_loaded['lora']:
|
| 280 |
-
try:
|
| 281 |
-
self.pipe.set_adapters(["retroart"], adapter_weights=[lora_scale])
|
| 282 |
-
print(f"[LORA] Scale: {lora_scale}")
|
| 283 |
except Exception as e:
|
| 284 |
-
print(f"[
|
| 285 |
-
|
| 286 |
-
# Prepare generation kwargs
|
| 287 |
-
pipe_kwargs = {
|
| 288 |
-
"image": resized_image,
|
| 289 |
-
"strength": strength,
|
| 290 |
-
"num_inference_steps": num_inference_steps,
|
| 291 |
-
"guidance_scale": guidance_scale,
|
| 292 |
-
}
|
| 293 |
-
|
| 294 |
-
# Setup generator with seed
|
| 295 |
-
if seed == -1:
|
| 296 |
-
generator = torch.Generator(device=self.device)
|
| 297 |
-
actual_seed = generator.seed()
|
| 298 |
-
print(f"[SEED] Random: {actual_seed}")
|
| 299 |
else:
|
| 300 |
-
|
| 301 |
-
actual_seed = seed
|
| 302 |
-
print(f"[SEED] Fixed: {actual_seed}")
|
| 303 |
-
|
| 304 |
-
pipe_kwargs["generator"] = generator
|
| 305 |
|
| 306 |
-
#
|
| 307 |
-
if
|
| 308 |
-
|
| 309 |
-
|
| 310 |
-
negative_conditioning = self.compel(negative_prompt)
|
| 311 |
|
| 312 |
-
|
| 313 |
-
|
| 314 |
-
|
| 315 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 316 |
|
| 317 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 318 |
except Exception as e:
|
| 319 |
-
print(f"
|
| 320 |
-
|
| 321 |
-
pipe_kwargs["negative_prompt"] = negative_prompt
|
| 322 |
else:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 323 |
pipe_kwargs["prompt"] = prompt
|
| 324 |
pipe_kwargs["negative_prompt"] = negative_prompt
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 325 |
|
| 326 |
-
|
| 327 |
-
|
| 328 |
-
|
| 329 |
-
|
| 330 |
-
|
| 331 |
-
|
| 332 |
-
|
| 333 |
-
|
| 334 |
-
|
| 335 |
-
|
| 336 |
-
|
| 337 |
-
|
| 338 |
-
|
| 339 |
-
|
| 340 |
-
|
| 341 |
-
|
| 342 |
-
|
| 343 |
-
|
| 344 |
-
|
| 345 |
-
|
| 346 |
-
|
| 347 |
-
|
| 348 |
-
|
| 349 |
-
|
| 350 |
-
|
| 351 |
-
|
| 352 |
-
|
| 353 |
-
|
| 354 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 355 |
else:
|
| 356 |
-
|
|
|
|
| 357 |
|
| 358 |
-
#
|
| 359 |
-
|
| 360 |
-
pipe_kwargs["controlnet_conditioning_scale"] = [0.0, depth_control_scale]
|
| 361 |
|
| 362 |
-
#
|
| 363 |
-
|
| 364 |
-
print(" [FIX] Providing dummy zero-embeddings for IP-Adapter")
|
| 365 |
|
| 366 |
-
#
|
| 367 |
-
|
|
|
|
| 368 |
|
| 369 |
-
|
| 370 |
-
|
| 371 |
-
|
| 372 |
-
|
| 373 |
-
|
| 374 |
-
|
| 375 |
-
|
| 376 |
-
|
| 377 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 378 |
|
| 379 |
-
|
| 380 |
-
|
| 381 |
-
|
| 382 |
-
|
| 383 |
-
|
| 384 |
-
|
| 385 |
-
|
| 386 |
-
|
| 387 |
-
|
| 388 |
-
|
| 389 |
-
|
| 390 |
-
|
| 391 |
-
|
| 392 |
-
|
| 393 |
-
|
| 394 |
-
|
| 395 |
-
|
| 396 |
-
|
| 397 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 398 |
generated_image = color_match(generated_image, resized_image, mode='mkl')
|
| 399 |
print("[OK] Standard color matching applied")
|
| 400 |
-
|
| 401 |
-
|
| 402 |
-
|
| 403 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 404 |
|
| 405 |
-
|
| 406 |
-
# Memory cleanup
|
| 407 |
-
if torch.cuda.is_available():
|
| 408 |
-
torch.cuda.empty_cache()
|
| 409 |
-
gc.collect()
|
| 410 |
|
| 411 |
|
| 412 |
-
print("[OK] Generator class ready
|
|
|
|
| 1 |
"""
|
| 2 |
+
Generation logic for Pixagram AI Pixel Art Generator
|
|
|
|
| 3 |
"""
|
| 4 |
+
import gc
|
| 5 |
import torch
|
| 6 |
import numpy as np
|
| 7 |
import cv2
|
| 8 |
from PIL import Image
|
| 9 |
+
import torch.nn.functional as F
|
| 10 |
+
from torchvision import transforms
|
| 11 |
+
import traceback
|
| 12 |
|
| 13 |
from config import (
|
| 14 |
+
device, dtype, TRIGGER_WORD, MULTI_SCALE_FACTORS,
|
| 15 |
+
ADAPTIVE_THRESHOLDS, ADAPTIVE_PARAMS, CAPTION_CONFIG, IDENTITY_BOOST_MULTIPLIER
|
| 16 |
)
|
| 17 |
from utils import (
|
| 18 |
+
sanitize_text, enhanced_color_match, color_match, create_face_mask,
|
| 19 |
+
draw_kps, get_demographic_description, calculate_optimal_size, enhance_face_crop
|
| 20 |
)
|
| 21 |
from models import (
|
| 22 |
+
load_face_analysis, load_depth_detector, load_controlnets, load_image_encoder,
|
| 23 |
+
load_sdxl_pipeline, load_loras, setup_ip_adapter,
|
| 24 |
+
# --- START FIX: Import setup_compel ---
|
| 25 |
+
setup_compel,
|
| 26 |
+
# --- END FIX ---
|
| 27 |
+
setup_scheduler, optimize_pipeline, load_caption_model, set_clip_skip,
|
| 28 |
+
load_openpose_detector, load_mediapipe_face_detector
|
| 29 |
)
|
| 30 |
|
| 31 |
|
| 32 |
class RetroArtConverter:
|
| 33 |
+
"""Main class for retro art generation"""
|
| 34 |
|
| 35 |
def __init__(self):
|
| 36 |
self.device = device
|
|
|
|
| 39 |
'custom_checkpoint': False,
|
| 40 |
'lora': False,
|
| 41 |
'instantid': False,
|
| 42 |
+
'depth_detector': False,
|
| 43 |
+
'depth_type': None,
|
| 44 |
+
'ip_adapter': False,
|
| 45 |
+
'openpose': False,
|
| 46 |
+
'mediapipe_face': False
|
| 47 |
}
|
| 48 |
+
self.loaded_loras = {} # Store status of each LORA
|
| 49 |
|
| 50 |
+
# Initialize face analysis (InsightFace)
|
| 51 |
self.face_app, self.face_detection_enabled = load_face_analysis()
|
| 52 |
|
| 53 |
+
# Load MediapipeFaceDetector (alternative face detection)
|
| 54 |
+
self.mediapipe_face, mediapipe_success = load_mediapipe_face_detector()
|
| 55 |
+
self.models_loaded['mediapipe_face'] = mediapipe_success
|
| 56 |
+
|
| 57 |
+
# Load Depth detector with fallback hierarchy (Leres → Zoe → Midas)
|
| 58 |
+
self.depth_detector, self.depth_type, depth_success = load_depth_detector()
|
| 59 |
+
self.models_loaded['depth_detector'] = depth_success
|
| 60 |
+
self.models_loaded['depth_type'] = self.depth_type
|
| 61 |
+
|
| 62 |
+
# --- NEW: Load OpenPose detector ---
|
| 63 |
+
self.openpose_detector, openpose_success = load_openpose_detector()
|
| 64 |
+
self.models_loaded['openpose'] = openpose_success
|
| 65 |
+
# --- END NEW ---
|
| 66 |
+
|
| 67 |
+
# Load ControlNets
|
| 68 |
+
# Now unpacks 3 models + success boolean
|
| 69 |
+
controlnet_depth, self.controlnet_instantid, self.controlnet_openpose, instantid_success = load_controlnets()
|
| 70 |
+
self.controlnet_depth = controlnet_depth
|
| 71 |
+
self.instantid_enabled = instantid_success
|
| 72 |
+
self.models_loaded['instantid'] = instantid_success
|
| 73 |
+
|
| 74 |
+
# Load image encoder
|
| 75 |
+
if self.instantid_enabled:
|
| 76 |
+
self.image_encoder = load_image_encoder()
|
| 77 |
+
else:
|
| 78 |
+
self.image_encoder = None
|
| 79 |
|
| 80 |
+
# --- FIX START: Robust ControlNet Loading ---
|
| 81 |
+
# Determine which controlnets to use
|
|
|
|
|
|
|
| 82 |
|
| 83 |
+
# Store booleans for which models are active
|
| 84 |
+
self.instantid_active = self.instantid_enabled and self.controlnet_instantid is not None
|
| 85 |
+
self.depth_active = self.controlnet_depth is not None
|
| 86 |
+
self.openpose_active = self.controlnet_openpose is not None
|
| 87 |
+
|
| 88 |
+
# Build the list of *active* controlnet models
|
| 89 |
+
controlnets = []
|
| 90 |
+
if self.instantid_active:
|
| 91 |
+
controlnets.append(self.controlnet_instantid)
|
| 92 |
+
print(" [CN] InstantID (Identity) active")
|
| 93 |
+
else:
|
| 94 |
+
print(" [CN] InstantID (Identity) DISABLED")
|
| 95 |
+
|
| 96 |
+
if self.depth_active:
|
| 97 |
+
controlnets.append(self.controlnet_depth)
|
| 98 |
+
print(" [CN] Depth active")
|
| 99 |
+
else:
|
| 100 |
+
print(" [CN] Depth DISABLED")
|
| 101 |
+
|
| 102 |
+
if self.openpose_active:
|
| 103 |
+
controlnets.append(self.controlnet_openpose)
|
| 104 |
+
print(" [CN] OpenPose (Expression) active")
|
| 105 |
+
else:
|
| 106 |
+
print(" [CN] OpenPose (Expression) DISABLED")
|
| 107 |
+
|
| 108 |
+
if not controlnets:
|
| 109 |
+
print("[WARNING] No ControlNets loaded!")
|
| 110 |
+
|
| 111 |
+
print(f"Initializing with {len(controlnets)} active ControlNet(s)")
|
| 112 |
+
|
| 113 |
+
# Load SDXL pipeline
|
| 114 |
+
# Pass the filtered list (or None if empty)
|
| 115 |
+
self.pipe, checkpoint_success = load_sdxl_pipeline(controlnets if controlnets else None)
|
| 116 |
+
# --- FIX END ---
|
| 117 |
|
|
|
|
|
|
|
| 118 |
self.models_loaded['custom_checkpoint'] = checkpoint_success
|
| 119 |
|
| 120 |
+
# Load LORAs
|
| 121 |
+
self.loaded_loras, lora_success = load_loras(self.pipe)
|
| 122 |
self.models_loaded['lora'] = lora_success
|
| 123 |
|
| 124 |
+
# Setup IP-Adapter
|
| 125 |
+
if self.instantid_active and self.image_encoder is not None: # <-- Check instantid_active
|
| 126 |
+
self.image_proj_model, ip_adapter_success = setup_ip_adapter(self.pipe, self.image_encoder)
|
| 127 |
+
self.models_loaded['ip_adapter'] = ip_adapter_success
|
| 128 |
+
else:
|
| 129 |
+
print("[INFO] Face preservation: IP-Adapter disabled (InstantID model failed or encoder failed)")
|
| 130 |
+
self.models_loaded['ip_adapter'] = False
|
| 131 |
+
self.image_proj_model = None
|
| 132 |
+
|
| 133 |
+
# --- START FIX: Setup Compel ---
|
| 134 |
self.compel, self.use_compel = setup_compel(self.pipe)
|
| 135 |
+
# --- END FIX ---
|
| 136 |
|
| 137 |
+
# Setup LCM scheduler
|
| 138 |
setup_scheduler(self.pipe)
|
| 139 |
|
| 140 |
+
# Optimize pipeline
|
| 141 |
optimize_pipeline(self.pipe)
|
| 142 |
|
| 143 |
# Load caption model
|
| 144 |
self.caption_processor, self.caption_model, self.caption_enabled, self.caption_model_type = load_caption_model()
|
| 145 |
|
| 146 |
+
# Report caption model status
|
| 147 |
+
if self.caption_enabled and self.caption_model is not None:
|
| 148 |
+
if self.caption_model_type == "git":
|
| 149 |
+
print(" [OK] Using GIT for detailed captions")
|
| 150 |
+
elif self.caption_model_type == "blip":
|
| 151 |
+
print(" [OK] Using BLIP for standard captions")
|
| 152 |
+
else:
|
| 153 |
+
print(" [OK] Caption model loaded")
|
| 154 |
+
|
| 155 |
+
|
| 156 |
# Set CLIP skip
|
| 157 |
set_clip_skip(self.pipe)
|
| 158 |
|
| 159 |
+
# Track controlnet configuration
|
| 160 |
+
self.using_multiple_controlnets = isinstance(controlnets, list)
|
| 161 |
+
print(f"Pipeline initialized with {'multiple' if self.using_multiple_controlnets else 'single'} ControlNet(s)")
|
| 162 |
+
|
| 163 |
+
# Print model status
|
| 164 |
self._print_status()
|
| 165 |
|
| 166 |
+
print(" [OK] Model initialization complete!")
|
| 167 |
|
| 168 |
def _print_status(self):
|
| 169 |
"""Print model loading status"""
|
| 170 |
print("\n=== MODEL STATUS ===")
|
| 171 |
for model, loaded in self.models_loaded.items():
|
| 172 |
+
if model == 'lora':
|
| 173 |
+
lora_status = 'DISABLED'
|
| 174 |
+
if loaded:
|
| 175 |
+
loaded_count = sum(1 for status in self.loaded_loras.values() if status)
|
| 176 |
+
lora_status = f"[OK] LOADED ({loaded_count}/3)"
|
| 177 |
+
print(f"loras: {lora_status}")
|
| 178 |
+
else:
|
| 179 |
+
status = "[OK] LOADED" if loaded else "[FALLBACK/DISABLED]"
|
| 180 |
+
print(f"{model}: {status}")
|
| 181 |
print("===================\n")
|
| 182 |
+
|
| 183 |
+
print("=== UPGRADE VERIFICATION ===")
|
| 184 |
+
try:
|
| 185 |
+
# --- FIX: Corrected import paths and class names ---
|
| 186 |
+
from resampler import Resampler
|
| 187 |
+
from attention_processor import IPAttnProcessor2_0
|
| 188 |
+
|
| 189 |
+
resampler_check = isinstance(self.image_proj_model, Resampler) if hasattr(self, 'image_proj_model') and self.image_proj_model is not None else False
|
| 190 |
+
custom_attn_check = any(isinstance(p, IPAttnProcessor2_0) for p in self.pipe.unet.attn_processors.values()) if hasattr(self, 'pipe') else False
|
| 191 |
+
# --- END FIX ---
|
| 192 |
+
|
| 193 |
+
print(f"Enhanced Perceiver Resampler: {'[OK] ACTIVE' if resampler_check else '[INFO] Not active'}")
|
| 194 |
+
print(f"Enhanced IP-Adapter Attention: {'[OK] ACTIVE' if custom_attn_check else '[INFO] Not active'}")
|
| 195 |
+
|
| 196 |
+
if resampler_check and custom_attn_check:
|
| 197 |
+
print("[SUCCESS] Face preservation upgrade fully active")
|
| 198 |
+
print(" Expected improvement: +10-15% face similarity")
|
| 199 |
+
elif resampler_check or custom_attn_check:
|
| 200 |
+
print("[PARTIAL] Some upgrades active")
|
| 201 |
+
else:
|
| 202 |
+
print("[INFO] Using standard components")
|
| 203 |
+
except Exception as e:
|
| 204 |
+
print(f"[INFO] Verification skipped: {e}")
|
| 205 |
+
print("============================\n")
|
| 206 |
+
|
| 207 |
|
| 208 |
def get_depth_map(self, image):
|
| 209 |
+
"""
|
| 210 |
+
Generate depth map using available depth detector.
|
| 211 |
+
Supports: LeresDetector, ZoeDetector, or MidasDetector.
|
| 212 |
+
"""
|
| 213 |
+
if self.depth_detector is not None:
|
| 214 |
try:
|
| 215 |
if image.mode != 'RGB':
|
| 216 |
image = image.convert('RGB')
|
| 217 |
|
| 218 |
+
orig_width, orig_height = image.size
|
| 219 |
+
orig_width = int(orig_width)
|
| 220 |
+
orig_height = int(orig_height)
|
| 221 |
|
|
|
|
| 222 |
target_width = int((orig_width // 64) * 64)
|
| 223 |
target_height = int((orig_height // 64) * 64)
|
| 224 |
|
|
|
|
| 228 |
size_for_depth = (int(target_width), int(target_height))
|
| 229 |
image_for_depth = image.resize(size_for_depth, Image.LANCZOS)
|
| 230 |
|
| 231 |
+
if target_width != orig_width or target_height != orig_height:
|
| 232 |
+
print(f"[DEPTH] Resized for {self.depth_type.upper()}Detector: {orig_width}x{orig_height} -> {target_width}x{target_height}")
|
| 233 |
+
|
| 234 |
+
# Use torch.no_grad() and clear cache
|
| 235 |
+
with torch.no_grad():
|
| 236 |
+
# --- FIX: Move model to GPU for inference and back to CPU ---
|
| 237 |
+
self.depth_detector.to(self.device)
|
| 238 |
+
depth_image = self.depth_detector(image_for_depth)
|
| 239 |
+
self.depth_detector.to("cpu")
|
| 240 |
+
|
| 241 |
+
# ADDED: Clear GPU cache after depth detection
|
| 242 |
+
if torch.cuda.is_available():
|
| 243 |
+
torch.cuda.empty_cache()
|
| 244 |
+
|
| 245 |
+
depth_width, depth_height = depth_image.size
|
| 246 |
+
if depth_width != orig_width or depth_height != orig_height:
|
| 247 |
+
depth_image = depth_image.resize((int(orig_width), int(orig_height)), Image.LANCZOS)
|
| 248 |
|
| 249 |
+
print(f"[DEPTH] {self.depth_type.upper()} depth map generated: {orig_width}x{orig_height}")
|
| 250 |
+
return depth_image
|
|
|
|
|
|
|
| 251 |
|
|
|
|
|
|
|
| 252 |
except Exception as e:
|
| 253 |
+
print(f"[DEPTH] {self.depth_type.upper()}Detector failed ({e}), falling back to grayscale depth")
|
| 254 |
+
# ADDED: Clear cache on error
|
| 255 |
+
if torch.cuda.is_available():
|
| 256 |
+
torch.cuda.empty_cache()
|
| 257 |
+
|
| 258 |
+
gray = cv2.cvtColor(np.array(image), cv2.COLOR_RGB2GRAY)
|
| 259 |
+
depth_colored = cv2.cvtColor(gray, cv2.COLOR_GRAY2RGB)
|
| 260 |
+
return Image.fromarray(depth_colored)
|
| 261 |
else:
|
| 262 |
+
print("[DEPTH] No depth detector available, using grayscale fallback")
|
| 263 |
+
gray = cv2.cvtColor(np.array(image), cv2.COLOR_RGB2GRAY)
|
| 264 |
+
depth_colored = cv2.cvtColor(gray, cv2.COLOR_GRAY2RGB)
|
| 265 |
+
return Image.fromarray(depth_colored)
|
| 266 |
+
|
| 267 |
|
| 268 |
+
# --- START FIX: Updated function to use lora_choice ---
|
| 269 |
+
def add_trigger_word(self, prompt, lora_choice="RetroArt"):
|
| 270 |
"""Add trigger word to prompt if not present"""
|
| 271 |
+
|
| 272 |
+
# Get the correct trigger word from the config dictionary
|
| 273 |
+
trigger = TRIGGER_WORD.get(lora_choice, TRIGGER_WORD["RetroArt"])
|
| 274 |
+
|
| 275 |
+
if not trigger:
|
| 276 |
+
return prompt
|
| 277 |
+
|
| 278 |
+
if trigger.lower() not in prompt.lower():
|
| 279 |
if not prompt or not prompt.strip():
|
| 280 |
+
return trigger
|
| 281 |
+
# Prepend the trigger word as requested
|
| 282 |
+
return f"{trigger}, {prompt}"
|
| 283 |
return prompt
|
| 284 |
+
# --- END FIX ---
|
| 285 |
+
|
| 286 |
+
def extract_multi_scale_face(self, face_crop, face):
|
| 287 |
+
"""
|
| 288 |
+
Extract face features at multiple scales for better detail.
|
| 289 |
+
+1-2% improvement in face preservation.
|
| 290 |
+
"""
|
| 291 |
+
try:
|
| 292 |
+
multi_scale_embeds = []
|
| 293 |
+
|
| 294 |
+
for scale in MULTI_SCALE_FACTORS:
|
| 295 |
+
# Resize
|
| 296 |
+
w, h = face_crop.size
|
| 297 |
+
scaled_size = (int(w * scale), int(h * scale))
|
| 298 |
+
scaled_crop = face_crop.resize(scaled_size, Image.LANCZOS)
|
| 299 |
+
|
| 300 |
+
# Pad/crop back to original
|
| 301 |
+
scaled_crop = scaled_crop.resize((w, h), Image.LANCZOS)
|
| 302 |
+
|
| 303 |
+
# Extract features
|
| 304 |
+
scaled_array = cv2.cvtColor(np.array(scaled_crop), cv2.COLOR_RGB2BGR)
|
| 305 |
+
scaled_faces = self.face_app.get(scaled_array)
|
| 306 |
+
|
| 307 |
+
if len(scaled_faces) > 0:
|
| 308 |
+
multi_scale_embeds.append(scaled_faces[0].normed_embedding)
|
| 309 |
+
|
| 310 |
+
# Average embeddings
|
| 311 |
+
if len(multi_scale_embeds) > 0:
|
| 312 |
+
averaged = np.mean(multi_scale_embeds, axis=0)
|
| 313 |
+
# Renormalize
|
| 314 |
+
averaged = averaged / np.linalg.norm(averaged)
|
| 315 |
+
print(f"[MULTI-SCALE] Combined {len(multi_scale_embeds)} scales")
|
| 316 |
+
return averaged
|
| 317 |
+
|
| 318 |
+
return face.normed_embedding
|
| 319 |
+
|
| 320 |
+
except Exception as e:
|
| 321 |
+
print(f"[MULTI-SCALE] Failed: {e}, using single scale")
|
| 322 |
+
return face.normed_embedding
|
| 323 |
|
| 324 |
def detect_face_quality(self, face):
|
| 325 |
+
"""
|
| 326 |
+
Detect face quality and adaptively adjust parameters.
|
| 327 |
+
+2-3% consistency improvement.
|
| 328 |
+
"""
|
| 329 |
try:
|
| 330 |
bbox = face.bbox
|
| 331 |
face_size = (bbox[2] - bbox[0]) * (bbox[3] - bbox[1])
|
| 332 |
det_score = float(face.det_score) if hasattr(face, 'det_score') else 1.0
|
| 333 |
|
| 334 |
+
# Small face -> boost identity preservation
|
| 335 |
if face_size < ADAPTIVE_THRESHOLDS['small_face_size']:
|
| 336 |
return ADAPTIVE_PARAMS['small_face'].copy()
|
| 337 |
|
|
|
|
| 339 |
elif det_score < ADAPTIVE_THRESHOLDS['low_confidence']:
|
| 340 |
return ADAPTIVE_PARAMS['low_confidence'].copy()
|
| 341 |
|
| 342 |
+
# Check for profile/side view (if pose available)
|
| 343 |
elif hasattr(face, 'pose') and len(face.pose) > 1:
|
| 344 |
try:
|
| 345 |
yaw = float(face.pose[1])
|
|
|
|
| 348 |
except (ValueError, TypeError, IndexError):
|
| 349 |
pass
|
| 350 |
|
| 351 |
+
# Good quality face - use provided parameters
|
| 352 |
return None
|
| 353 |
|
| 354 |
except Exception as e:
|
| 355 |
print(f"[ADAPTIVE] Quality detection failed: {e}")
|
| 356 |
return None
|
| 357 |
|
| 358 |
+
def validate_and_adjust_parameters(self, strength, guidance_scale, lora_scale,
|
| 359 |
+
identity_preservation, identity_control_scale,
|
| 360 |
+
depth_control_scale, consistency_mode=True,
|
| 361 |
+
expression_control_scale=0.6):
|
| 362 |
+
"""
|
| 363 |
+
Enhanced parameter validation with stricter rules for consistency.
|
| 364 |
+
"""
|
| 365 |
+
if consistency_mode:
|
| 366 |
+
print("[CONSISTENCY] Applying strict parameter validation...")
|
| 367 |
+
adjustments = []
|
| 368 |
+
|
| 369 |
+
# Rule 1: Strong inverse relationship between identity and LORA
|
| 370 |
+
if identity_preservation > 1.2:
|
| 371 |
+
original_lora = lora_scale
|
| 372 |
+
lora_scale = min(lora_scale, 1.0)
|
| 373 |
+
if abs(lora_scale - original_lora) > 0.01:
|
| 374 |
+
adjustments.append(f"LORA: {original_lora:.2f}->{lora_scale:.2f} (high identity)")
|
| 375 |
+
|
| 376 |
+
# Rule 2: Strength-based profile activation
|
| 377 |
+
if strength < 0.5:
|
| 378 |
+
# Maximum preservation mode
|
| 379 |
+
if identity_preservation < 1.3:
|
| 380 |
+
original_identity = identity_preservation
|
| 381 |
+
identity_preservation = 1.3
|
| 382 |
+
adjustments.append(f"Identity: {original_identity:.2f}->{identity_preservation:.2f} (max preservation)")
|
| 383 |
+
if lora_scale > 0.9:
|
| 384 |
+
original_lora = lora_scale
|
| 385 |
+
lora_scale = 0.9
|
| 386 |
+
adjustments.append(f"LORA: {original_lora:.2f}->{lora_scale:.2f} (max preservation)")
|
| 387 |
+
if guidance_scale > 1.3:
|
| 388 |
+
original_cfg = guidance_scale
|
| 389 |
+
guidance_scale = 1.3
|
| 390 |
+
adjustments.append(f"CFG: {original_cfg:.2f}->{guidance_scale:.2f} (max preservation)")
|
| 391 |
+
|
| 392 |
+
elif strength > 0.7:
|
| 393 |
+
# Artistic transformation mode
|
| 394 |
+
if identity_preservation > 1.0:
|
| 395 |
+
original_identity = identity_preservation
|
| 396 |
+
identity_preservation = 1.0
|
| 397 |
+
adjustments.append(f"Identity: {original_identity:.2f}->{identity_preservation:.2f} (artistic mode)")
|
| 398 |
+
if lora_scale < 1.2:
|
| 399 |
+
original_lora = lora_scale
|
| 400 |
+
lora_scale = 1.2
|
| 401 |
+
adjustments.append(f"LORA: {original_lora:.2f}->{lora_scale:.2f} (artistic mode)")
|
| 402 |
+
|
| 403 |
+
# Rule 3: CFG-LORA relationship
|
| 404 |
+
if guidance_scale > 1.4 and lora_scale > 1.2:
|
| 405 |
+
original_lora = lora_scale
|
| 406 |
+
lora_scale = 1.1
|
| 407 |
+
adjustments.append(f"LORA: {original_lora:.2f}->{lora_scale:.2f} (high CFG detected)")
|
| 408 |
+
|
| 409 |
+
# Rule 4: LCM sweet spot enforcement
|
| 410 |
+
original_cfg = guidance_scale
|
| 411 |
+
guidance_scale = max(1.0, min(guidance_scale, 1.5))
|
| 412 |
+
if abs(guidance_scale - original_cfg) > 0.01:
|
| 413 |
+
adjustments.append(f"CFG: {original_cfg:.2f}->{guidance_scale:.2f} (LCM optimal)")
|
| 414 |
+
|
| 415 |
+
# Rule 5: ControlNet balance
|
| 416 |
+
# MODIFIED: Only sum *active* controlnets
|
| 417 |
+
total_control = 0
|
| 418 |
+
if self.instantid_active:
|
| 419 |
+
total_control += identity_control_scale
|
| 420 |
+
if self.depth_active:
|
| 421 |
+
total_control += depth_control_scale
|
| 422 |
+
if self.openpose_active:
|
| 423 |
+
total_control += expression_control_scale
|
| 424 |
+
|
| 425 |
+
if total_control > 2.0: # Increased max total from 1.7 to 2.0
|
| 426 |
+
scale_factor = 2.0 / total_control
|
| 427 |
+
original_id_ctrl = identity_control_scale
|
| 428 |
+
original_depth_ctrl = depth_control_scale
|
| 429 |
+
original_expr_ctrl = expression_control_scale
|
| 430 |
+
|
| 431 |
+
# Only scale active controlnets
|
| 432 |
+
if self.instantid_active:
|
| 433 |
+
identity_control_scale *= scale_factor
|
| 434 |
+
if self.depth_active:
|
| 435 |
+
depth_control_scale *= scale_factor
|
| 436 |
+
if self.openpose_active:
|
| 437 |
+
expression_control_scale *= scale_factor
|
| 438 |
+
|
| 439 |
+
adjustments.append(f"ControlNets balanced: ID {original_id_ctrl:.2f}->{identity_control_scale:.2f}, Depth {original_depth_ctrl:.2f}->{depth_control_scale:.2f}, Expr {original_expr_ctrl:.2f}->{expression_control_scale:.2f}")
|
| 440 |
+
|
| 441 |
+
# Report adjustments
|
| 442 |
+
if adjustments:
|
| 443 |
+
print(" [OK] Applied adjustments:")
|
| 444 |
+
for adj in adjustments:
|
| 445 |
+
print(f" - {adj}")
|
| 446 |
+
else:
|
| 447 |
+
print(" [OK] Parameters already optimal")
|
| 448 |
+
|
| 449 |
+
return strength, guidance_scale, lora_scale, identity_preservation, identity_control_scale, depth_control_scale, expression_control_scale
|
| 450 |
+
|
| 451 |
+
def generate_caption(self, image, max_length=None, num_beams=None):
|
| 452 |
+
"""Generate a descriptive caption for the image (supports BLIP-2, GIT, BLIP)."""
|
| 453 |
if not self.caption_enabled or self.caption_model is None:
|
| 454 |
return None
|
| 455 |
|
| 456 |
+
# Set defaults based on model type
|
| 457 |
+
if max_length is None:
|
| 458 |
+
if self.caption_model_type == "blip2":
|
| 459 |
+
max_length = 50 # BLIP-2 can handle longer captions
|
| 460 |
+
elif self.caption_model_type == "git":
|
| 461 |
+
max_length = 40 # GIT also produces good long captions
|
| 462 |
+
else:
|
| 463 |
+
max_length = CAPTION_CONFIG['max_length'] # BLIP base (20)
|
| 464 |
+
|
| 465 |
+
if num_beams is None:
|
| 466 |
+
num_beams = CAPTION_CONFIG['num_beams']
|
| 467 |
+
|
| 468 |
try:
|
| 469 |
+
# --- FIX: Move model to GPU for inference and back to CPU ---
|
| 470 |
+
self.caption_model.to(self.device)
|
| 471 |
+
|
| 472 |
+
if self.caption_model_type == "blip2":
|
| 473 |
+
# BLIP-2 specific processing
|
| 474 |
+
inputs = self.caption_processor(image, return_tensors="pt").to(self.device, self.dtype)
|
| 475 |
+
|
| 476 |
+
with torch.no_grad():
|
| 477 |
+
output = self.caption_model.generate(
|
| 478 |
+
**inputs,
|
| 479 |
+
max_length=max_length,
|
| 480 |
+
num_beams=num_beams,
|
| 481 |
+
min_length=10, # Encourage longer captions
|
| 482 |
+
length_penalty=1.0,
|
| 483 |
+
repetition_penalty=1.5,
|
| 484 |
+
early_stopping=True
|
| 485 |
+
)
|
| 486 |
+
|
| 487 |
+
caption = self.caption_processor.decode(output[0], skip_special_tokens=True)
|
| 488 |
+
|
| 489 |
+
elif self.caption_model_type == "git":
|
| 490 |
+
# GIT specific processing
|
| 491 |
+
inputs = self.caption_processor(images=image, return_tensors="pt").to(self.device, self.dtype)
|
| 492 |
+
|
| 493 |
+
with torch.no_grad():
|
| 494 |
+
output = self.caption_model.generate(
|
| 495 |
+
pixel_values=inputs.pixel_values,
|
| 496 |
+
max_length=max_length,
|
| 497 |
+
num_beams=num_beams,
|
| 498 |
+
min_length=10,
|
| 499 |
+
length_penalty=1.0,
|
| 500 |
+
repetition_penalty=1.5,
|
| 501 |
+
early_stopping=True
|
| 502 |
+
)
|
| 503 |
+
|
| 504 |
+
caption = self.caption_processor.batch_decode(output, skip_special_tokens=True)[0]
|
| 505 |
+
|
| 506 |
else:
|
| 507 |
+
# BLIP base processing
|
| 508 |
+
inputs = self.caption_processor(image, return_tensors="pt").to(self.device, self.dtype)
|
| 509 |
+
|
| 510 |
+
with torch.no_grad():
|
| 511 |
+
output = self.caption_model.generate(
|
| 512 |
+
**inputs,
|
| 513 |
+
max_length=max_length,
|
| 514 |
+
num_beams=num_beams,
|
| 515 |
+
early_stopping=True
|
| 516 |
+
)
|
| 517 |
+
|
| 518 |
+
caption = self.caption_processor.decode(output[0], skip_special_tokens=True)
|
| 519 |
|
| 520 |
+
self.caption_model.to("cpu")
|
| 521 |
+
return caption.strip()
|
| 522 |
+
|
| 523 |
except Exception as e:
|
| 524 |
+
print(f"Caption generation failed: {e}")
|
| 525 |
+
self.caption_model.to("cpu")
|
| 526 |
return None
|
| 527 |
|
| 528 |
def generate_retro_art(
|
| 529 |
self,
|
| 530 |
input_image,
|
| 531 |
+
prompt="retro game character, vibrant colors, detailed",
|
| 532 |
+
negative_prompt="blurry, low quality, ugly, distorted",
|
| 533 |
num_inference_steps=12,
|
| 534 |
+
guidance_scale=1.0,
|
| 535 |
+
depth_control_scale=0.8,
|
| 536 |
identity_control_scale=0.85,
|
| 537 |
+
expression_control_scale=0.6,
|
| 538 |
+
lora_choice="RetroArt",
|
| 539 |
lora_scale=1.0,
|
| 540 |
+
identity_preservation=0.8,
|
| 541 |
+
strength=0.75,
|
| 542 |
enable_color_matching=False,
|
| 543 |
consistency_mode=True,
|
| 544 |
seed=-1
|
| 545 |
):
|
| 546 |
+
"""Generate retro art with img2img pipeline and enhanced InstantID"""
|
| 547 |
|
| 548 |
+
# Sanitize text inputs
|
| 549 |
+
prompt = sanitize_text(prompt)
|
| 550 |
+
negative_prompt = sanitize_text(negative_prompt)
|
| 551 |
+
|
| 552 |
+
if not negative_prompt or not negative_prompt.strip():
|
| 553 |
+
negative_prompt = ""
|
| 554 |
+
|
| 555 |
+
# Apply parameter validation
|
| 556 |
+
if consistency_mode:
|
| 557 |
+
print("\n[CONSISTENCY] Validating and adjusting parameters...")
|
| 558 |
+
strength, guidance_scale, lora_scale, identity_preservation, identity_control_scale, depth_control_scale, expression_control_scale = \
|
| 559 |
+
self.validate_and_adjust_parameters(
|
| 560 |
+
strength, guidance_scale, lora_scale, identity_preservation,
|
| 561 |
+
identity_control_scale, depth_control_scale, consistency_mode,
|
| 562 |
+
expression_control_scale
|
| 563 |
+
)
|
| 564 |
+
|
| 565 |
+
# --- START FIX: Pass lora_choice to add_trigger_word ---
|
| 566 |
+
prompt = self.add_trigger_word(prompt, lora_choice)
|
| 567 |
+
# --- END FIX ---
|
| 568 |
+
|
| 569 |
+
# Calculate optimal size with flexible aspect ratio support
|
| 570 |
+
original_width, original_height = input_image.size
|
| 571 |
+
target_width, target_height = calculate_optimal_size(original_width, original_height)
|
| 572 |
+
|
| 573 |
+
print(f"Resizing from {original_width}x{original_height} to {target_width}x{target_height}")
|
| 574 |
+
print(f"Prompt: {prompt}")
|
| 575 |
+
print(f"Img2Img Strength: {strength}")
|
| 576 |
+
|
| 577 |
+
# Resize with high quality
|
| 578 |
+
resized_image = input_image.resize((int(target_width), int(target_height)), Image.LANCZOS)
|
| 579 |
+
|
| 580 |
+
# --- FIX START: Generate control images only if models are active ---
|
| 581 |
+
|
| 582 |
+
# Generate depth map
|
| 583 |
+
depth_image = None
|
| 584 |
+
if self.depth_active:
|
| 585 |
+
print("Generating Zoe depth map...")
|
| 586 |
+
depth_image = self.get_depth_map(resized_image)
|
| 587 |
+
if depth_image.size != (target_width, target_height):
|
| 588 |
+
depth_image = depth_image.resize((int(target_width), int(target_height)), Image.LANCZOS)
|
| 589 |
+
|
| 590 |
+
# Generate OpenPose map
|
| 591 |
+
openpose_image = None
|
| 592 |
+
if self.openpose_active:
|
| 593 |
+
print("Generating OpenPose map...")
|
| 594 |
+
try:
|
| 595 |
+
# --- FIX: Move model to GPU for inference and back to CPU ---
|
| 596 |
+
self.openpose_detector.to(self.device)
|
| 597 |
+
openpose_image = self.openpose_detector(resized_image, face_only=True)
|
| 598 |
+
self.openpose_detector.to("cpu")
|
| 599 |
+
except Exception as e:
|
| 600 |
+
print(f"OpenPose failed, using blank map: {e}")
|
| 601 |
+
self.openpose_detector.to("cpu")
|
| 602 |
+
openpose_image = Image.new("RGB", (target_width, target_height), (0,0,0))
|
| 603 |
+
|
| 604 |
+
# --- FIX END ---
|
| 605 |
|
| 606 |
+
|
| 607 |
+
# Handle face detection
|
| 608 |
+
face_kps_image = None
|
| 609 |
+
face_embeddings = None
|
| 610 |
+
face_crop_enhanced = None
|
| 611 |
+
has_detected_faces = False
|
| 612 |
+
face_bbox_original = None
|
| 613 |
+
|
| 614 |
+
if self.instantid_active:
|
| 615 |
+
# Try InsightFace first (if available)
|
| 616 |
+
insightface_tried = False
|
| 617 |
+
insightface_success = False
|
| 618 |
|
| 619 |
+
if self.face_app is not None:
|
| 620 |
+
print("Detecting faces with InsightFace...")
|
| 621 |
+
insightface_tried = True
|
| 622 |
+
|
| 623 |
try:
|
| 624 |
+
img_array = cv2.cvtColor(np.array(resized_image), cv2.COLOR_RGB2BGR)
|
| 625 |
+
faces = self.face_app.get(img_array)
|
|
|
|
|
|
|
| 626 |
|
| 627 |
if len(faces) > 0:
|
| 628 |
+
insightface_success = True
|
| 629 |
has_detected_faces = True
|
| 630 |
+
print(f"✓ InsightFace detected {len(faces)} face(s)")
|
|
|
|
|
|
|
|
|
|
| 631 |
|
| 632 |
+
# Get largest face
|
| 633 |
+
face = sorted(faces, key=lambda x: (x.bbox[2] - x.bbox[0]) * (x.bbox[3] - x.bbox[1]))[-1]
|
|
|
|
| 634 |
|
| 635 |
+
# ADAPTIVE PARAMETERS
|
|
|
|
|
|
|
|
|
|
| 636 |
adaptive_params = self.detect_face_quality(face)
|
| 637 |
+
if adaptive_params is not None:
|
| 638 |
print(f"[ADAPTIVE] {adaptive_params['reason']}")
|
| 639 |
+
identity_preservation = adaptive_params['identity_preservation']
|
| 640 |
+
identity_control_scale = adaptive_params['identity_control_scale']
|
| 641 |
+
guidance_scale = adaptive_params['guidance_scale']
|
| 642 |
+
lora_scale = adaptive_params['lora_scale']
|
| 643 |
+
|
| 644 |
+
# Extract face embeddings
|
| 645 |
+
face_embeddings_base = face.normed_embedding
|
| 646 |
+
|
| 647 |
+
# Extract face crop
|
| 648 |
+
bbox = face.bbox.astype(int)
|
| 649 |
+
x1, y1, x2, y2 = bbox[0], bbox[1], bbox[2], bbox[3]
|
| 650 |
+
face_bbox_original = [x1, y1, x2, y2]
|
| 651 |
+
|
| 652 |
+
# Add padding
|
| 653 |
+
face_width = x2 - x1
|
| 654 |
+
face_height = y2 - y1
|
| 655 |
+
padding_x = int(face_width * 0.3)
|
| 656 |
+
padding_y = int(face_height * 0.3)
|
| 657 |
+
x1 = max(0, x1 - padding_x)
|
| 658 |
+
y1 = max(0, y1 - padding_y)
|
| 659 |
+
x2 = min(resized_image.width, x2 + padding_x)
|
| 660 |
+
y2 = min(resized_image.height, y2 + padding_y)
|
| 661 |
+
|
| 662 |
+
# Crop face region
|
| 663 |
+
face_crop = resized_image.crop((x1, y1, x2, y2))
|
| 664 |
+
|
| 665 |
+
# MULTI-SCALE PROCESSING
|
| 666 |
+
face_embeddings = self.extract_multi_scale_face(face_crop, face)
|
| 667 |
+
|
| 668 |
+
# Enhance face crop
|
| 669 |
+
face_crop_enhanced = enhance_face_crop(face_crop)
|
| 670 |
+
|
| 671 |
+
# Draw keypoints
|
| 672 |
+
face_kps = face.kps
|
| 673 |
+
face_kps_image = draw_kps(resized_image, face_kps)
|
| 674 |
+
|
| 675 |
+
# ENHANCED: Extract comprehensive facial attributes
|
| 676 |
+
from utils import get_facial_attributes, build_enhanced_prompt
|
| 677 |
+
facial_attrs = get_facial_attributes(face)
|
| 678 |
|
| 679 |
+
# Update prompt with detected attributes
|
| 680 |
+
prompt = build_enhanced_prompt(prompt, facial_attrs, TRIGGER_WORD[lora_choice])
|
| 681 |
+
|
| 682 |
+
# Legacy output for compatibility
|
| 683 |
+
age = facial_attrs['age']
|
| 684 |
+
gender_code = facial_attrs['gender']
|
| 685 |
+
det_score = facial_attrs['quality']
|
| 686 |
+
|
| 687 |
+
gender_str = 'M' if gender_code == 1 else ('F' if gender_code == 0 else 'N/A')
|
| 688 |
+
print(f"Face info: bbox={face.bbox}, age={age if age else 'N/A'}, gender={gender_str}")
|
| 689 |
+
print(f"Face crop size: {face_crop.size}, enhanced: {face_crop_enhanced.size if face_crop_enhanced else 'N/A'}")
|
| 690 |
else:
|
| 691 |
+
print("✗ InsightFace found no faces")
|
| 692 |
+
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 693 |
except Exception as e:
|
| 694 |
+
print(f"[ERROR] InsightFace detection failed: {e}")
|
| 695 |
+
traceback.print_exc()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 696 |
else:
|
| 697 |
+
print("[INFO] InsightFace not available (face_app is None)")
|
|
|
|
|
|
|
|
|
|
|
|
|
| 698 |
|
| 699 |
+
# If InsightFace didn't succeed, try MediapipeFace
|
| 700 |
+
if not insightface_success:
|
| 701 |
+
if self.mediapipe_face is not None:
|
| 702 |
+
print("Trying MediapipeFaceDetector as fallback...")
|
|
|
|
| 703 |
|
| 704 |
+
try:
|
| 705 |
+
# MediapipeFace returns an annotated image with keypoints
|
| 706 |
+
mediapipe_result = self.mediapipe_face(resized_image)
|
| 707 |
+
|
| 708 |
+
# Check if face was detected (result is not blank/black)
|
| 709 |
+
mediapipe_array = np.array(mediapipe_result)
|
| 710 |
+
if mediapipe_array.sum() > 1000: # If image has significant content
|
| 711 |
+
has_detected_faces = True
|
| 712 |
+
face_kps_image = mediapipe_result
|
| 713 |
+
print(f"✓ MediapipeFace detected face(s)")
|
| 714 |
+
print(f"[INFO] Using MediapipeFace keypoints (no embeddings available)")
|
| 715 |
+
|
| 716 |
+
# Note: MediapipeFace doesn't provide embeddings or detailed info
|
| 717 |
+
# So face_embeddings, face_crop_enhanced remain None
|
| 718 |
+
# InstantID will work with keypoints only (reduced quality)
|
| 719 |
+
else:
|
| 720 |
+
print("✗ MediapipeFace found no faces")
|
| 721 |
+
except Exception as e:
|
| 722 |
+
print(f"[ERROR] MediapipeFace detection failed: {e}")
|
| 723 |
+
traceback.print_exc()
|
| 724 |
+
else:
|
| 725 |
+
print("[INFO] MediapipeFaceDetector not available")
|
| 726 |
+
|
| 727 |
+
# Final summary
|
| 728 |
+
if not has_detected_faces:
|
| 729 |
+
print("\n[SUMMARY] No faces detected by any detector")
|
| 730 |
+
if insightface_tried:
|
| 731 |
+
print(" - InsightFace: tried, found nothing")
|
| 732 |
+
else:
|
| 733 |
+
print(" - InsightFace: not available")
|
| 734 |
|
| 735 |
+
if self.mediapipe_face is not None:
|
| 736 |
+
print(" - MediapipeFace: tried, found nothing")
|
| 737 |
+
else:
|
| 738 |
+
print(" - MediapipeFace: not available")
|
| 739 |
+
print()
|
| 740 |
+
|
| 741 |
+
# Set LORA
|
| 742 |
+
if hasattr(self.pipe, 'set_adapters') and self.models_loaded['lora']:
|
| 743 |
+
adapter_name = lora_choice.lower() # "retroart", "vga", "lucasart", or "none"
|
| 744 |
+
|
| 745 |
+
if adapter_name != "none" and self.loaded_loras.get(adapter_name, False):
|
| 746 |
+
try:
|
| 747 |
+
self.pipe.set_adapters([adapter_name], adapter_weights=[lora_scale])
|
| 748 |
+
print(f"LORA: Set adapter '{adapter_name}' with scale: {lora_scale}")
|
| 749 |
except Exception as e:
|
| 750 |
+
print(f"Could not set LORA adapter '{adapter_name}': {e}")
|
| 751 |
+
self.pipe.set_adapters([]) # Disable LORAs if setting failed
|
|
|
|
| 752 |
else:
|
| 753 |
+
if adapter_name == "none":
|
| 754 |
+
print("LORAs disabled by user choice.")
|
| 755 |
+
else:
|
| 756 |
+
print(f"LORA '{adapter_name}' not loaded or available, disabling LORAs.")
|
| 757 |
+
self.pipe.set_adapters([]) # Disable all LORAs
|
| 758 |
+
|
| 759 |
+
|
| 760 |
+
# Prepare generation kwargs
|
| 761 |
+
pipe_kwargs = {
|
| 762 |
+
"image": resized_image,
|
| 763 |
+
"strength": strength,
|
| 764 |
+
"num_inference_steps": num_inference_steps,
|
| 765 |
+
"guidance_scale": guidance_scale,
|
| 766 |
+
}
|
| 767 |
+
|
| 768 |
+
# Setup generator with seed control
|
| 769 |
+
if seed == -1:
|
| 770 |
+
generator = torch.Generator(device=self.device)
|
| 771 |
+
actual_seed = generator.seed()
|
| 772 |
+
print(f"[SEED] Using random seed: {actual_seed}")
|
| 773 |
+
else:
|
| 774 |
+
generator = torch.Generator(device=self.device).manual_seed(seed)
|
| 775 |
+
actual_seed = seed
|
| 776 |
+
print(f"[SEED] Using fixed seed: {actual_seed}")
|
| 777 |
+
|
| 778 |
+
pipe_kwargs["generator"] = generator
|
| 779 |
+
|
| 780 |
+
# --- START FIX: Use Compel instead of Cappella ---
|
| 781 |
+
if self.use_compel and self.compel is not None:
|
| 782 |
+
try:
|
| 783 |
+
print("Encoding prompts with Compel...")
|
| 784 |
+
conditioning = self.compel(prompt)
|
| 785 |
+
negative_conditioning = self.compel(negative_prompt)
|
| 786 |
+
|
| 787 |
+
pipe_kwargs["prompt_embeds"] = conditioning[0]
|
| 788 |
+
pipe_kwargs["pooled_prompt_embeds"] = conditioning[1]
|
| 789 |
+
pipe_kwargs["negative_prompt_embeds"] = negative_conditioning[0]
|
| 790 |
+
pipe_kwargs["negative_pooled_prompt_embeds"] = negative_conditioning[1]
|
| 791 |
+
|
| 792 |
+
print(f"[OK] Compel encoded - Prompt: {pipe_kwargs['prompt_embeds'].shape}, Negative: {pipe_kwargs['negative_prompt_embeds'].shape}")
|
| 793 |
+
except Exception as e:
|
| 794 |
+
print(f"Compel encoding failed, using standard prompts: {e}")
|
| 795 |
+
traceback.print_exc()
|
| 796 |
pipe_kwargs["prompt"] = prompt
|
| 797 |
pipe_kwargs["negative_prompt"] = negative_prompt
|
| 798 |
+
else:
|
| 799 |
+
print("[WARNING] Compel not found, using standard prompt encoding.")
|
| 800 |
+
pipe_kwargs["prompt"] = prompt
|
| 801 |
+
pipe_kwargs["negative_prompt"] = negative_prompt
|
| 802 |
+
# --- END FIX ---
|
| 803 |
+
|
| 804 |
+
# Add CLIP skip
|
| 805 |
+
if hasattr(self.pipe, 'text_encoder'):
|
| 806 |
+
pipe_kwargs["clip_skip"] = 2
|
| 807 |
+
|
| 808 |
+
control_images = []
|
| 809 |
+
conditioning_scales = []
|
| 810 |
+
scale_debug_str = []
|
| 811 |
+
|
| 812 |
+
# Helper function to ensure control image has correct dimensions
|
| 813 |
+
def ensure_correct_size(img, target_w, target_h, name="control"):
|
| 814 |
+
"""Ensure image matches target dimensions exactly"""
|
| 815 |
+
if img is None:
|
| 816 |
+
return Image.new("RGB", (target_w, target_h), (0,0,0))
|
| 817 |
|
| 818 |
+
if img.size != (target_w, target_h):
|
| 819 |
+
print(f" [RESIZE] {name}: {img.size} -> ({target_w}, {target_h})")
|
| 820 |
+
img = img.resize((target_w, target_h), Image.LANCZOS)
|
| 821 |
+
return img
|
| 822 |
+
|
| 823 |
+
# --- START FIX: Re-written IP-Adapter/ControlNet logic ---
|
| 824 |
+
|
| 825 |
+
# 1. InstantID (Identity)
|
| 826 |
+
if self.instantid_active:
|
| 827 |
+
if has_detected_faces and face_kps_image is not None and face_embeddings is not None:
|
| 828 |
+
# Case 1: Face + Embeddings found
|
| 829 |
+
|
| 830 |
+
# A. Set the IP-Adapter (face) strength
|
| 831 |
+
boosted_scale = identity_preservation * IDENTITY_BOOST_MULTIPLIER
|
| 832 |
+
self.pipe.set_ip_adapter_scale(boosted_scale)
|
| 833 |
+
|
| 834 |
+
# B. Pass the raw 512-dim face embeddings to the pipeline
|
| 835 |
+
pipe_kwargs["image_embeds"] = face_embeddings
|
| 836 |
+
|
| 837 |
+
# C. Add the face keypoints (ControlNet) image
|
| 838 |
+
face_kps_image = ensure_correct_size(face_kps_image, target_width, target_height, "InstantID")
|
| 839 |
+
control_images.append(face_kps_image)
|
| 840 |
+
conditioning_scales.append(identity_control_scale)
|
| 841 |
+
|
| 842 |
+
scale_debug_str.append(f"Identity (IP): {boosted_scale:.2f}")
|
| 843 |
+
scale_debug_str.append(f"Identity (CN): {identity_control_scale:.2f}")
|
| 844 |
+
print(f"[OK] InstantID active: IP-Adapter scale set to {boosted_scale:.2f}, ControlNet scale set to {identity_control_scale:.2f}")
|
| 845 |
+
|
| 846 |
+
elif has_detected_faces:
|
| 847 |
+
# Case 2: Face detected (e.g., Mediapipe) but no embeddings available
|
| 848 |
+
print("[INSTANTID] Using keypoints only (no face embeddings for IP-Adapter).")
|
| 849 |
+
|
| 850 |
+
# A. Turn off IP-Adapter
|
| 851 |
+
self.pipe.set_ip_adapter_scale(0.0)
|
| 852 |
+
|
| 853 |
+
# B. Pass dummy embeddings to prevent crash
|
| 854 |
+
pipe_kwargs["image_embeds"] = np.zeros(512)
|
| 855 |
+
|
| 856 |
+
# C. Add face keypoints (ControlNet)
|
| 857 |
+
face_kps_image = ensure_correct_size(face_kps_image, target_width, target_height, "InstantID")
|
| 858 |
+
control_images.append(face_kps_image)
|
| 859 |
+
conditioning_scales.append(identity_control_scale) # Use the CN scale
|
| 860 |
+
|
| 861 |
+
scale_debug_str.append("Identity (IP): 0.00")
|
| 862 |
+
scale_debug_str.append(f"Identity (CN): {identity_control_scale:.2f}")
|
| 863 |
+
|
| 864 |
else:
|
| 865 |
+
# Case 3: No face detected at all
|
| 866 |
+
print("[INSTANTID] No face detected. Disabling face identity.")
|
| 867 |
|
| 868 |
+
# A. Turn off IP-Adapter
|
| 869 |
+
self.pipe.set_ip_adapter_scale(0.0)
|
|
|
|
| 870 |
|
| 871 |
+
# B. Pass dummy embeddings to prevent crash
|
| 872 |
+
pipe_kwargs["image_embeds"] = np.zeros(512)
|
|
|
|
| 873 |
|
| 874 |
+
# C. Add blank image for ControlNet (to keep list order)
|
| 875 |
+
control_images.append(Image.new("RGB", (target_width, target_height), (0,0,0)))
|
| 876 |
+
conditioning_scales.append(0.0) # Set CN scale to 0
|
| 877 |
|
| 878 |
+
scale_debug_str.append("Identity (IP): 0.00")
|
| 879 |
+
scale_debug_str.append("Identity (CN): 0.00")
|
| 880 |
+
|
| 881 |
+
# --- END FIX ---
|
| 882 |
+
|
| 883 |
+
# 2. Depth
|
| 884 |
+
if self.depth_active:
|
| 885 |
+
# Ensure depth image has correct size
|
| 886 |
+
depth_image = ensure_correct_size(depth_image, target_width, target_height, "Depth")
|
| 887 |
+
control_images.append(depth_image)
|
| 888 |
+
conditioning_scales.append(depth_control_scale)
|
| 889 |
+
scale_debug_str.append(f"Depth: {depth_control_scale:.2f}")
|
| 890 |
+
|
| 891 |
+
# 3. OpenPose (Expression)
|
| 892 |
+
if self.openpose_active:
|
| 893 |
+
# Ensure openpose image has correct size
|
| 894 |
+
openpose_image = ensure_correct_size(openpose_image, target_width, target_height, "OpenPose")
|
| 895 |
+
control_images.append(openpose_image)
|
| 896 |
+
conditioning_scales.append(expression_control_scale)
|
| 897 |
+
scale_debug_str.append(f"Expression: {expression_control_scale:.2f}")
|
| 898 |
+
|
| 899 |
+
# Final validation: ensure all control images have identical dimensions
|
| 900 |
+
if control_images:
|
| 901 |
+
expected_size = (target_width, target_height)
|
| 902 |
+
for idx, img in enumerate(control_images):
|
| 903 |
+
if img.size != expected_size:
|
| 904 |
+
print(f" [WARNING] Control image {idx} size mismatch: {img.size} vs expected {expected_size}")
|
| 905 |
+
control_images[idx] = img.resize(expected_size, Image.LANCZOS)
|
| 906 |
|
| 907 |
+
pipe_kwargs["control_image"] = control_images
|
| 908 |
+
pipe_kwargs["controlnet_conditioning_scale"] = conditioning_scales
|
| 909 |
+
print(f"Active ControlNets: {len(control_images)} (all {target_width}x{target_height})")
|
| 910 |
+
else:
|
| 911 |
+
print("No active ControlNets, running standard Img2Img")
|
| 912 |
+
|
| 913 |
+
# Generate
|
| 914 |
+
print(f"Generating with LCM: Steps={num_inference_steps}, CFG={guidance_scale}, Strength={strength}")
|
| 915 |
+
print(f"Controlnet scales - {' | '.join(scale_debug_str)}")
|
| 916 |
+
result = self.pipe(**pipe_kwargs)
|
| 917 |
+
|
| 918 |
+
generated_image = result.images[0]
|
| 919 |
+
|
| 920 |
+
# Post-processing
|
| 921 |
+
if enable_color_matching and has_detected_faces:
|
| 922 |
+
print("Applying enhanced face-aware color matching...")
|
| 923 |
+
try:
|
| 924 |
+
if face_bbox_original is not None:
|
| 925 |
+
generated_image = enhanced_color_match(
|
| 926 |
+
generated_image,
|
| 927 |
+
resized_image,
|
| 928 |
+
face_bbox=face_bbox_original
|
| 929 |
+
)
|
| 930 |
+
print("[OK] Enhanced color matching applied (face-aware)")
|
| 931 |
+
else:
|
| 932 |
generated_image = color_match(generated_image, resized_image, mode='mkl')
|
| 933 |
print("[OK] Standard color matching applied")
|
| 934 |
+
except Exception as e:
|
| 935 |
+
print(f"Color matching failed: {e}")
|
| 936 |
+
elif enable_color_matching:
|
| 937 |
+
print("Applying standard color matching...")
|
| 938 |
+
try:
|
| 939 |
+
generated_image = color_match(generated_image, resized_image, mode='mkl')
|
| 940 |
+
print("[OK] Standard color matching applied")
|
| 941 |
+
except Exception as e:
|
| 942 |
+
print(f"Color matching failed: {e}")
|
| 943 |
|
| 944 |
+
return generated_image
|
|
|
|
|
|
|
|
|
|
|
|
|
| 945 |
|
| 946 |
|
| 947 |
+
print("[OK] Generator class ready")
|