import torch import os import cv2 import numpy as np from config import Config from diffusers import ( ControlNetModel, LCMScheduler ) from diffusers.models.controlnets.multicontrolnet import MultiControlNetModel # Import the custom pipeline from your local file from pipeline_stable_diffusion_xl_instantid_img2img import StableDiffusionXLInstantIDImg2ImgPipeline from huggingface_hub import snapshot_download, hf_hub_download from insightface.app import FaceAnalysis # --- MODIFIED: Import new detectors --- from controlnet_aux import LeresDetector, LineartAnimeDetector # --- END MODIFIED --- class ModelHandler: def __init__(self): self.pipeline = None self.app = None # InsightFace # --- MODIFIED: Rename detectors --- self.leres_detector = None self.lineart_anime_detector = None # --- END MODIFIED --- self.face_analysis_loaded = False def load_face_analysis(self): """ Load face analysis model. Downloads from HF Hub to the path insightface expects. Forces CPU to avoid ZeroGPU initialization errors. """ print("Loading face analysis model...") # insightface expects models in '{root}/models/{name}' # Since our root='.' and name='antelopev2', the expected path is './models/antelopev2' model_path = os.path.join(Config.ANTELOPEV2_ROOT, "models", Config.ANTELOPEV2_NAME) if not os.path.exists(os.path.join(model_path, "scrfd_10g_bnkps.onnx")): print(f"Downloading AntelopeV2 models from {Config.ANTELOPEV2_REPO} to {model_path}...") try: snapshot_download( repo_id=Config.ANTELOPEV2_REPO, local_dir=model_path, # Download to the correct expected path ) except Exception as e: print(f" [ERROR] Failed to download AntelopeV2 models: {e}") return False try: # Initialize with root='.' and name='antelopev2' self.app = FaceAnalysis( name=Config.ANTELOPEV2_NAME, root=Config.ANTELOPEV2_ROOT, providers=['CPUExecutionProvider'] ) self.app.prepare(ctx_id=0, det_size=(640, 640)) print(f" [OK] Face analysis model loaded successfully.") return True except Exception as e: print(f" [WARNING] Face detection system failed to initialize: {e}") return False def load_models(self): # 1. Load Face Analysis self.face_analysis_loaded = self.load_face_analysis() # 2. Load ControlNets print("Loading ControlNets (InstantID, Zoe, LineArt)...") # Load the InstantID ControlNet from the correct subfolder print("Loading InstantID ControlNet from subfolder 'ControlNetModel'...") cn_instantid = ControlNetModel.from_pretrained( Config.INSTANTID_REPO, # "InstantX/InstantID" subfolder="ControlNetModel", # Correct casing torch_dtype=Config.DTYPE ) print(" [OK] Loaded InstantID ControlNet.") # Load other ControlNets normally print("Loading Zoe and LineArt ControlNets...") cn_zoe = ControlNetModel.from_pretrained(Config.CN_ZOE_REPO, torch_dtype=Config.DTYPE) cn_lineart = ControlNetModel.from_pretrained(Config.CN_LINEART_REPO, torch_dtype=Config.DTYPE) # --- Manually wrap the list of models in a MultiControlNetModel --- print("Wrapping ControlNets in MultiControlNetModel...") controlnet_list = [cn_instantid, cn_zoe, cn_lineart] controlnet = MultiControlNetModel(controlnet_list) # --- End wrapping --- # 3. Load SDXL Pipeline print(f"Loading SDXL Pipeline ({Config.CHECKPOINT_FILENAME})...") # Manually download the checkpoint file first. checkpoint_local_path = os.path.join("./models", Config.CHECKPOINT_FILENAME) if not os.path.exists(checkpoint_local_path): print(f"Downloading checkpoint to {checkpoint_local_path}...") hf_hub_download( repo_id=Config.REPO_ID, filename=Config.CHECKPOINT_FILENAME, local_dir="./models", local_dir_use_symlinks=False ) # Use the custom Img2Img pipeline class you provided, loading from the LOCAL FILE print(f"Loading pipeline from local file: {checkpoint_local_path}") self.pipeline = StableDiffusionXLInstantIDImg2ImgPipeline.from_single_file( checkpoint_local_path, # Pass the local path controlnet=controlnet, # Pass the single, wrapped object torch_dtype=Config.DTYPE, use_safetensors=True ) self.pipeline.to(Config.DEVICE) # 4. Set Scheduler self.pipeline.scheduler = LCMScheduler.from_config(self.pipeline.scheduler.config) # 5. Load Adapters (IP-Adapter & LoRA) print("Loading Adapters (IP-Adapter & LoRA)...") # Download the ip-adapter.bin file and pass its local path ip_adapter_filename = "ip-adapter.bin" ip_adapter_local_path = os.path.join("./models", ip_adapter_filename) if not os.path.exists(ip_adapter_local_path): print(f"Downloading IP-Adapter to {ip_adapter_local_path}...") hf_hub_download( repo_id=Config.INSTANTID_REPO, filename=ip_adapter_filename, local_dir="./models", local_dir_use_symlinks=False ) print(f"Loading IP-Adapter from local file: {ip_adapter_local_path}") self.pipeline.load_ip_adapter_instantid(ip_adapter_local_path) # Pass local path print("Loading LoRA weights...") self.pipeline.load_lora_weights(Config.REPO_ID, weight_name=Config.LORA_FILENAME) # --- NEW: Fuse LoRA at build time with fixed strength --- print(f"Fusing LoRA with scale {Config.LORA_STRENGTH}...") self.pipeline.fuse_lora(lora_scale=Config.LORA_STRENGTH) print(" [OK] LoRA fused.") # --- DISABLED torch.compile due to runtime errors --- # try: # print("Compiling UNet with torch.compile...") # self.pipeline.unet = torch.compile(self.pipeline.unet, mode="reduce-overhead", fullgraph=True) # print(" [OK] UNet compiled.") # except Exception as e: # print(f" [WARNING] torch.compile failed: {e}. Running without compilation.") # 6. Load Preprocessors # --- MODIFIED: Load new detectors --- print("Loading Preprocessors (LeReS, LineArtAnime)...") self.leres_detector = LeresDetector.from_pretrained(Config.ANNOTATOR_REPO) self.lineart_anime_detector = LineartAnimeDetector.from_pretrained(Config.ANNOTATOR_REPO) # --- END MODIFIED --- print("--- All models loaded successfully ---") def get_face_embedding(self, image): """Extracts face embedding, returns None if no face is found.""" if not self.face_analysis_loaded: return None try: # Convert PIL to CV2 # --- FIX: Corrected OpenCV attribute --- cv2_img = cv2.cvtColor(np.array(image), cv2.COLOR_RGB2BGR) faces = self.app.get(cv2_img) if len(faces) == 0: return None # Sort by size (width * height) to find the main character faces = sorted(faces, key=lambda x: (x['bbox'][2]-x['bbox'][0])*(x['bbox'][3]-x['bbox'][1]), reverse=True) # Return the largest face return torch.tensor(faces[0].normed_embedding).unsqueeze(0) except Exception as e: print(f"Face embedding extraction failed: {e}") return None