Update app.py

app.py

@@ -30,12 +30,16 @@ controlnets = [
     ),
 ]
 
-pipe_CN = StableDiffusionXLControlNetPipeline.from_pretrained("SG161222/RealVisXL_V5.0", torch_dtype=torch.float16,controlnet=controlnets, use_safetensors=True, variant='fp16')
-pipe_CN.vae = AutoencoderTiny.from_pretrained("madebyollin/taesdxl", torch_dtype=torch.float16)
+pipe_CN = StableDiffusionXLControlNetPipeline.from_pretrained("SG161222/RealVisXL_V5.0", torch_dtype=torch.float16,controlnet=[controlnets[0],controlnets[0]], use_safetensors=True, variant='fp16')
+###pipe_CN.vae = AutoencoderTiny.from_pretrained("madebyollin/taesdxl", torch_dtype=torch.float16)
 pipe_CN.scheduler=DPMSolverSDEScheduler.from_pretrained("SG161222/RealVisXL_V5.0",subfolder="scheduler",use_karras_sigmas=True)
 
 pipe_CN.to("cuda")
-pipe_CN.load_lora_weights('CreativesCombined/hb8_cases_dreambooth_lora_test_1_14', weight_name='pytorch_lora_weights.safetensors',adapter_name='cases')
+state_dict, network_alphas = StableDiffusionXLControlNetPipeline.lora_state_dict('CreativesCombined/hb8_cases_dreambooth_lora_test_1_14', weight_name='pytorch_lora_weights.safetensors')
+pipe_CN.load_lora_into_unet(state_dict, network_alphas, pipe_CN.unet, adapter_name='unet_cases')
+pipe_CN.load_lora_into_text_encoder(state_dict, network_alphas, pipe_CN.text_encoder, adapter_name='text_cases')
+pipe_CN.load_lora_into_text_encoder(state_dict, network_alphas, pipe_CN.text_encoder ,prefix='2', adapter_name='text_2_cases')
+pipe_CN.set_adapters(["unet_cases","text_cases","text_2_cases"], adapter_weights=[1.0, 0.5,0.5])
 
 refiner = StableDiffusionXLImg2ImgPipeline.from_pretrained("stabilityai/stable-diffusion-xl-refiner-1.0",text_encoder_2=pipe_CN.text_encoder_2,vae=pipe_CN.vae,torch_dtype=torch.float16,use_safetensors=True,variant="fp16")
 refiner.to("cuda")
@@ -44,7 +48,16 @@ pipe_IN = StableDiffusionXLControlNetInpaintPipeline.from_pretrained("diffusers/
 pipe_IN.load_lora_weights('Tonioesparza/ourhood_training_dreambooth_lora_2_0', weight_name='pytorch_lora_weights.safetensors',adapter_name='ourhood')
 pipe_IN.to("cuda")
 
-def ourhood_inference(prompt1=str,num_inference_steps=int,scaffold=int,seed=int):
+def make_inpaint_condition(image, image_mask):
+    image = np.array(image.convert("RGB")).astype(np.float32) / 255.0
+    image_mask = np.array(image_mask.convert("L")).astype(np.float32) / 255.0
+    assert image.shape[0:1] == image_mask.shape[0:1]
+    image[image_mask > 0.5] = -1.0  # set as masked pixel
+    image = np.expand_dims(image, 0).transpose(0, 3, 1, 2)
+    image = torch.from_numpy(image)
+    return image
+
+def ourhood_inference(prompt=str,num_inference_steps=int,scaffold=int,seed=int):
 
 ###pro_encode = pipe_cn.encode_text(prompt)
 
@@ -76,6 +89,9 @@ def ourhood_inference(prompt1=str,num_inference_steps=int,scaffold=int,seed=int)
     mask1 = load_image(scaff_dic[scaffold]['mask1'])
     masks = processor_mask.preprocess([mask1], height=output_height, width=output_width)
     masks = [masks.reshape(1, masks.shape[0], masks.shape[2], masks.shape[3])]
+
+    
+    
     ###ip_images init
     ###ip_img_1 = load_image(r"C:\Users\AntonioEsparzaGlisma\PycharmProjects\hB8\Cases\a-place-to_210930_HAY_A-PLACE-TO_091-768x1024.png")
     ###ip_images = [[ip_img_1]]
@@ -84,8 +100,13 @@ def ourhood_inference(prompt1=str,num_inference_steps=int,scaffold=int,seed=int)
     ###precomputed depth image
     depth_image = load_image(scaff_dic[scaffold]['depth_image'])
     canny_image = load_image(scaff_dic[scaffold]['canny_image'])
+
+    masked_depth=make_inpaint_condition(depth_image,mask1)
+    
     images_CN = [depth_image, canny_image]
 
+    
+    prompt1 = 'A frontpage still-life photograph, an 8-foot wooden crate, '+ prompt
     neg1 = 'text,watermark'
     prompt2 = 'Photorealistic rendering, of an OurHood privacy booth, with a silken oak frame, hickory stained melange polyester fabric, windows'
     neg2 = 'curtains, pillows'
@@ -98,8 +119,8 @@ def ourhood_inference(prompt1=str,num_inference_steps=int,scaffold=int,seed=int)
             num_inference_steps=n_steps,
             num_images_per_prompt=1,
             generator=generator,
-            denoising_end=0.8,
-            image=images_CN,
+            denoising_end=0.9,
+            image=[depth_image,masked_depth],
             output_type="latent",
             control_guidance_start=[0.0,0.5],
             control_guidance_end=[0.5,1.0],
@@ -113,7 +134,7 @@ def ourhood_inference(prompt1=str,num_inference_steps=int,scaffold=int,seed=int)
     image = refiner(
         prompt=prompt1,
         num_inference_steps=n_steps,
-        denoising_start=0.8,
+        denoising_start=0.9,
         image=results).images[0]
 
 
@@ -125,10 +146,10 @@ def ourhood_inference(prompt1=str,num_inference_steps=int,scaffold=int,seed=int)
             negative_prompt=neg2,
             image=image,
             mask_image=mask1,
-            num_inference_steps=n_steps,
+            num_inference_steps=65,
             strength=1.0,
             control_guidance_end=[0.9,0.9],
-            controlnet_conditioning_scale=[0.3, 0.45],
+            controlnet_conditioning_scale=[0.35, 0.65],
             control_image=images_CN,
             generator=generator,
         ).images[0]