Modified pipeline ONNX

code_inference/pipeline_sdxl_cnext_ipadapter.py

@@ -310,7 +310,7 @@ class StableDiffusionXLControlNeXtPipeline():
         controlnext: ort.InferenceSession,
         image_proj:  ort.InferenceSession,
         scheduler: DDPMScheduler,
-        image_encoder: CLIPVisionModelWithProjection = None,
+        image_encoder: ort.InferenceSession,
         feature_extractor: CLIPImageProcessor = None,
         add_watermarker: Optional[bool] = None,
         device=None,
@@ -1122,8 +1122,8 @@ class StableDiffusionXLControlNeXtPipeline():
             image_prompt_embeds = image_prompt_embeds.view(bs_embed * num_images_per_prompt, seq_len, -1)
             uncond_image_prompt_embeds = uncond_image_prompt_embeds.repeat(1, num_images_per_prompt, 1)
             uncond_image_prompt_embeds = uncond_image_prompt_embeds.view(bs_embed * num_images_per_prompt, seq_len, -1)
-            prompt_embeds = torch.cat([torch.zeros_like(prompt_embeds), image_prompt_embeds], dim=1)
-            negative_prompt_embeds = torch.cat([torch.zeros_like(negative_prompt_embeds), uncond_image_prompt_embeds], dim=1)
+            prompt_embeds = torch.cat([prompt_embeds, image_prompt_embeds], dim=1)
+            negative_prompt_embeds = torch.cat([negative_prompt_embeds, uncond_image_prompt_embeds], dim=1)
 
         if self.do_classifier_free_guidance:
             prompt_embeds = torch.cat([negative_prompt_embeds, prompt_embeds], dim=0)
@@ -1182,7 +1182,7 @@ class StableDiffusionXLControlNeXtPipeline():
             controls = self.controlnet.run(None, {'controlnext_image': controlnet_image.cpu().numpy(),
                         'timestep': t.unsqueeze(0).cpu().numpy().astype(np.float32),})
                         
-            scale = torch.tensor([1.00])
+            scale = torch.tensor([control_scale])
 
             noise_pred = self.unet.run(None, {'sample': latent_model_input.cpu().numpy().astype(np.float32),
                         'timestep': t.unsqueeze(0).cpu().numpy().astype(np.float32),