try fix controlnet batch processing

pipeline.py

@@ -980,6 +980,24 @@ class StableDiffusionControlNetImg2ImgPipeline(DiffusionPipeline, TextualInversi
                 # compute the percentage of total steps we are at
                 current_sampling_percent = i / len(timesteps)
 
+                # if (
+                #     current_sampling_percent < controlnet_guidance_start
+                #     or current_sampling_percent > controlnet_guidance_end
+                # ):
+                #     # do not apply the controlnet
+                #     down_block_res_samples = None
+                #     mid_block_res_sample = None
+                # else:
+                #     # apply the controlnet
+                #     down_block_res_samples, mid_block_res_sample = self.controlnet(
+                #         latent_model_input,
+                #         t,
+                #         encoder_hidden_states=prompt_embeds,
+                #         controlnet_cond=controlnet_conditioning_image,
+                #         conditioning_scale=controlnet_conditioning_scale,
+                #         return_dict=False,
+                #     )
+
                 if (
                     current_sampling_percent < controlnet_guidance_start
                     or current_sampling_percent > controlnet_guidance_end
@@ -988,15 +1006,24 @@ class StableDiffusionControlNetImg2ImgPipeline(DiffusionPipeline, TextualInversi
                     down_block_res_samples = None
                     mid_block_res_sample = None
                 else:
-                    # apply the controlnet
-                    down_block_res_samples, mid_block_res_sample = self.controlnet(
-                        latent_model_input,
-                        t,
-                        encoder_hidden_states=prompt_embeds,
-                        controlnet_cond=controlnet_conditioning_image,
-                        conditioning_scale=controlnet_conditioning_scale,
-                        return_dict=False,
-                    )
+                    down_block_res_samples = []
+                    mid_block_res_samples = []
+                    for i in range(batch_size):
+                        # apply the controlnet
+                        down_block_res_sample, mid_block_res_sample = self.controlnet(
+                            latent_model_input[i * num_images_per_prompt:(i + 1) * num_images_per_prompt],
+                            t,
+                            encoder_hidden_states=prompt_embeds,
+                            controlnet_cond=controlnet_conditioning_image[i],
+                            conditioning_scale=controlnet_conditioning_scale,
+                            return_dict=False,
+                        )
+                        down_block_res_samples.append(down_block_res_sample)
+                        mid_block_res_samples.append(mid_block_res_sample)
+                
+                    down_block_res_samples = torch.cat(down_block_res_samples, dim=0)
+                    mid_block_res_sample = torch.cat(mid_block_res_samples, dim=0)
+
 
                 # predict the noise residual
                 noise_pred = self.unet(