Update README.md

README.md

@@ -22,20 +22,73 @@ Unlike the inpaint controlnets used for general scenarios, this model is fine-tu
 
 <span style="width: 150px !important;display: inline-block;">`Foreground`<span> | <span style="width: 150px !important;display: inline-block;">`Mask`<span> | <span style="width: 150px !important;display: inline-block;">`w/o instance mask`<span> | <span style="width: 150px !important;display: inline-block;">`w/ instance mask`<span>
 :--:|:--:|:--:|:--:
-![images)](./images/inp_0.png) | ![images)](./images/inp_1.png) | ![images)](./images/inp_3.png) | ![images)](./images/inp_3.png)
-<!-- <img src="https://huggingface.co/alimama-creative/EcomXL/resolve/main/images/inp_0.png" width="300"/> | <img src="https://huggingface.co/alimama-creative/EcomXL/resolve/main/images/inp_1.png" width="300"/> | <img src="https://huggingface.co/alimama-creative/EcomXL/resolve/main/images/inp_2.png" width="300"/> | <img src="https://huggingface.co/alimama-creative/EcomXL/resolve/main/images/inp_3.png" width="300"/> -->
-
-Using this ControlNet with a control weight of 0.5 may achieve better results.
+![images)](./images/inp_0.png) | ![images)](./images/inp_1.png) | ![images)](./images/inp_2.png) | ![images)](./images/inp_3.png)
+![images)](./images/inp1_0.png) | ![images)](./images/inp1_1.png) | ![images)](./images/inp1_2.png) | ![images)](./images/inp1_3.png)
+![images)](./images/inp2_0.png) | ![images)](./images/inp2_1.png) | ![images)](./images/inp2_2.png) | ![images)](./images/inp2_3.png)
 
 ## Usage with Diffusers
 ```python
-from diffusers import ControlNetModel
+from diffusers import (
+    ControlNetModel,
+    StableDiffusionXLControlNetInpaintPipeline
+)
+from diffusers.utils import load_image
 import torch
+from PIL import Image
+
+def make_inpaint_condition(init_image, mask_image):
+    init_image = np.array(init_image.convert("RGB")).astype(np.float32) / 255.0
+    mask_image = np.array(mask_image.convert("L")).astype(np.float32) / 255.0
+
+    assert init_image.shape[0:1] == mask_image.shape[0:1], "image and image_mask must have the same image size"
+    init_image[mask_image > 0.5] = -1.0  # set as masked pixel
+    init_image = np.expand_dims(init_image, 0).transpose(0, 3, 1, 2)
+    init_image = torch.from_numpy(init_image)
+    return init_image
+
 
 controlnet = ControlNetModel.from_pretrained(
     "alimama-creative/EcomXL_controlnet_inpaint", torch_dtype=torch.float16, use_safetensors=True
 )
+
+pipe = StableDiffusionXLControlNetInpaintPipeline.from_pretrained(
+    "stabilityai/stable-diffusion-xl-base-1.0", 
+    controlnet=controlnet, 
+    torch_dtype=torch.float16
+)
+
+pipe.scheduler = DPMSolverMultistepScheduler.from_config(pipe.scheduler.config)
+# pipe.enable_xformers_memory_efficient_attention()
+pipe.enable_vae_slicing()
+
+image = load_image(
+    "https://huggingface.co/alimama-creative/EcomXL_controlnet_inpaint/resolve/main/images/inp_0.png"
+)
+mask = load_image(
+    "https://huggingface.co/alimama-creative/EcomXL_controlnet_inpaint/resolve/main/images/inp_1.png"
+)
+mask = Image.fromarray(255 - np.array(mask))
+
+control_image = make_inpaint_condition(img, mask)
+
+prompt="a product on the table"
+
+images = pipe(
+    prompt,
+    image=img,
+    mask_image=mask,
+    control_image=control_image,
+    controlnet_conditioning_scale=0.5,
+    guidance_scale=7,
+    strength=0.75,
+    width=1024,
+    height=1024,
+).images[0]
+
+image.save(f'test_inp.png')
+
 ```
+The model exhibits good performance when the controlnet weight (controllet_condition_scale) is 0.5.
 
 ## Training details
 In the first phase, the model was trained on 12M laion2B and internal source images with random masks for 20k steps. In the second phase, the model was trained on 3M e-commerce images with the instance mask for 20k steps.<br>