Add files using upload-large-folder tool

.gitattributes

@@ -48,3 +48,55 @@ Training_Data/Inperson_7-Garment-V2.jpg filter=lfs diff=lfs merge=lfs -text
 Training_Data/Venusperson_7-Garment.jpg filter=lfs diff=lfs merge=lfs -text
 Training_Data/Venusperson_1.jpg filter=lfs diff=lfs merge=lfs -text
 Training_Data/Venusperson_2.jpg filter=lfs diff=lfs merge=lfs -text
+Training_Data/TigcPerson_3.jpg filter=lfs diff=lfs merge=lfs -text
+Training_Data/Inperson_4-V2.png filter=lfs diff=lfs merge=lfs -text
+Training_Data/Venusperson_7.jpg filter=lfs diff=lfs merge=lfs -text
+Training_Data/TigcPerson_3-Mask.jpg filter=lfs diff=lfs merge=lfs -text
+Training_Data/TigcPerson_1-Mask.jpg filter=lfs diff=lfs merge=lfs -text
+Training_Data/TigcPerson_5-Mask.jpg filter=lfs diff=lfs merge=lfs -text
+Training_Data/TigcPerson_5.jpg filter=lfs diff=lfs merge=lfs -text
+Training_Data/Pinaperson_1-Garment-V1.jpg filter=lfs diff=lfs merge=lfs -text
+Training_Data/Pinaperson_2-Garment-V1.jpg filter=lfs diff=lfs merge=lfs -text
+Training_Data/TigcPerson_5-Garment.jpg filter=lfs diff=lfs merge=lfs -text
+Training_Data/Inperson_4-Garment-V2.png filter=lfs diff=lfs merge=lfs -text
+Training_Data/TigcPerson_2-Mask.jpg filter=lfs diff=lfs merge=lfs -text
+Training_Data/Inperson_7-V2.jpg filter=lfs diff=lfs merge=lfs -text
+Training_Data/Inperson_5-Garment-V2.jpg filter=lfs diff=lfs merge=lfs -text
+Training_Data/TigcPerson_1.jpg filter=lfs diff=lfs merge=lfs -text
+Training_Data/Venusperson_11.jpg filter=lfs diff=lfs merge=lfs -text
+Training_Data/Pinaperson_3-V1.jpg filter=lfs diff=lfs merge=lfs -text
+Training_Data/Venusperson_10.jpg filter=lfs diff=lfs merge=lfs -text
+Training_Data/Venusperson_2-Garment.jpg filter=lfs diff=lfs merge=lfs -text
+Training_Data/Pinaperson_3-Garment-V1.jpg filter=lfs diff=lfs merge=lfs -text
+Training_Data/Venusperson_6-Garment.jpg filter=lfs diff=lfs merge=lfs -text
+Training_Data/Inperson_3-Mask-V1.png filter=lfs diff=lfs merge=lfs -text
+Training_Data/Venusperson_9.jpg filter=lfs diff=lfs merge=lfs -text
+Training_Data/Venusperson_8.jpg filter=lfs diff=lfs merge=lfs -text
+Training_Data/Inperson_4-Mask-V2.png filter=lfs diff=lfs merge=lfs -text
+Training_Data/Pinaperson_4-Garment-V1.jpg filter=lfs diff=lfs merge=lfs -text
+Training_Data/Inperson_3-Garment-V2.png filter=lfs diff=lfs merge=lfs -text
+Training_Data/Inperson_5-V2.png filter=lfs diff=lfs merge=lfs -text
+Training_Data/Venusperson_4.jpg filter=lfs diff=lfs merge=lfs -text
+Training_Data/Venusperson_3-Garment.jpg filter=lfs diff=lfs merge=lfs -text
+Training_Data/Inperson_7-V2.png filter=lfs diff=lfs merge=lfs -text
+Training_Data/Venusperson_12.jpg filter=lfs diff=lfs merge=lfs -text
+Training_Data/Venusperson_1-Garment.jpg filter=lfs diff=lfs merge=lfs -text
+Training_Data/Venusperson_10-Garment.jpg filter=lfs diff=lfs merge=lfs -text
+Training_Data/Inperson_5-Garment-V2.png filter=lfs diff=lfs merge=lfs -text
+Training_Data/TigcPerson_1-Garment.jpg filter=lfs diff=lfs merge=lfs -text
+Training_Data/Inperson_5-V2.jpg filter=lfs diff=lfs merge=lfs -text
+Training_Data/Pinaperson_4-V1.jpg filter=lfs diff=lfs merge=lfs -text
+Training_Data/Venusperson_6.jpg filter=lfs diff=lfs merge=lfs -text
+Training_Data/Venusperson_3.jpg filter=lfs diff=lfs merge=lfs -text
+Training_Data/Venusperson_8-Garment.jpg filter=lfs diff=lfs merge=lfs -text
+Training_Data/Inperson_3-Garment-V2.jpg filter=lfs diff=lfs merge=lfs -text
+Training_Data/Venusperson_4-Garment.jpg filter=lfs diff=lfs merge=lfs -text
+Training_Data/Pinaperson_2-V1.jpg filter=lfs diff=lfs merge=lfs -text
+Training_Data/Venusperson_9-Garment.jpg filter=lfs diff=lfs merge=lfs -text
+Training_Data/Inperson_7-Garment-V2.png filter=lfs diff=lfs merge=lfs -text
+Training_Data/Venusperson_12-Garment.jpg filter=lfs diff=lfs merge=lfs -text
+Training_Data/Pinaperson_1-V1.jpg filter=lfs diff=lfs merge=lfs -text
+Training_Data/TigcPerson_4.jpg filter=lfs diff=lfs merge=lfs -text
+Training_Data/Inperson_3-V1.png filter=lfs diff=lfs merge=lfs -text
+Training_Data/Venusperson_11-Garment.jpg filter=lfs diff=lfs merge=lfs -text
+Training_Data/TigcPerson_2.jpg filter=lfs diff=lfs merge=lfs -text

ACE_plus/.gitignore

@@ -0,0 +1 @@
+.idea

ACE_plus/README.md

@@ -0,0 +1,600 @@
+<p align="center">
+
+  <h2 align="center"><img src="assets/figures/icon.png" height=16> ++: Instruction-Based Image Creation and Editing <br> via Context-Aware Content Filling </h2>
+
+  <p align="center">
+    <a href="https://arxiv.org/abs/2501.02487"><img src='https://img.shields.io/badge/arXiv-ACE++-red' alt='Paper PDF'></a>
+    <a href='https://ali-vilab.github.io/ACE_plus_page/'><img src='https://img.shields.io/badge/Project_Page-ACE++-blue' alt='Project Page'></a>
+    <a href='https://github.com/modelscope/scepter'><img src='https://img.shields.io/badge/Scepter-ACE++-green'></a>
+    <a href='https://huggingface.co/spaces/scepter-studio/ACE-Plus'><img src='https://img.shields.io/badge/%F0%9F%A4%97%20Hugging%20Face-Space-orange'></a>
+    <a href='https://huggingface.co/ali-vilab/ACE_Plus/tree/main'><img src='https://img.shields.io/badge/%F0%9F%A4%97%20Hugging%20Face-Model-orange'></a>
+    <a href='https://modelscope.cn/models/iic/ACE_Plus/summary'><img src='https://img.shields.io/badge/ModelScope-Model-purple'></a>
+    <br>
+    <strong>Chaojie Mao</strong>
+    ·
+    <strong>Jingfeng Zhang</strong>
+    ·
+    <strong>Yulin Pan</strong>
+    ·
+    <strong>Zeyinzi Jiang</strong>
+    ·
+    <strong>Zhen Han</strong>
+    <br>
+    ·
+    <strong>Yu Liu</strong>
+    ·
+    <strong>Jingren Zhou</strong>
+    <br>
+    Tongyi Lab, Alibaba Group
+  </p>
+  <table align="center">
+    <tr>
+    <td>
+      <img src="assets/ace_method/method++.png">
+    </td>
+    </tr>
+  </table>
+
+## 📚 Introduction
+
+The original intention behind the design of ACE++ was to unify reference image generation, local editing, 
+and controllable generation into a single framework, and to enable one model to adapt to a wider range of tasks. 
+A more versatile model is often capable of handling more complex tasks. We have released three LoRA models for 
+specific vertical domains and a more versatile FFT model (the performance of the FFT model declines compared 
+to the LoRA model across various tasks). Users can flexibly utilize these models and their 
+combinations for their own scenarios. 
+
+## 📢 News
+- [x] **[2025.01.06]** Release the code and models of ACE++.
+- [x] **[2025.01.07]** Release the demo on [HuggingFace](https://huggingface.co/spaces/scepter-studio/ACE-Plus).
+- [x] **[2025.01.16]** Release the training code for lora.
+- [x] **[2025.02.15]** Collection of workflows in Comfyui.
+- [x] **[2025.02.15]** Release the config for fully fine-tuning.
+- [x] **[2025.03.03]** Release the fft model for ACE++, support more image to image tasks.
+- [x] **[2025.03.11]** Release some comfyui workflow examples for ACE++ model. 
+
+- We sincerely apologize 
+for the delayed responses and updates regarding ACE++ issues. 
+Further development of the ACE model through post-training on the FLUX model must be suspended. 
+We have identified several significant challenges in post-training on the FLUX foundation. 
+The primary issue is the high degree of heterogeneity between the training dataset and the FLUX model, 
+which results in highly unstable training. Moreover, FLUX-Dev is a distilled model, and the influence of its original negative prompts on its final performance is uncertain.
+As a result, subsequent efforts will be focused on post-training the ACE model using the Wan series of foundational models. __Due to the reasons mentioned earlier, the performance of the FFT model may decline compared 
+to the LoRA model across various tasks. Therefore, we recommend continuing to use the LoRA model to achieve better results. 
+We provide the FFT model with the hope that it may facilitate academic exploration and research in this area.__
+
+
+## Models
+
+### ACE++ Portrait LoRA
+
+Portrait-consistent generation to maintain the consistency of the portrait.
+
+
+<table><tbody>
+  <tr>
+    <td>Tuning Method</td>
+    <td>Input</td>
+    <td>Output</td>
+    <td>Instruction</td>
+    <td>Models</td>
+  </tr>
+  <tr>
+    <td>LoRA <br>+ ACE Data</td>
+    <td><img src="./assets/samples/portrait/human_1.jpg" width="200"></td>
+    <td><img src="./assets/samples/portrait/human_1_1.jpg" width="200"></td>
+    <td style="word-wrap:break-word;word-break:break-all;" width="250px";>"Maintain the facial features. A girl is wearing a neat police uniform and sporting a badge. She is smiling with a friendly and confident demeanor. The background is blurred, featuring a cartoon logo."</td>
+    <td align="center" style="word-wrap:break-word;word-break:break-all;" width="200px";><a href="https://www.modelscope.cn/models/iic/ACE_Plus/"><img src="https://img.shields.io/badge/ModelScope-Model-blue" alt="ModelScope link"> </a> <a href="https://huggingface.co/ali-vilab/ACE_Plus/tree/main/portrait/"><img src="https://img.shields.io/badge/%F0%9F%A4%97%20Hugging%20Face-Model-yellow" alt="HuggingFace link"> </a> </td>
+  </tr>
+</tbody>
+</table>
+
+
+Models' scepter_path: 
+
+- **ModelScope:** ms://iic/ACE_Plus@portrait/xxxx.safetensors
+
+- **HuggingFace:** hf://ali-vilab/ACE_Plus@portrait/xxxx.safetensors
+
+
+
+### ACE++ Subject LoRA
+
+Subject-driven image generation task to maintain the consistency of a specific subject in different scenes.
+
+<table><tbody>
+  <tr>
+    <td>Tuning Method</td>
+    <td>Input</td>
+    <td>Output</td>
+    <td>Instruction</td>
+    <td>Models</td>
+  </tr>
+  <tr>
+    <td>LoRA <br>+ ACE Data</td>
+    <td><img src="./assets/samples/subject/subject_1.jpg" width="200"></td>
+    <td><img src="./assets/samples/subject/subject_1_1.jpg" width="200"></td>
+    <td style="word-wrap:break-word;word-break:break-all;" width="250px";>"Display the logo in a minimalist style printed in white on a matte black ceramic coffee mug, alongside a steaming cup of coffee on a cozy cafe table."</td>
+    <td align="center" style="word-wrap:break-word;word-break:break-all;" width="200px";><a href="https://www.modelscope.cn/models/iic/ACE_Plus/"><img src="https://img.shields.io/badge/ModelScope-Model-blue" alt="ModelScope link"> </a> <a href="https://huggingface.co/ali-vilab/ACE_Plus/tree/main/subject/"><img src="https://img.shields.io/badge/%F0%9F%A4%97%20Hugging%20Face-Model-yellow" alt="HuggingFace link"> </a> </td>
+  </tr>
+</tbody>
+</table>
+
+
+Models' scepter_path: 
+
+- **ModelScope:** ms://iic/ACE_Plus@subject/xxxx.safetensors
+
+- **HuggingFace:** hf://ali-vilab/ACE_Plus@subject/xxxx.safetensors
+
+
+
+### ACE++ LocalEditing LoRA
+
+Redrawing the mask area of images while maintaining the original structural information of the edited area.
+
+<table><tbody>
+  <tr>
+    <td>Tuning Method</td>
+    <td>Input</td>
+    <td>Output</td>
+    <td>Instruction</td>
+    <td>Models</td>
+  </tr>
+  <tr>
+    <td>LoRA <br>+ ACE Data</td>
+    <td><img src="./assets/samples/local/local_1.webp" width="200"><br><img src="./assets/samples/local/local_1_m.webp" width="200"></td>
+    <td><img src="./assets/samples/local/local_1_1.jpg" width="200"></td>
+    <td style="word-wrap:break-word;word-break:break-all;" width="250px";>"By referencing the mask, restore a partial image from the doodle {image} that aligns with the textual explanation: "1 white old owl"."</td>
+    <td align="center" style="word-wrap:break-word;word-break:break-all;" width="200px";><a href="https://www.modelscope.cn/models/iic/ACE_Plus/"><img src="https://img.shields.io/badge/ModelScope-Model-blue" alt="ModelScope link"> </a> <a href="https://huggingface.co/ali-vilab/ACE_Plus/tree/main/local_editing/"><img src="https://img.shields.io/badge/%F0%9F%A4%97%20Hugging%20Face-Model-yellow" alt="HuggingFace link"> </a> </td>
+  </tr>
+</tbody>
+
+</table>
+
+
+Models' scepter_path: 
+
+- **ModelScope:** ms://iic/ACE_Plus@local_editing/xxxx.safetensors
+
+- **HuggingFace:** hf://ali-vilab/ACE_Plus@local_editing/xxxx.safetensors
+
+### ACE++ FFT model  
+Fully finetuning a composite model with ACE’s data to support various editing and reference generation tasks through an instructive approach.
+
+We introduced 64 additional channels in the channel dimension to differentiate between the repainting task and the editing task. In these channels, we place the latent representation of the pixel space from the edited image, while keeping other channels consistent with the repainting task. One issue with this approach is that it changes the input channel number of the FLUX-Fill-Dev model from 384 to 448. The specific configuration can be referenced in the [configuration file](config/ace_plus_fft.yaml).
+
+
+### Examples
+The ACE++ model supports a wide range of downstream tasks through simple adaptations. Here are some examples.
+
+<table><tbody>
+  <tr>
+    <th align="center" colspan="1">ACE++ Model</th>
+    <td>Input Reference Image</td>
+    <td>Input Edit Image</td>
+    <td>Input Edit Mask</td>
+    <td>Output</td>
+    <td>Instruction</td>
+    <td>Function</td>
+  </tr>
+  <tr>
+    <td>Portrait LoRA(recommended) / FFT model</td>
+    <td><img src="./assets/samples/portrait/human_1.jpg" width="200"></td>
+    <td></td>
+    <td></td>
+    <td><img src="./assets/samples/portrait/human_1_fft.webp" width="200"></td>
+    <td style="word-wrap:break-word;word-break:break-all;" width="250px";>"Maintain the facial features, A girl is wearing a neat police uniform and sporting a badge. She is smiling with a friendly and confident demeanor. The background is blurred, featuring a cartoon logo."</td>
+    <td style="word-wrap:break-word;word-break:break-all;" width="250px";>"Character ID Consistency Generation"</td>
+  </tr>
+  <tr>
+    <td>Subject LoRA(recommended) / FFT model</td>
+    <td><img src="./assets/samples/subject/subject_1.jpg" width="200"></td>
+    <td></td>
+    <td></td>
+    <td><img src="./assets/samples/subject/subject_1_fft.webp" width="200"></td>
+    <td style="word-wrap:break-word;word-break:break-all;" width="250px";>"Display the logo in a minimalist style printed in white on a matte black ceramic coffee mug, alongside a steaming cup of coffee on a cozy cafe table."</td>
+    <td style="word-wrap:break-word;word-break:break-all;" width="250px";>"Subject Consistency Generation"</td>
+  </tr>
+  <tr>
+    <td>Subject LoRA(recommended) / FFT model</td>
+    <td><img src="./assets/samples/application/photo_editing/1_ref.png" width="200"></td>
+    <td><img src="./assets/samples/application/photo_editing/1_2_edit.jpg" width="200"></td>
+    <td><img src="./assets/samples/application/photo_editing/1_2_m.webp" width="200"></td>
+    <td><img src="./assets/samples/application/photo_editing/1_2_fft.webp" width="200"></td>
+    <td style="word-wrap:break-word;word-break:break-all;" width="250px";>"The item is put on the table."</td>
+    <td style="word-wrap:break-word;word-break:break-all;" width="250px";>"Subject Consistency Editing"</td>
+  </tr>
+  <tr>
+    <td>Subject LoRA(recommended) / FFT model</td>
+    <td><img src="./assets/samples/application/logo_paste/1_ref.png" width="200"></td>
+    <td><img src="./assets/samples/application/logo_paste/1_1_edit.png" width="200"></td>
+    <td><img src="./assets/samples/application/logo_paste/1_1_m.png" width="200"></td>
+    <td><img src="./assets/samples/application/logo_paste/1_1_fft.webp" width="200"></td>
+    <td style="word-wrap:break-word;word-break:break-all;" width="250px";>"The logo is printed on the headphones."</td>
+    <td style="word-wrap:break-word;word-break:break-all;" width="250px";>"Subject Consistency Editing"</td>
+  </tr>
+  <tr>
+    <td>Subject LoRA(recommended) / FFT model</td>
+    <td><img src="./assets/samples/application/try_on/1_ref.png" width="200"></td>
+    <td><img src="./assets/samples/application/try_on/1_1_edit.png" width="200"></td>
+    <td><img src="./assets/samples/application/try_on/1_1_m.png" width="200"></td>
+    <td><img src="./assets/samples/application/try_on/1_1_fft.webp" width="200"></td>
+    <td style="word-wrap:break-word;word-break:break-all;" width="250px";>"The woman dresses this skirt."</td>
+    <td style="word-wrap:break-word;word-break:break-all;" width="250px";>"Try On"</td>
+  </tr>
+  <tr>
+    <td>Portrait LoRA(recommended) / FFT model</td>
+    <td><img src="./assets/samples/application/movie_poster/1_ref.png" width="200"></td>
+    <td><img src="./assets/samples/portrait/human_1.jpg" width="200"></td>
+    <td><img src="./assets/samples/application/movie_poster/1_2_m.webp" width="200"></td>
+    <td><img src="./assets/samples/application/movie_poster/1_1_fft.webp" width="200"></td>
+    <td style="word-wrap:break-word;word-break:break-all;" width="250px";>"{image}, the man faces the camera."</td>
+    <td style="word-wrap:break-word;word-break:break-all;" width="250px";>"Face swap"</td>
+  </tr>
+ <tr>
+    <td>FFT model</td>
+    <td></td>
+    <td><img src="./assets/samples/application/sr/sr_tiger.png" width="200"></td>
+    <td><img src="./assets/samples/application/sr/sr_tiger_m.webp" width="200"></td>
+    <td><img src="./assets/samples/application/sr/sr_tiger_fft.webp" width="200"></td>
+    <td style="word-wrap:break-word;word-break:break-all;" width="250px";>"{image} features a close-up of a young, furry tiger cub on a rock. The tiger, which appears to be quite young, has distinctive orange, black, and white striped fur, typical of tigers. The cub's eyes have a bright and curious expression, and its ears are perked up, indicating alertness. The cub seems to be in the act of climbing or resting on the rock. The background is a blurred grassland with trees, but the focus is on the cub, which is vividly colored while the rest of the image is in grayscale, drawing attention to the tiger's details. The photo captures a moment in the wild, depicting the charming and tenacious nature of this young tiger, as well as its typical interaction with the environment."</td>
+    <td style="word-wrap:break-word;word-break:break-all;" width="250px";>"Super-resolution"</td>
+  </tr>
+  <tr>
+    <td>FFT model</td>
+    <td></td>
+    <td><img src="./assets/samples/application/photo_editing/1_ref.png" width="200"></td>
+    <td><img src="./assets/samples/application/photo_editing/1_1_orm.webp" width="200"></td>
+    <td><img src="./assets/samples/application/regional_editing/1_1_fft.webp" width="200"></td>
+    <td style="word-wrap:break-word;word-break:break-all;" width="250px";>"a blue hand"</td>
+    <td style="word-wrap:break-word;word-break:break-all;" width="250px";>"Regional Editing"</td>
+  </tr>
+  <tr>
+    <td>FFT model</td>
+    <td></td>
+    <td><img src="./assets/samples/application/photo_editing/1_ref.png" width="200"></td>
+    <td><img src="./assets/samples/application/photo_editing/1_1_rm.webp" width="200"></td>
+    <td><img src="./assets/samples/application/regional_editing/1_2_fft.webp" width="200"></td>
+    <td style="word-wrap:break-word;word-break:break-all;" width="250px";>"Mechanical  hands like a robot"</td>
+    <td style="word-wrap:break-word;word-break:break-all;" width="250px";>"Regional Editing"</td>
+  </tr>
+  <tr>
+    <td>Local Editing LoRA/FFT model</td>
+    <td></td>
+    <td><img src="./assets/samples/control/1_1_recolor.webp" width="200"></td>
+    <td><img src="./assets/samples/control/1_1_m.webp" width="200"></td>
+    <td><img src="./assets/samples/control/1_1_fft_recolor.webp" width="200"></td>
+    <td style="word-wrap:break-word;word-break:break-all;" width="250px";>"{image} Beautiful female portrait, Robot with smooth White transparent carbon shell, rococo detailing, Natural lighting, Highly detailed, Cinematic, 4K."</td>
+    <td style="word-wrap:break-word;word-break:break-all;" width="250px";>"Recolorizing"</td>
+  </tr>
+
+  <tr>
+    <td>Local Editing LoRA/FFT model</td>
+    <td></td>
+    <td><img src="./assets/samples/control/1_1_depth.webp" width="200"></td>
+    <td><img src="./assets/samples/control/1_1_m.webp" width="200"></td>
+    <td><img src="./assets/samples/control/1_1_fft_depth.webp" width="200"></td>
+    <td style="word-wrap:break-word;word-break:break-all;" width="250px";>"{image} Beautiful female portrait, Robot with smooth White transparent carbon shell, rococo detailing, Natural lighting, Highly detailed, Cinematic, 4K."</td>
+    <td style="word-wrap:break-word;word-break:break-all;" width="250px";>"Depth Guided Generation"</td>
+  </tr>
+  <tr>
+    <td>Local Editing LoRA/FFT model</td>
+    <td></td>
+    <td><img src="./assets/samples/control/1_1_contourc.webp" width="200"></td>
+    <td><img src="./assets/samples/control/1_1_m.webp" width="200"></td>
+    <td><img src="./assets/samples/control/1_1_fft_contour.webp" width="200"></td>
+    <td style="word-wrap:break-word;word-break:break-all;" width="250px";>"{image} Beautiful female portrait, Robot with smooth White transparent carbon shell, rococo detailing, Natural lighting, Highly detailed, Cinematic, 4K."</td>
+    <td style="word-wrap:break-word;word-break:break-all;" width="250px";>"Contour Guided Generation"</td>
+  </tr>
+</tbody>
+</table>
+
+##  Comfyui Workflows in community
+We are deeply grateful to the community developers for building many fascinating applications based on the ACE++ series of models. 
+During this process, we have received valuable feedback, particularly regarding artifacts in generated images and the stability of the results. 
+In response to these issues, many developers have proposed creative solutions, which have greatly inspired us, and we pay tribute to them. 
+At the same time, we will take these concerns into account in our further optimization efforts, carefully evaluating and testing before releasing new models.
+
+In the table below, we have briefly listed some workflows for everyone to use.
+
+<table><tbody>
+  <tr>
+    <td>Workflow</td>
+    <td>Author</td>
+    <td>Example</td>
+  </tr>
+  <tr>
+    <td><a href="https://www.runninghub.cn/post/1890052281759571970"> 【飞翔鲸】王炸！FLUX+ACE++换头 </a> </td>
+    <td><a href="https://www.runninghub.cn/user-center/1851827947798740994"> 飞翔鲸 </a></td>
+    <td><img src="./assets/comfyui/feixiangjing_face.png" height="200"></td>
+  </tr>
+  <tr>
+    <td><a href="https://www.runninghub.cn/post/1890014204382916609"> 最新ACE++ Redux 万物迁移 AI模特换装 UP 楚门的AI世界 </a> </td>
+    <td><a href="https://www.runninghub.cn/user-center/1865415077888405506"> 楚门的AI世界 </a></td>
+    <td><img src="./assets/comfyui/chumen_tryon.jpg" height="200"></td>
+  </tr>
+  <tr>
+    <td><a href="https://openart.ai/workflows/t8star/ace-plusfillredux/bgQDNz8SeySMDqn13ZBv"> Ace Plus+Fill+Redux稳定无抽卡换装工作流</a> </td>
+    <td><a href="https://openart.ai/workflows/profile/t8star?sort=latest"> T8star-Aix </a></td>
+    <td><img src="./assets/comfyui/t8_star_tryon.jpg" height="200"></td>
+  </tr>
+  <tr>
+    <td><a href="https://openart.ai/workflows/t8star/ace-plusfillredux/ifIvaWXW9QkLtNV405j7"> Ace Plus+Fill+Redux稳定少抽卡标志工作流</a> </td>
+    <td><a href="https://openart.ai/workflows/profile/t8star?sort=latest"> T8star-Aix </a></td>
+    <td><img src="./assets/comfyui/t8_star_logo.jpg" height="200"></td>
+  </tr>
+  <tr>
+    <td><a href="https://openart.ai/workflows/t8star/ace-plusfillredux/WdwUwGXPLHhnSOlSEfTg"> Ace Plus+Fill+Redux稳定无抽卡换脸工作流</a> </td>
+    <td><a href="https://openart.ai/workflows/profile/t8star?sort=latest"> T8star-Aix </a></td>
+    <td><img src="./assets/comfyui/t8_star_face.jpg" height="200"></td>
+  </tr>
+  <tr>
+    <td><a href="https://openart.ai/workflows/cat_untimely_42/ace-face-swap-in-different-styles/VocvdfQrvDhmKNLEBwJY"> ace++ face swap in different styles </a> </td>
+    <td><a href="https://openart.ai/workflows/profile/cat_untimely_42?sort=latest"> jax </a></td>
+    <td><img src="./assets/comfyui/jax_face_swap.jpg" height="200"></td>
+  </tr>
+  <tr>
+    <td><a href="https://openart.ai/workflows/leeguandong/fllux-ace-subject-without-reference-image/HjYf6Eae2PRGACJWXdrE"> fllux ace++ subject without reference image </a> </td>
+    <td><a href="https://openart.ai/workflows/profile/leeguandong?sort=latest"> leeguandong </a></td>
+    <td><img src="./assets/comfyui/leeguandong_subject.jpg" height="200"></td>
+  </tr>
+  <tr>
+    <td><a href="https://openart.ai/workflows/whale_waterlogged_60/scepter-ace-more-convenient-replacement-of-everything/gjAsh5rGjfC6OEB2AUZv"> Scepter-ACE++ More convenient replacement of everything</a> </td>
+    <td><a href="https://openart.ai/workflows/profile/whale_waterlogged_60?sort=latest"> HaoBeen </a></td>
+    <td><img src="./assets/comfyui/haobeen_ace_plus.jpg" height="200"></td>
+  </tr>
+</tbody>
+</table>
+
+Additionally, many bloggers have published tutorials on how to use it, which are listed in the table below.
+
+<table><tbody>
+  <tr>
+    <td>Tutorial</td>
+  </tr>
+  <tr>
+    <td><a href="https://www.youtube.com/watch?v=5OwcxugdWxI"> Best Faceswapper I've Seen. ACE++ in ComfyUI. </a> </td>
+  </tr>
+  <tr>
+    <td><a href="https://www.youtube.com/watch?v=2fgT35H_tuE&pp=ygUIYWNlIHBsdXM%3D"> ACE ++ In ComfyUI All-round Creator & Editor - More Than Just A Faceswap AI </a> </td>
+  </tr>
+  <tr>
+    <td><a href="https://www.youtube.com/watch?v=XU376PzgnXc"> Ai绘画进阶140-咦？大家用的都不对？！Ace Plus工作流正确搭建方式及逻辑，参数详解，Flux Fill，Redux联用-T8 Comfyui教程</a> </td>
+  </tr>
+  <tr>
+    <td><a href="https://www.youtube.com/watch?v=1cbOkN0mTw0"> ace++：告别 Lora 训练，无需pulid，轻松打造专属角色！ | No Lora Training, Easily Create Exclusive Characters!</a> </td>
+  </tr>
+  <tr>
+    <td><a href="https://www.youtube.com/watch?v=0wMoWSTm5Hc"> Ace++ and Flux Fill: Advanced Face Swapping Made Easy in ComfyUI | No Lora Training, Easily Create Exclusive Characters!</a> </td>
+  </tr>
+  <tr>
+    <td><a href="https://www.youtube.com/watch?v=7GrkIFuRQAc"> ComfyUI - ACE Plus Subject Portrait Lora </a> </td>
+  </tr>
+  <tr>
+    <td><a href="https://www.bilibili.com/video/BV1HiKpeuE8o/?spm_id_from=333.337.search-card.all.click&vd_source=927630f34c77eee560afd69cfdba3f47"> 🤗AI一致性技术新突破！ACE++技术一致性comfyui工作流🍋‍ </a> </td>
+  </tr>
+  <tr>
+    <td><a href="https://www.bilibili.com/video/BV1obN9enEvp/?spm_id_from=333.337.search-card.all.click&vd_source=927630f34c77eee560afd69cfdba3f47"> ComfyUI 第55集 人像换脸 FLUX的FILL模型+ACE LORA </a> </td>
+  </tr>
+  <tr>
+    <td><a href="https://www.bilibili.com/video/BV1pPN3eBEtr/?spm_id_from=333.337.search-card.all.click&vd_source=927630f34c77eee560afd69cfdba3f47"> 换装变脸贴logo，无所不能的Ace_Plus lora </a> </td>
+  </tr>
+</tbody>
+</table>
+
+## ComfyUI Workflow Examples
+
+Copy the workflow/ComfyUI-ACE_Plus folder into ComfyUI’s custom_nodes directory. Launch ComfyUI, and we have provided some example workflows in workflow_example with the following explanations. It is recommended to use the LoRA model workflow, as it offers more stable results compared to the FFT model.
+
+<table><tbody>
+  <tr>
+    <td>Workflow</td>
+    <td>Description</td>
+    <td>Other dependency models</td>
+    <td>Setting</td>
+  </tr>
+  <tr>
+    <td>ACE_Plus_LoRA_workflow_reference_generation.json</td>
+    <td>Reference image generation capability for portrait or subject.</td>
+    <td>Potrait or subject LoRA Model + FLUX.1-Fill-dev</td>
+    <td>Task_type: repainting (you don't need to install dependencies like scepter)</td>
+  </tr>
+  <tr>
+    <td>ACE_Plus_LoRA_workflow_redux_reference_generation.json</td>
+    <td>Reference image generation capability for portrait or subject used in conjunction with Redux.</td>
+    <td>Potrait or subject LoRA Model + FLUX.1-Fill-dev + FLUX.1-Redux</td>
+    <td>Task_type: repainting (you don't need to install dependencies like scepter)</td>
+  </tr>
+  <tr>
+    <td>ACE_Plus_LoRA_workflow_reference_editing.json</td>
+    <td>Reference image editing capability such as logo paste, face swap.</td>
+    <td>Potrait or subject LoRA Model + FLUX.1-Fill-dev</td>
+    <td>Task_type: repainting (you don't need to install dependencies like scepter)</td>
+  </tr>
+  <tr>
+    <td>ACE_Plus_LoRA_workflow_redux_reference_editing.json</td>
+    <td>Reference image editing capability such as logo paste, face swap used in conjunction with Redux.</td>
+    <td>Potrait or subject LoRA Model + FLUX.1-Fill-dev + FLUX.1-Redux</td>
+    <td>Task_type: repainting (you don't need to install dependencies like scepter)</td>
+  </tr>
+  <tr>
+    <td>ACE_Plus_LoRA_workflow_localcontrol_generation.json</td>
+    <td>Controllable image-to-image translation capability. To preprocess depth and contour information from images, 
+    we use externally-provided models that are typically downloaded from the ModelScope Hub. Because download success 
+    can vary depending on the user's environment, we offer alternatives: users can either leverage existing community 
+    nodes (depth extration node or contour extraction node) for this task (then choosing the 'no_preprocess' option), 
+    or users can pre-download the required models 
+    <a href="https://www.modelscope.cn/models/iic/scepter_annotator/file/view/master?fileName=annotator%252Fckpts%252Finformative_drawing_contour_style.pth&status=2">contour</a> and
+    <a href="https://www.modelscope.cn/models/iic/scepter_annotator/file/view/master?fileName=annotator%252Fckpts%252Fdpt_hybrid-midas-501f0c75.pt&status=2">depth</a>    
+    and adjust
+    the configuration file 'workflow/ComfyUI-ACE_Plus/config/ace_plus_fft_processor.yaml' to
+    specify the models' local paths.</td>
+    <td>Local editing LoRA Model + FLUX.1-Fill-dev + Preprocessing model (depth or contour) </td>
+    <td>Task_type: contour_repainting/depth_repainting/recolorizing (you need to install dependencies like scepter)</td>
+  </tr>
+  <tr>
+    <td>ACE_Plus_FFT_workflow_referenceediting_generation.json</td>
+    <td>Reference image editing capability</td>
+    <td>FFT model</td>
+    <td>Task_type: repainting (you don't need to install dependencies like scepter)</td>
+  </tr>
+  <tr>
+    <td>ACE_Plus_FFT_workflow_no_preprocess.json</td>
+    <td>Use the preprocessed images, such as depth and contour, as input, or the super-resolution.</td>
+    <td>FFT model</td>
+    <td>Task_type: no_preprocess (you don't need to install dependencies like scepter)</td>
+  </tr>
+  <tr>
+    <td>ACE_Plus_FFT_workflow_controlpreprocess.json</td>
+    <td>Controllable image-to-image translation capability. To preprocess depth and contour information from images, 
+    we use externally-provided models that are typically downloaded from the ModelScope Hub. Because download success 
+    can vary depending on the user's environment, we offer alternatives: users can either leverage existing community 
+    nodes (depth extration node or contour extraction node) for this task (then choosing the 'no_preprocess' option), 
+    or users can pre-download the required models 
+    <a href="https://www.modelscope.cn/models/iic/scepter_annotator/file/view/master?fileName=annotator%252Fckpts%252Finformative_drawing_contour_style.pth&status=2">contour</a> and
+    <a href="https://www.modelscope.cn/models/iic/scepter_annotator/file/view/master?fileName=annotator%252Fckpts%252Fdpt_hybrid-midas-501f0c75.pt&status=2">depth</a>    
+    and adjust
+    the configuration file 'workflow/ComfyUI-ACE_Plus/config/ace_plus_fft_processor.yaml' to
+    specify the models' local paths.</td>
+    <td>FFT model</td>
+    <td>Task_type: contour_repainting/depth_repainting/recolorizing (you need to install dependencies like scepter)</td>
+  </tr>
+  <tr>
+    <td>ACE_Plus_FFT_workflow_reference_generation.json</td>
+    <td>Reference image generation capability for portrait or subject.</td>
+    <td>FFT model</td>
+    <td>Task_type: repainting (you don't need to install dependencies like scepter)</td>
+  </tr>
+  <tr>
+    <td>ACE_Plus_FFT_workflow_referenceediting_generation.json</td>
+    <td>Reference image editing capability</td>
+    <td>FFT model</td>
+    <td>Task_type: repainting (you don't need to install dependencies like scepter)</td>
+  </tr>
+ <tbody>
+<table>
+
+As shown in the figure below, max_seq_length controls the length of the token sequence during inference, thereby controlling the model's inference memory consumption. 
+The range of this value is from 2048 to 8192, and it correspondingly affects the clarity of the generated image. The smaller the value, the lower the image clarity.
+
+<img src="./assets/comfyui/snapshot.jpg" width="200">
+
+
+## ⚙️️ Installation
+Download the code using the following command:
+```bash
+git clone https://github.com/ali-vilab/ACE_plus.git
+```
+
+Install the necessary packages with `pip`: 
+```bash
+cd ACE_plus
+pip install -r repo_requirements.txt
+```
+ACE++ depends on FLUX.1-Fill-dev as its base model, which you can download from [![HuggingFace link](https://img.shields.io/badge/%F0%9F%A4%97%20Hugging%20Face-Model-yellow)](https://huggingface.co/black-forest-labs/FLUX.1-Fill-dev). 
+In order to run the inference code or Gradio demo normally, we have defined the relevant environment variables to specify the location of the model. 
+For model preparation, we provide three methods for downloading the model. The summary of relevant settings is as follows.
+
+|   Model Downloading Method    | Clone to Local Path                                                                                                                                                                                                                                         | Automatic Downloading during Runtime<br>(Setting the Environment Variables using scepter_path in [ACE Models](#-ace-models))                                                                                                       |
+|:-----------------------------:|:------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|:-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| Environment Variables Setting | <pre><code>export FLUX_FILL_PATH="path/to/FLUX.1-Fill-dev"<br>export PORTRAIT_MODEL_PATH="path/to/ACE++ PORTRAIT PATH"<br>export SUBJECT_MODEL_PATH="path/to/ACE++ SUBJECT PATH"<br>export LOCAL_MODEL_PATH="path/to/ACE++ LOCAL EDITING PATH"</code></pre> | <pre><code>export FLUX_FILL_PATH="hf://black-forest-labs/FLUX.1-Fill-dev"<br>export PORTRAIT_MODEL_PATH="${scepter_path}"<br>export SUBJECT_MODEL_PATH="${scepter_path}"<br>export LOCAL_MODEL_PATH="${scepter_path}"</code></pre> |
+
+## 🚀 Inference
+Under the condition that the environment variables defined in [Installation](#-installation), users can run examples and test your own samples by executing infer.py. 
+The relevant commands for lora models are as follows:
+```bash
+export FLUX_FILL_PATH="hf://black-forest-labs/FLUX.1-Fill-dev"
+export PORTRAIT_MODEL_PATH="ms://iic/ACE_Plus@portrait/comfyui_portrait_lora64.safetensors"                                                                                                                                      
+export SUBJECT_MODEL_PATH="ms://iic/ACE_Plus@subject/comfyui_subject_lora16.safetensors"                                                                                                                                         
+export LOCAL_MODEL_PATH="ms://iic/ACE_Plus@local_editing/comfyui_local_lora16.safetensors" 
+# Use the model from huggingface
+# export PORTRAIT_MODEL_PATH="hf://ali-vilab/ACE_Plus@portrait/comfyui_portrait_lora64.safetensors"        
+# export SUBJECT_MODEL_PATH="hf://ali-vilab/ACE_Plus@subject/comfyui_subject_lora16.safetensors"        
+# export LOCAL_MODEL_PATH="hf://ali-vilab/ACE_Plus@local_editing/comfyui_local_lora16.safetensors" 
+python infer_lora.py
+```
+The relevant commands for fft models are as follows:
+```bash
+export FLUX_FILL_PATH="hf://black-forest-labs/FLUX.1-Fill-dev"
+export ACE_PLUS_FFT_MODEL="ms://iic/ACE_Plus@ace_plus_fft.safetensors.safetensors"                                                                                                                                      
+python infer_fft.py
+```
+
+## 🚀 Train
+We provide training code that allows users to train on their own data. Reference the data in 'data/train.csv' and 'data/eval.csv' to construct the training data and test data, respectively. We use '#;#' to separate fields. 
+The required fields include the following six, with their explanations as follows.
+```angular2html
+"edit_image": represents the input image for the editing task. If it is not an editing task but a reference generation, this field can be left empty.
+"edit_mask": represents the input image mask for the editing task, used to specify the editing area. If it is not an editing task but rather for reference generation, this field can be left empty.
+"ref_image": represents the input image for the reference image generation task; if it is a pure editing task, this field can be left empty.
+"target_image": represents the generated target image and cannot be empty.
+"prompt": represents the prompt for the generation task.
+"data_type": represents the type of data, which can be 'portrait', 'subject', or 'local'. This field is not used in training phase.
+```
+
+All parameters related to training are stored in 'train_config/ace_plus_lora.yaml'. With the following default configuration, the memory usage for LoRA training is between 38GB and 40GB. 
+
+| Hyperparameter | Value     | Description |
+| --- | --- | --- |
+| ATTN_BACKEND |   flash_attn / pytorch  |Set 'flash_attn' to use flash_attn2(Make sure you have installed flash-attn2 correctly). If the version of PyTorch is greater than 2.4.0, use 'pytorch' to utilize PyTorch's implementation.|
+| USE_GRAD_CHECKPOINT |   True / False  |Using gradient checkpointing can also significantly reduce GPU memory usage, but it may slow down the training speed. |
+| MAX_SEQ_LEN |   2048  | The MAX_SEQ_LEN refers to the sequence size limit for a single input image (calculated as H/16 * W/16). A larger value indicates a longer computation sequence and a higher training resolution. The default value I provided is 2048.|
+
+To run the training code, execute the following command.
+
+
+```bash
+export FLUX_FILL_PATH="{path to FLUX.1-Fill-dev}"
+python run_train.py  --cfg train_config/ace_plus_lora.yaml
+# Training from fft model
+export FLUX_FILL_PATH="{path to FLUX.1-Fill-dev}"
+export ACE_PLUS_FFT_MODEL="path to ace_plus_fft.safetensors.safetensors"       
+python run_train.py  --cfg train_config/ace_plus_fft.yaml 
+```
+
+The models trained by ACE++ can be found in ./examples/exp_example/xxxx/checkpoints/xxxx/0_SwiftLoRA/comfyui_model.safetensors.
+
+## 💻 Demo
+We have built a GUI demo based on Gradio to help users better utilize the ACE++ model. Just execute the following command.
+```bash
+export FLUX_FILL_PATH="hf://black-forest-labs/FLUX.1-Fill-dev"
+export PORTRAIT_MODEL_PATH="ms://iic/ACE_Plus@portrait/comfyui_portrait_lora64.safetensors"                                                                                                                                      
+export SUBJECT_MODEL_PATH="ms://iic/ACE_Plus@subject/comfyui_subject_lora16.safetensors"                                                                                                                                         
+export LOCAL_MODEL_PATH="ms://iic/ACE_Plus@local_editing/comfyui_local_lora16.safetensors" 
+# Use the model from huggingface
+# export PORTRAIT_MODEL_PATH="hf://ali-vilab/ACE_Plus@portrait/comfyui_portrait_lora64.safetensors"        
+# export SUBJECT_MODEL_PATH="hf://ali-vilab/ACE_Plus@subject/comfyui_subject_lora16.safetensors"        
+# export LOCAL_MODEL_PATH="hf://ali-vilab/ACE_Plus@local_editing/comfyui_local_lora16.safetensors" 
+python demo_lora.py
+# Use the fft model
+export FLUX_FILL_PATH="hf://black-forest-labs/FLUX.1-Fill-dev"
+export ACE_PLUS_FFT_MODEL="ms://iic/ACE_Plus@ace_plus_fft.safetensors.safetensors"       
+python demo_fft.py
+```
+
+
+## 📚 Limitations
+* For certain tasks, such as deleting and adding objects, there are flaws in instruction following. For adding and replacing objects, we recommend trying the repainting method of the local editing model to achieve this.
+* The generated results may contain artifacts, especially when it comes to the generation of hands, which still exhibit distortions.
+
+## 📝 Citation
+ACE++ is a post-training model based on the FLUX.1-dev series from black-forest-labs. Please adhere to its open-source license. The test materials used in ACE++ come from the internet and are intended for academic research and communication purposes. If the original creators feel uncomfortable, please contact us to have them removed. 
+
+If you use this model in your research, please cite the works of FLUX.1-dev and the following papers:
+```bibtex
+@article{mao2025ace++,
+  title={ACE++: Instruction-Based Image Creation and Editing via Context-Aware Content Filling},
+  author={Mao, Chaojie and Zhang, Jingfeng and Pan, Yulin and Jiang, Zeyinzi and Han, Zhen and Liu, Yu and Zhou, Jingren},
+  journal={arXiv preprint arXiv:2501.02487},
+  year={2025}
+}
+```
+```bibtex
+@article{han2024ace,
+  title={ACE: All-round Creator and Editor Following Instructions via Diffusion Transformer},
+  author={Han, Zhen and Jiang, Zeyinzi and Pan, Yulin and Zhang, Jingfeng and Mao, Chaojie and Xie, Chenwei and Liu, Yu and Zhou, Jingren},
+  journal={arXiv preprint arXiv:2410.00086},
+  year={2024}
+}
+```

ACE_plus/__init__.py

@@ -0,0 +1 @@
+from . import modules

ACE_plus/demo_fft.py

@@ -0,0 +1,545 @@
+# -*- coding: utf-8 -*-
+# Copyright (c) Alibaba, Inc. and its affiliates.
+import argparse
+import csv
+import glob
+import os
+import sys
+import threading
+import time
+
+import gradio as gr
+import numpy as np
+import torch, importlib
+from PIL import Image
+from scepter.modules.transform.io import pillow_convert
+from scepter.modules.utils.config import Config
+from scepter.modules.utils.distribute import we
+from scepter.modules.utils.file_system import FS
+
+if os.path.exists('__init__.py'):
+    package_name = 'scepter_ext'
+    spec = importlib.util.spec_from_file_location(package_name, '__init__.py')
+    package = importlib.util.module_from_spec(spec)
+    sys.modules[package_name] = package
+    spec.loader.exec_module(package)
+
+from examples.examples import fft_examples
+from inference.registry import INFERENCES
+from inference.utils import edit_preprocess
+
+
+fs_list = [
+    Config(cfg_dict={"NAME": "HuggingfaceFs", "TEMP_DIR": os.environ["TEMP_DIR"]}, load=False),
+    Config(cfg_dict={"NAME": "ModelscopeFs", "TEMP_DIR": os.environ["TEMP_DIR"]}, load=False),
+    Config(cfg_dict={"NAME": "HttpFs", "TEMP_DIR": os.environ["TEMP_DIR"]}, load=False),
+    Config(cfg_dict={"NAME": "LocalFs", "TEMP_DIR": os.environ["TEMP_DIR"]}, load=False),
+    Config(cfg_dict={"NAME": "LocalFs", "TEMP_DIR": os.environ["TEMP_DIR"]}, load=False)
+]
+
+for one_fs in fs_list:
+    FS.init_fs_client(one_fs)
+
+
+csv.field_size_limit(sys.maxsize)
+refresh_sty = '\U0001f504'  # 🔄
+clear_sty = '\U0001f5d1'  # 🗑️
+upload_sty = '\U0001f5bc'  # 🖼️
+sync_sty = '\U0001f4be'  # 💾
+chat_sty = '\U0001F4AC'  # 💬
+video_sty = '\U0001f3a5'  # 🎥
+
+lock = threading.Lock()
+class DemoUI(object):
+    def __init__(self,
+                 infer_dir = "./config/ace_plus_fft.yaml"
+                 ):
+        self.model_yamls = [infer_dir]
+        self.model_choices = dict()
+        self.default_model_name = ''
+        self.edit_type_dict = {}
+        self.edit_type_list = []
+        self.default_type_list = []
+        for i in self.model_yamls:
+            model_cfg = Config(load=True, cfg_file=i)
+            model_name = model_cfg.VERSION
+            if model_cfg.IS_DEFAULT: self.default_model_name = model_name
+            self.model_choices[model_name] = model_cfg
+            for preprocessor in model_cfg.get("PREPROCESSOR", []):
+                if preprocessor["TYPE"] in self.edit_type_dict:
+                    continue
+                self.edit_type_dict[preprocessor["TYPE"]] = preprocessor
+                self.default_type_list.append(preprocessor["TYPE"])
+        print('Models: ', self.model_choices.keys())
+        assert len(self.model_choices) > 0
+        if self.default_model_name == "": self.default_model_name = list(self.model_choices.keys())[0]
+        self.model_name = self.default_model_name
+        pipe_cfg = self.model_choices[self.default_model_name]
+        self.pipe = INFERENCES.build(pipe_cfg)
+        # reformat examples
+        self.all_examples = [
+            [
+             one_example["edit_type"], one_example["instruction"],
+             one_example["input_reference_image"], one_example["input_image"],
+             one_example["input_mask"], one_example["output_h"],
+             one_example["output_w"], one_example["seed"]
+             ]
+            for one_example in fft_examples
+        ]
+
+    def construct_edit_image(self, edit_image, edit_mask):
+        if edit_image is not None and edit_mask is not None:
+            edit_image_rgb = pillow_convert(edit_image, "RGB")
+            edit_image_rgba = pillow_convert(edit_image, "RGBA")
+            edit_mask = pillow_convert(edit_mask, "L")
+
+            arr1 = np.array(edit_image_rgb)
+            arr2 = np.array(edit_mask)[:, :, np.newaxis]
+            result_array = np.concatenate((arr1, arr2), axis=2)
+            layer = Image.fromarray(result_array)
+
+            ret_data = {
+                "background": edit_image_rgba,
+                "composite": edit_image_rgba,
+                "layers": [layer]
+            }
+            return ret_data
+        else:
+            return None
+
+    def create_ui(self):
+        with gr.Row(equal_height=True, visible=True):
+            with gr.Column(scale=2):
+                self.gallery_image = gr.Image(
+                    height=600,
+                    interactive=False,
+                    type='pil',
+                    elem_id='Reference_image'
+                )
+            with gr.Column(scale=1, visible=True) as self.edit_preprocess_panel:
+                with gr.Row():
+                    with gr.Accordion(label='Related Input Image', open=False):
+                        self.edit_preprocess_preview = gr.Image(
+                            height=600,
+                            interactive=False,
+                            type='pil',
+                            elem_id='preprocess_image',
+                            label='edit image'
+                        )
+
+                        self.edit_preprocess_mask_preview = gr.Image(
+                            height=600,
+                            interactive=False,
+                            type='pil',
+                            elem_id='preprocess_image_mask',
+                            label='edit mask'
+                        )
+
+                        self.change_preprocess_preview = gr.Image(
+                            height=600,
+                            interactive=False,
+                            type='pil',
+                            elem_id='preprocess_change_image',
+                            label='change image'
+                        )
+                with gr.Row():
+                    instruction = """
+                               **Instruction**:
+                               Users can perform reference generation or editing tasks by uploading reference images 
+                               and editing images. When uploading the editing image, various editing types are available
+                                for selection. Users can choose different dimensions of information preservation, 
+                                such as edge information, color information, and more. Pre-processing information 
+                                can be viewed in the 'related input image' tab.
+                            """
+                    self.instruction = gr.Markdown(value=instruction)
+                with gr.Row():
+                    self.icon = gr.Image(
+                        value=None,
+                        interactive=False,
+                        height=150,
+                        type='pil',
+                        elem_id='icon',
+                        label='icon'
+                    )
+        with gr.Row():
+            self.model_name_dd = gr.Dropdown(
+                choices=self.model_choices,
+                value=self.default_model_name,
+                label='Model Version')
+            self.edit_type = gr.Dropdown(choices=self.default_type_list,
+                                         interactive=True,
+                                         value=self.default_type_list[0],
+                                         label='Edit Type')
+        with gr.Row():
+            self.step = gr.Slider(minimum=1,
+                                  maximum=1000,
+                                  value=self.pipe.input.get("sample_steps", 20),
+                                  visible=self.pipe.input.get("sample_steps", None) is not None,
+                                  label='Sample Step')
+            self.cfg_scale = gr.Slider(
+                minimum=1.0,
+                maximum=100.0,
+                value=self.pipe.input.get("guide_scale", 4.5),
+                visible=self.pipe.input.get("guide_scale", None) is not None,
+                label='Guidance Scale')
+            self.seed = gr.Slider(minimum=-1,
+                                  maximum=1000000000000,
+                                  value=-1,
+                                  label='Seed')
+            self.output_height = gr.Slider(
+                minimum=256,
+                maximum=1440,
+                value=self.pipe.input.get("image_size", [1024, 1024])[0],
+                visible=self.pipe.input.get("image_size", None) is not None,
+                label='Output Height')
+            self.output_width = gr.Slider(
+                minimum=256,
+                maximum=1440,
+                value=self.pipe.input.get("image_size", [1024, 1024])[1],
+                visible=self.pipe.input.get("image_size", None) is not None,
+                label='Output Width')
+
+            self.repainting_scale = gr.Slider(
+                minimum=0.0,
+                maximum=1.0,
+                value=self.pipe.input.get("repainting_scale", 1.0),
+                visible=True,
+                label='Repainting Scale')
+            self.use_change = gr.Checkbox(
+                value=self.pipe.input.get("use_change", True),
+                visible=True,
+                label='Use Change')
+            self.keep_pixel = gr.Checkbox(
+                value=self.pipe.input.get("keep_pixel", True),
+                visible=True,
+                label='Keep Pixels')
+            self.keep_pixels_rate = gr.Slider(
+                minimum=0.5,
+                maximum=1.0,
+                value=0.8,
+                visible=True,
+                label='keep_pixel rate')
+        with gr.Row():
+            self.generation_info_preview = gr.Markdown(
+                label='System Log.',
+                show_label=True)
+        with gr.Row(variant='panel',
+                    equal_height=True,
+                    show_progress=False):
+            with gr.Column(scale=10, min_width=500):
+                self.text = gr.Textbox(
+                    placeholder='Input "@" find history of image',
+                    label='Instruction',
+                    container=False,
+                    lines = 1)
+            with gr.Column(scale=2, min_width=100):
+                with gr.Row():
+                    with gr.Column(scale=1, min_width=100):
+                        self.chat_btn = gr.Button(value='Generate', variant = "primary")
+
+        with gr.Accordion(label='Advance', open=True):
+            with gr.Row(visible=True):
+                with gr.Column():
+                    self.reference_image = gr.Image(
+                        height=1000,
+                        interactive=True,
+                        image_mode='RGB',
+                        type='pil',
+                        label='Reference Image',
+                        elem_id='reference_image'
+                    )
+                with gr.Column():
+                    self.edit_image = gr.ImageMask(
+                        height=1000,
+                        interactive=True,
+                        value=None,
+                        sources=['upload'],
+                        type='pil',
+                        layers=False,
+                        label='Edit Image',
+                        elem_id='image_editor',
+                        show_fullscreen_button=True,
+                        format="png"
+                    )
+
+
+        with gr.Row():
+            self.eg = gr.Column(visible=True)
+
+
+
+    def set_callbacks(self, *args, **kwargs):
+        ########################################
+        def change_model(model_name):
+            if model_name not in self.model_choices:
+                gr.Info('The provided model name is not a valid choice!')
+                return model_name, gr.update(), gr.update()
+
+            if model_name != self.model_name:
+                lock.acquire()
+                del self.pipe
+                torch.cuda.empty_cache()
+                torch.cuda.ipc_collect()
+                pipe_cfg = self.model_choices[model_name]
+                self.pipe = INFERENCES.build(pipe_cfg)
+                self.model_name = model_name
+                lock.release()
+
+            return (model_name, gr.update(),
+                    gr.Slider(
+                              value=self.pipe.input.get("sample_steps", 20),
+                              visible=self.pipe.input.get("sample_steps", None) is not None),
+                    gr.Slider(
+                        value=self.pipe.input.get("guide_scale", 4.5),
+                        visible=self.pipe.input.get("guide_scale", None) is not None),
+                    gr.Slider(
+                        value=self.pipe.input.get("image_size", [1024, 1024])[0],
+                        visible=self.pipe.input.get("image_size", None) is not None),
+                    gr.Slider(
+                        value=self.pipe.input.get("image_size", [1024, 1024])[1],
+                        visible=self.pipe.input.get("image_size", None) is not None),
+                    gr.Slider(value=self.pipe.input.get("repainting_scale", 1.0))
+                    )
+
+        self.model_name_dd.change(
+            change_model,
+            inputs=[self.model_name_dd],
+            outputs=[
+                self.model_name_dd, self.text,
+                self.step,
+                self.cfg_scale,
+                self.output_height,
+                self.output_width,
+                self.repainting_scale])
+
+        def change_edit_type(edit_type):
+            edit_info = self.edit_type_dict[edit_type]
+            edit_info = edit_info or {}
+            repainting_scale = edit_info.get("REPAINTING_SCALE", 1.0)
+            return  gr.Slider(value=repainting_scale)
+
+        self.edit_type.change(change_edit_type, inputs=[self.edit_type], outputs=[self.repainting_scale])
+
+        def resize_image(image, h):
+            ow, oh = image.size
+            w = int(h * ow / oh)
+            image = image.resize((w, h), Image.LANCZOS)
+            return image
+
+        def preprocess_input(ref_image, edit_image_dict, preprocess = None):
+            err_msg = ""
+            is_suc = True
+            if ref_image is not None:
+                ref_image = pillow_convert(ref_image, "RGB")
+
+            if edit_image_dict is None:
+                edit_image = None
+                edit_mask = None
+            else:
+                edit_image = edit_image_dict["background"]
+                edit_mask = np.array(edit_image_dict["layers"][0])[:, :, 3]
+                if np.sum(np.array(edit_image)) < 1:
+                    edit_image = None
+                    edit_mask = None
+                elif np.sum(np.array(edit_mask)) < 1:
+                    edit_image = pillow_convert(edit_image, "RGB")
+                    w, h = edit_image.size
+                    edit_mask = Image.new("L", (w, h), 255)
+                else:
+                    edit_image = pillow_convert(edit_image, "RGB")
+                    edit_mask = Image.fromarray(edit_mask).convert('L')
+            if ref_image is None and edit_image is None:
+                err_msg = "Please provide the reference image or edited image."
+                return None, None, None, False, err_msg
+            return edit_image, edit_mask, ref_image, is_suc, err_msg
+
+        def run_chat(
+                     prompt,
+                     ref_image,
+                     edit_image,
+                     edit_type,
+                     cfg_scale,
+                     step,
+                     seed,
+                     output_h,
+                     output_w,
+                     repainting_scale,
+                     use_change,
+                     keep_pixel,
+                     keep_pixels_rate,
+                     progress=gr.Progress(track_tqdm=True)
+                    ):
+            edit_info = self.edit_type_dict[edit_type]
+            pre_edit_image, pre_edit_mask, pre_ref_image, is_suc, err_msg = preprocess_input(ref_image, edit_image)
+            icon = pre_edit_image or pre_ref_image
+            if not is_suc:
+                err_msg = f"<mark>{err_msg}</mark>"
+                return (gr.Image(), gr.Column(visible=True),
+                        gr.Image(),
+                        gr.Image(),
+                        gr.Image(),
+                        gr.Text(value=err_msg))
+            pre_edit_image = edit_preprocess(edit_info.ANNOTATOR, we.device_id, pre_edit_image, pre_edit_mask)
+            # edit_image["background"] = pre_edit_image
+            st = time.time()
+            image, edit_image, change_image, mask, seed = self.pipe(
+                reference_image=pre_ref_image,
+                edit_image=pre_edit_image,
+                edit_mask=pre_edit_mask,
+                prompt=prompt,
+                output_height=output_h,
+                output_width=output_w,
+                sampler='flow_euler',
+                sample_steps=step,
+                guide_scale=cfg_scale,
+                seed=seed,
+                repainting_scale=repainting_scale,
+                use_change=use_change,
+                keep_pixels=keep_pixel,
+                keep_pixels_rate=keep_pixels_rate
+            )
+            et = time.time()
+            msg = f"prompt: {prompt}; seed: {seed}; cost time: {et - st}s; repaiting scale: {repainting_scale}"
+
+            if icon is not None:
+                icon = resize_image(icon, 150)
+
+            return (gr.Image(value=image), gr.Column(visible=True),
+                    gr.Image(value=edit_image if edit_image is not None else edit_image),
+                    gr.Image(value=change_image),
+                    gr.Image(value=pre_edit_mask if pre_edit_mask is not None else None),
+                    gr.Text(value=msg),
+                    gr.Image(value=icon))
+
+        chat_inputs = [
+            self.reference_image,
+            self.edit_image,
+            self.edit_type,
+            self.cfg_scale,
+            self.step,
+            self.seed,
+            self.output_height,
+            self.output_width,
+            self.repainting_scale,
+            self.use_change,
+            self.keep_pixel,
+            self.keep_pixels_rate
+        ]
+
+        chat_outputs = [
+           self.gallery_image, self.edit_preprocess_panel, self.edit_preprocess_preview,
+            self.change_preprocess_preview,
+            self.edit_preprocess_mask_preview, self.generation_info_preview,
+            self.icon
+        ]
+
+        self.chat_btn.click(run_chat,
+                            inputs=[self.text] + chat_inputs,
+                            outputs=chat_outputs,
+                            queue=True)
+
+        self.text.submit(run_chat,
+                         inputs=[self.text] + chat_inputs,
+                         outputs=chat_outputs,
+                         queue=True)
+
+        def run_example(edit_type, prompt, ref_image, edit_image, edit_mask,
+                        output_h, output_w, seed, use_change, keep_pixel,
+                        keep_pixels_rate,
+                        progress=gr.Progress(track_tqdm=True)):
+
+            step = self.pipe.input.get("sample_steps", 20)
+            cfg_scale = self.pipe.input.get("guide_scale", 20)
+            edit_info = self.edit_type_dict[edit_type]
+
+            edit_image = self.construct_edit_image(edit_image, edit_mask)
+
+            pre_edit_image, pre_edit_mask, pre_ref_image, _, _ = preprocess_input(ref_image, edit_image)
+
+            icon = pre_edit_image or pre_ref_image
+
+            pre_edit_image = edit_preprocess(edit_info.ANNOTATOR, we.device_id, pre_edit_image, pre_edit_mask)
+            edit_info = edit_info or {}
+            repainting_scale = edit_info.get("REPAINTING_SCALE", 1.0)
+            st = time.time()
+            image, edit_image, change_image, mask, seed = self.pipe(
+                reference_image=pre_ref_image,
+                edit_image=pre_edit_image,
+                edit_mask=pre_edit_mask,
+                prompt=prompt,
+                output_height=output_h,
+                output_width=output_w,
+                sampler='flow_euler',
+                sample_steps=step,
+                guide_scale=cfg_scale,
+                seed=seed,
+                repainting_scale=repainting_scale,
+                use_change=use_change,
+                keep_pixels=keep_pixel,
+                keep_pixels_rate=keep_pixels_rate
+            )
+            et = time.time()
+            msg = f"prompt: {prompt}; seed: {seed}; cost time: {et - st}s; repaiting scale: {repainting_scale}"
+            if pre_edit_image is not None:
+                ret_image = Image.composite(Image.new("RGB", pre_edit_image.size, (0, 0, 0)), pre_edit_image,  pre_edit_mask)
+            else:
+                ret_image = None
+
+            if icon is not None:
+                icon = resize_image(icon, 150)
+            return (gr.Image(value=image), gr.Column(visible=True),
+                    gr.Image(value=edit_image if edit_image is not None else edit_image),
+                    gr.Image(value=change_image),
+                    gr.Image(value=pre_edit_mask if pre_edit_mask is not None else None),
+                    gr.Text(value=msg),
+                    gr.update(value=ret_image),
+                    gr.Image(value= icon))
+
+        with self.eg:
+            self.example_edit_image = gr.Image(label='Edit Image',
+                                          type='pil',
+                                          image_mode='RGB',
+                                          visible=False)
+            self.example_edit_mask = gr.Image(label='Edit Image Mask',
+                                         type='pil',
+                                         image_mode='L',
+                                         visible=False)
+
+            self.examples = gr.Examples(
+                fn=run_example,
+                examples=self.all_examples,
+                inputs=[
+                    self.edit_type, self.text, self.reference_image, self.example_edit_image,
+                    self.example_edit_mask, self.output_height, self.output_width, self.seed,
+                    self.use_change, self.keep_pixel, self.keep_pixels_rate
+                ],
+                outputs=[self.gallery_image, self.edit_preprocess_panel, self.edit_preprocess_preview,
+                         self.change_preprocess_preview,
+                         self.edit_preprocess_mask_preview, self.generation_info_preview,
+                         self.edit_image,
+                         self.icon],
+                examples_per_page=15,
+                cache_examples=False,
+                run_on_click=True)
+
+
+def run_gr(cfg):
+    with gr.Blocks() as demo:
+        chatbot = DemoUI()
+        chatbot.create_ui()
+        chatbot.set_callbacks()
+        demo.launch(server_name='0.0.0.0',
+                    server_port=cfg.args.server_port,
+                    root_path=cfg.args.root_path)
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(description='Argparser for Scepter:\n')
+    parser.add_argument('--server_port',
+                        dest='server_port',
+                        help='',
+                        type=int,
+                        default=2345)
+    parser.add_argument('--root_path', dest='root_path', help='', default='')
+    cfg = Config(load=True, parser_ins=parser)
+    run_gr(cfg)

ACE_plus/demo_lora.py

@@ -0,0 +1,523 @@
+# -*- coding: utf-8 -*-
+# Copyright (c) Alibaba, Inc. and its affiliates.
+import argparse
+import csv
+import glob
+import os
+import sys
+import threading
+import time
+
+import gradio as gr
+import numpy as np
+import torch, importlib
+from PIL import Image
+from scepter.modules.transform.io import pillow_convert
+from scepter.modules.utils.config import Config
+from scepter.modules.utils.distribute import we
+from scepter.modules.utils.file_system import FS
+
+if os.path.exists('__init__.py'):
+    package_name = 'scepter_ext'
+    spec = importlib.util.spec_from_file_location(package_name, '__init__.py')
+    package = importlib.util.module_from_spec(spec)
+    sys.modules[package_name] = package
+    spec.loader.exec_module(package)
+
+from inference.ace_plus_diffusers import ACEPlusDiffuserInference
+from inference.utils import edit_preprocess
+from examples.examples import all_examples
+
+inference_dict = {
+    "ACE_DIFFUSER_PLUS": ACEPlusDiffuserInference
+}
+
+fs_list = [
+    Config(cfg_dict={"NAME": "HuggingfaceFs", "TEMP_DIR": "./cache"}, load=False),
+    Config(cfg_dict={"NAME": "ModelscopeFs", "TEMP_DIR": "./cache"}, load=False),
+    Config(cfg_dict={"NAME": "HttpFs", "TEMP_DIR": "./cache"}, load=False),
+    Config(cfg_dict={"NAME": "LocalFs", "TEMP_DIR": "./cache"}, load=False),
+]
+
+for one_fs in fs_list:
+    FS.init_fs_client(one_fs)
+
+
+csv.field_size_limit(sys.maxsize)
+refresh_sty = '\U0001f504'  # 🔄
+clear_sty = '\U0001f5d1'  # 🗑️
+upload_sty = '\U0001f5bc'  # 🖼️
+sync_sty = '\U0001f4be'  # 💾
+chat_sty = '\U0001F4AC'  # 💬
+video_sty = '\U0001f3a5'  # 🎥
+
+lock = threading.Lock()
+class DemoUI(object):
+    def __init__(self,
+                 infer_dir = "./config/ace_plus_diffusers_infer.yaml",
+                 model_list='./models/model_zoo.yaml'
+                 ):
+        self.model_yamls = [infer_dir]
+        self.model_choices = dict()
+        self.default_model_name = ''
+        for i in self.model_yamls:
+            model_cfg = Config(load=True, cfg_file=i)
+            model_name = model_cfg.NAME
+            if model_cfg.IS_DEFAULT: self.default_model_name = model_name
+            self.model_choices[model_name] = model_cfg
+        print('Models: ', self.model_choices.keys())
+        assert len(self.model_choices) > 0
+        if self.default_model_name == "": self.default_model_name = list(self.model_choices.keys())[0]
+        self.model_name = self.default_model_name
+        pipe_cfg = self.model_choices[self.default_model_name]
+        infer_name = pipe_cfg.get("INFERENCE_TYPE", "ACE")
+        self.pipe = inference_dict[infer_name]()
+        self.pipe.init_from_cfg(pipe_cfg)
+
+        # choose different model
+        self.task_model_cfg = Config(load=True, cfg_file=model_list)
+        self.task_model = {}
+        self.task_model_list = []
+        self.edit_type_dict = {"repainting": None}
+        self.edit_type_list = ["repainting"]
+        for task_name, task_model in self.task_model_cfg.MODEL.items():
+            self.task_model[task_name.lower()] = task_model
+            self.task_model_list.append(task_name.lower())
+            for preprocessor in task_model.get("PREPROCESSOR", []):
+                if preprocessor["TYPE"] in self.edit_type_dict:
+                    continue
+                preprocessor["REPAINTING_SCALE"] = task_model.get("REPAINTING_SCALE", 1.0)
+                self.edit_type_dict[preprocessor["TYPE"]] = preprocessor
+        self.max_msgs = 20
+        # reformat examples
+        self.all_examples = [
+            [
+             one_example["task_type"], one_example["edit_type"], one_example["instruction"],
+             one_example["input_reference_image"], one_example["input_image"],
+             one_example["input_mask"], one_example["output_h"],
+             one_example["output_w"], one_example["seed"]
+             ]
+            for one_example in all_examples
+        ]
+
+    def construct_edit_image(self, edit_image, edit_mask):
+        if edit_image is not None and edit_mask is not None:
+            edit_image_rgb = pillow_convert(edit_image, "RGB")
+            edit_image_rgba = pillow_convert(edit_image, "RGBA")
+            edit_mask = pillow_convert(edit_mask, "L")
+
+            arr1 = np.array(edit_image_rgb)
+            arr2 = np.array(edit_mask)[:, :, np.newaxis]
+            result_array = np.concatenate((arr1, arr2), axis=2)
+            layer = Image.fromarray(result_array)
+
+            ret_data = {
+                "background": edit_image_rgba,
+                "composite": edit_image_rgba,
+                "layers": [layer]
+            }
+            return ret_data
+        else:
+            return None
+
+
+
+
+    def create_ui(self):
+        with gr.Row(equal_height=True, visible=True):
+            with gr.Column(scale=2):
+                self.gallery_image = gr.Image(
+                    height=600,
+                    interactive=False,
+                    type='pil',
+                    elem_id='Reference_image'
+                )
+            with gr.Column(scale=1, visible=True) as self.edit_preprocess_panel:
+                with gr.Row():
+                    with gr.Accordion(label='Related Input Image', open=False):
+                        self.edit_preprocess_preview = gr.Image(
+                            height=600,
+                            interactive=False,
+                            type='pil',
+                            elem_id='preprocess_image'
+                        )
+
+                        self.edit_preprocess_mask_preview = gr.Image(
+                            height=600,
+                            interactive=False,
+                            type='pil',
+                            elem_id='preprocess_image_mask'
+                        )
+                with gr.Row():
+                    instruction = """
+                               **Instruction**:
+                               1. Please choose the Task Type based on the scenario of the generation task. We provide three types of generation capabilities: Portrait ID Preservation Generation(portrait), 
+                                Object ID Preservation Generation(subject), and Local Controlled Generation(local editing), which can be selected from the task dropdown menu.
+                               2. When uploading images in the Reference Image section, the generated image will reference the ID information of that image. Please ensure that the ID information is clear. 
+                                In the Edit Image section, the uploaded image will maintain its structural and content information, and you must draw a mask area to specify the region to be regenerated.
+                               3. When the task type is local editing, there are various editing types to choose from. Users can select different information preserving dimensions, such as edge information, 
+                                color information, and more. The pre-processing information can be viewed in the 'related input image' tab.
+                            """
+                    self.instruction = gr.Markdown(value=instruction)
+        with gr.Row():
+            self.model_name_dd = gr.Dropdown(
+                choices=self.model_choices,
+                value=self.default_model_name,
+                label='Model Version')
+            self.task_type = gr.Dropdown(choices=self.task_model_list,
+                                         interactive=True,
+                                         value=self.task_model_list[0],
+                                         label='Task Type')
+            self.edit_type = gr.Dropdown(choices=self.edit_type_list,
+                                         interactive=True,
+                                         value=self.edit_type_list[0],
+                                         label='Edit Type')
+        with gr.Row():
+            self.generation_info_preview = gr.Markdown(
+                label='System Log.',
+                show_label=True)
+        with gr.Row(variant='panel',
+                    equal_height=True,
+                    show_progress=False):
+            with gr.Column(scale=10, min_width=500):
+                self.text = gr.Textbox(
+                    placeholder='Input "@" find history of image',
+                    label='Instruction',
+                    container=False,
+                    lines = 1)
+            with gr.Column(scale=2, min_width=100):
+                with gr.Row():
+                    with gr.Column(scale=1, min_width=100):
+                        self.chat_btn = gr.Button(value='Generate', variant = "primary")
+
+        with gr.Accordion(label='Advance', open=True):
+            with gr.Row(visible=True):
+                with gr.Column():
+                    self.reference_image = gr.Image(
+                        height=1000,
+                        interactive=True,
+                        image_mode='RGB',
+                        type='pil',
+                        label='Reference Image',
+                        elem_id='reference_image'
+                    )
+                with gr.Column():
+                    self.edit_image = gr.ImageMask(
+                        height=1000,
+                        interactive=True,
+                        value=None,
+                        sources=['upload'],
+                        type='pil',
+                        layers=False,
+                        label='Edit Image',
+                        elem_id='image_editor',
+                        show_fullscreen_button=True,
+                        format="png"
+                    )
+
+            with gr.Row():
+                self.step = gr.Slider(minimum=1,
+                                      maximum=1000,
+                                      value=self.pipe.input.get("sample_steps", 20),
+                                      visible=self.pipe.input.get("sample_steps", None) is not None,
+                                      label='Sample Step')
+                self.cfg_scale = gr.Slider(
+                    minimum=1.0,
+                    maximum=100.0,
+                    value=self.pipe.input.get("guide_scale", 4.5),
+                    visible=self.pipe.input.get("guide_scale", None) is not None,
+                    label='Guidance Scale')
+                self.seed = gr.Slider(minimum=-1,
+                                      maximum=10000000,
+                                      value=-1,
+                                      label='Seed')
+                self.output_height = gr.Slider(
+                    minimum=256,
+                    maximum=1440,
+                    value=self.pipe.input.get("output_height", 1024),
+                    visible=self.pipe.input.get("output_height", None) is not None,
+                    label='Output Height')
+                self.output_width = gr.Slider(
+                    minimum=256,
+                    maximum=1440,
+                    value=self.pipe.input.get("output_width", 1024),
+                    visible=self.pipe.input.get("output_width", None) is not None,
+                    label='Output Width')
+
+                self.repainting_scale = gr.Slider(
+                    minimum=0.0,
+                    maximum=1.0,
+                    value=self.pipe.input.get("repainting_scale", 1.0),
+                    visible=True,
+                    label='Repainting Scale')
+            with gr.Row():
+                self.eg = gr.Column(visible=True)
+
+
+
+    def set_callbacks(self, *args, **kwargs):
+        ########################################
+        def change_model(model_name):
+            if model_name not in self.model_choices:
+                gr.Info('The provided model name is not a valid choice!')
+                return model_name, gr.update(), gr.update()
+
+            if model_name != self.model_name:
+                lock.acquire()
+                del self.pipe
+                torch.cuda.empty_cache()
+                torch.cuda.ipc_collect()
+                pipe_cfg = self.model_choices[model_name]
+                infer_name = pipe_cfg.get("INFERENCE_TYPE", "ACE")
+                self.pipe = inference_dict[infer_name]()
+                self.pipe.init_from_cfg(pipe_cfg)
+                self.model_name = model_name
+                lock.release()
+
+            return (model_name, gr.update(),
+                    gr.Slider(
+                              value=self.pipe.input.get("sample_steps", 20),
+                              visible=self.pipe.input.get("sample_steps", None) is not None),
+                    gr.Slider(
+                        value=self.pipe.input.get("guide_scale", 4.5),
+                        visible=self.pipe.input.get("guide_scale", None) is not None),
+                    gr.Slider(
+                        value=self.pipe.input.get("output_height", 1024),
+                        visible=self.pipe.input.get("output_height", None) is not None),
+                    gr.Slider(
+                        value=self.pipe.input.get("output_width", 1024),
+                        visible=self.pipe.input.get("output_width", None) is not None),
+                    gr.Slider(value=self.pipe.input.get("repainting_scale", 1.0))
+                    )
+
+        self.model_name_dd.change(
+            change_model,
+            inputs=[self.model_name_dd],
+            outputs=[
+                self.model_name_dd, self.text,
+                self.step,
+                self.cfg_scale,
+                self.output_height,
+                self.output_width,
+                self.repainting_scale])
+
+        def change_task_type(task_type):
+            task_info = self.task_model[task_type]
+            edit_type_list = [self.edit_type_list[0]]
+            for preprocessor in task_info.get("PREPROCESSOR", []):
+                preprocessor["REPAINTING_SCALE"] = task_info.get("REPAINTING_SCALE", 1.0)
+                self.edit_type_dict[preprocessor["TYPE"]] = preprocessor
+                edit_type_list.append(preprocessor["TYPE"])
+
+            return gr.update(choices=edit_type_list, value=edit_type_list[0])
+
+        self.task_type.change(change_task_type, inputs=[self.task_type], outputs=[self.edit_type])
+
+        def change_edit_type(edit_type):
+            edit_info = self.edit_type_dict[edit_type]
+            edit_info = edit_info or {}
+            repainting_scale = edit_info.get("REPAINTING_SCALE", 1.0)
+            if edit_type == self.edit_type_list[0]:
+                return  gr.Slider(value=1.0)
+            else:
+                return  gr.Slider(
+                    value=repainting_scale)
+
+        self.edit_type.change(change_edit_type, inputs=[self.edit_type], outputs=[self.repainting_scale])
+
+        def preprocess_input(ref_image, edit_image_dict, preprocess = None):
+            err_msg = ""
+            is_suc = True
+            if ref_image is not None:
+                ref_image = pillow_convert(ref_image, "RGB")
+
+            if edit_image_dict is None:
+                edit_image = None
+                edit_mask = None
+            else:
+                edit_image = edit_image_dict["background"]
+                edit_mask = np.array(edit_image_dict["layers"][0])[:, :, 3]
+                if np.sum(np.array(edit_image)) < 1:
+                    edit_image = None
+                    edit_mask = None
+                elif np.sum(np.array(edit_mask)) < 1:
+                    err_msg = "You must draw the repainting area for the edited image."
+                    return None, None, None, False, err_msg
+                else:
+                    edit_image = pillow_convert(edit_image, "RGB")
+                    edit_mask = Image.fromarray(edit_mask).convert('L')
+            if ref_image is None and edit_image is None:
+                err_msg = "Please provide the reference image or edited image."
+                return None, None, None, False, err_msg
+            return edit_image, edit_mask, ref_image, is_suc, err_msg
+
+        def run_chat(
+                     prompt,
+                     ref_image,
+                     edit_image,
+                     task_type,
+                     edit_type,
+                     cfg_scale,
+                     step,
+                     seed,
+                     output_h,
+                     output_w,
+                     repainting_scale,
+                progress=gr.Progress(track_tqdm=True)
+        ):
+            model_path = self.task_model[task_type]["MODEL_PATH"]
+            edit_info = self.edit_type_dict[edit_type]
+
+            if task_type in ["portrait", "subject"] and ref_image is None:
+                err_msg = "<mark>Please provide the reference image.</mark>"
+                return (gr.Image(), gr.Column(visible=True),
+                        gr.Image(),
+                        gr.Image(),
+                        gr.Text(value=err_msg))
+
+            pre_edit_image, pre_edit_mask, pre_ref_image, is_suc, err_msg = preprocess_input(ref_image, edit_image)
+            if not is_suc:
+                err_msg = f"<mark>{err_msg}</mark>"
+                return (gr.Image(), gr.Column(visible=True),
+                        gr.Image(),
+                        gr.Image(),
+                        gr.Text(value=err_msg))
+            pre_edit_image = edit_preprocess(edit_info, we.device_id, pre_edit_image, pre_edit_mask)
+            # edit_image["background"] = pre_edit_image
+            st = time.time()
+            image, seed = self.pipe(
+                reference_image=pre_ref_image,
+                edit_image=pre_edit_image,
+                edit_mask=pre_edit_mask,
+                prompt=prompt,
+                output_height=output_h,
+                output_width=output_w,
+                sampler='flow_euler',
+                sample_steps=step,
+                guide_scale=cfg_scale,
+                seed=seed,
+                repainting_scale=repainting_scale,
+                lora_path = model_path
+            )
+            et = time.time()
+            msg = f"prompt: {prompt}; seed: {seed}; cost time: {et - st}s; repaiting scale: {repainting_scale}"
+
+            return (gr.Image(value=image), gr.Column(visible=True),
+                    gr.Image(value=pre_edit_image if pre_edit_image is not None else pre_ref_image),
+                    gr.Image(value=pre_edit_mask if pre_edit_mask is not None else None),
+                    gr.Text(value=msg))
+
+        chat_inputs = [
+            self.reference_image,
+            self.edit_image,
+            self.task_type,
+            self.edit_type,
+            self.cfg_scale,
+            self.step,
+            self.seed,
+            self.output_height,
+            self.output_width,
+            self.repainting_scale
+        ]
+
+        chat_outputs = [
+           self.gallery_image, self.edit_preprocess_panel, self.edit_preprocess_preview,
+            self.edit_preprocess_mask_preview, self.generation_info_preview
+        ]
+
+        self.chat_btn.click(run_chat,
+                            inputs=[self.text] + chat_inputs,
+                            outputs=chat_outputs,
+                            queue=True)
+
+        self.text.submit(run_chat,
+                         inputs=[self.text] + chat_inputs,
+                         outputs=chat_outputs,
+                         queue=True)
+
+        def run_example(task_type, edit_type, prompt, ref_image, edit_image, edit_mask,
+                        output_h, output_w, seed, progress=gr.Progress(track_tqdm=True)):
+            model_path = self.task_model[task_type]["MODEL_PATH"]
+
+            step = self.pipe.input.get("sample_steps", 20)
+            cfg_scale = self.pipe.input.get("guide_scale", 20)
+
+            edit_info = self.edit_type_dict[edit_type]
+
+            edit_image = self.construct_edit_image(edit_image, edit_mask)
+
+            pre_edit_image, pre_edit_mask, pre_ref_image, _, _ = preprocess_input(ref_image, edit_image)
+            pre_edit_image = edit_preprocess(edit_info, we.device_id, pre_edit_image, pre_edit_mask)
+            edit_info = edit_info or {}
+            repainting_scale = edit_info.get("REPAINTING_SCALE", 1.0)
+            st = time.time()
+            image, seed = self.pipe(
+                reference_image=pre_ref_image,
+                edit_image=pre_edit_image,
+                edit_mask=pre_edit_mask,
+                prompt=prompt,
+                output_height=output_h,
+                output_width=output_w,
+                sampler='flow_euler',
+                sample_steps=step,
+                guide_scale=cfg_scale,
+                seed=seed,
+                repainting_scale=repainting_scale,
+                lora_path=model_path
+            )
+            et = time.time()
+            msg = f"prompt: {prompt}; seed: {seed}; cost time: {et - st}s; repaiting scale: {repainting_scale}"
+            if pre_edit_image is not None:
+                ret_image = Image.composite(Image.new("RGB", pre_edit_image.size, (0, 0, 0)), pre_edit_image,  pre_edit_mask)
+            else:
+                ret_image = None
+            return (gr.Image(value=image), gr.Column(visible=True),
+                    gr.Image(value=pre_edit_image if pre_edit_image is not None else pre_ref_image),
+                    gr.Image(value=pre_edit_mask if pre_edit_mask is not None else None),
+                    gr.Text(value=msg),
+                    gr.update(value=ret_image))
+
+        with self.eg:
+            self.example_edit_image = gr.Image(label='Edit Image',
+                                          type='pil',
+                                          image_mode='RGB',
+                                          visible=False)
+            self.example_edit_mask = gr.Image(label='Edit Image Mask',
+                                         type='pil',
+                                         image_mode='L',
+                                         visible=False)
+
+            self.examples = gr.Examples(
+                fn=run_example,
+                examples=self.all_examples,
+                inputs=[
+                    self.task_type, self.edit_type, self.text, self.reference_image, self.example_edit_image,
+                    self.example_edit_mask, self.output_height, self.output_width, self.seed
+                ],
+                outputs=[self.gallery_image, self.edit_preprocess_panel, self.edit_preprocess_preview,
+                         self.edit_preprocess_mask_preview, self.generation_info_preview, self.edit_image],
+                examples_per_page=6,
+                cache_examples=False,
+                run_on_click=True)
+
+
+def run_gr(cfg):
+    with gr.Blocks() as demo:
+        chatbot = DemoUI()
+        chatbot.create_ui()
+        chatbot.set_callbacks()
+        demo.launch(server_name='0.0.0.0',
+                    server_port=cfg.args.server_port,
+                    root_path=cfg.args.root_path)
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(description='Argparser for Scepter:\n')
+    parser.add_argument('--server_port',
+                        dest='server_port',
+                        help='',
+                        type=int,
+                        default=2345)
+    parser.add_argument('--root_path', dest='root_path', help='', default='')
+    cfg = Config(load=True, parser_ins=parser)
+    run_gr(cfg)

ACE_plus/infer_fft.py

@@ -0,0 +1,178 @@
+# -*- coding: utf-8 -*-
+# Copyright (c) Alibaba, Inc. and its affiliates.
+import argparse
+import glob
+import importlib
+import io
+import os
+import sys
+
+from PIL import Image
+from scepter.modules.transform.io import pillow_convert
+from scepter.modules.utils.config import Config
+from scepter.modules.utils.file_system import FS
+
+if os.path.exists('__init__.py'):
+    package_name = 'scepter_ext'
+    spec = importlib.util.spec_from_file_location(package_name, '__init__.py')
+    package = importlib.util.module_from_spec(spec)
+    sys.modules[package_name] = package
+    spec.loader.exec_module(package)
+
+from examples.examples import fft_examples as all_examples
+from inference.registry import INFERENCES
+fs_list = [
+    Config(cfg_dict={"NAME": "HuggingfaceFs", "TEMP_DIR": "./cache"}, load=False),
+    Config(cfg_dict={"NAME": "ModelscopeFs", "TEMP_DIR": "./cache"}, load=False),
+    Config(cfg_dict={"NAME": "HttpFs", "TEMP_DIR": "./cache"}, load=False),
+    Config(cfg_dict={"NAME": "LocalFs", "TEMP_DIR": "./cache"}, load=False),
+]
+
+for one_fs in fs_list:
+    FS.init_fs_client(one_fs)
+
+
+def run_one_case(pipe,
+                input_image = None,
+                input_mask = None,
+                input_reference_image = None,
+                save_path = "examples/output/example.png",
+                instruction = "",
+                output_h = 1024,
+                output_w = 1024,
+                seed = -1,
+                sample_steps = None,
+                guide_scale = None,
+                repainting_scale = None,
+                use_change=True,
+                keep_pixels=True,
+                keep_pixels_rate=0.8,
+                **kwargs):
+    if input_image is not None:
+        input_image = Image.open(io.BytesIO(FS.get_object(input_image)))
+        input_image = pillow_convert(input_image, "RGB")
+    if input_mask is not None:
+        input_mask = Image.open(io.BytesIO(FS.get_object(input_mask)))
+        input_mask = pillow_convert(input_mask, "L")
+    if input_reference_image is not None:
+        input_reference_image = Image.open(io.BytesIO(FS.get_object(input_reference_image)))
+        input_reference_image = pillow_convert(input_reference_image, "RGB")
+    print(repainting_scale)
+    image, _, _, _, seed = pipe(
+        reference_image=input_reference_image,
+        edit_image=input_image,
+        edit_mask=input_mask,
+        prompt=instruction,
+        output_height=output_h,
+        output_width=output_w,
+        sampler='flow_euler',
+        sample_steps=sample_steps or pipe.input.get("sample_steps", 28),
+        guide_scale=guide_scale or pipe.input.get("guide_scale", 50),
+        seed=seed,
+        repainting_scale=repainting_scale,
+        use_change=use_change,
+        keep_pixels=keep_pixels,
+        keep_pixels_rate=keep_pixels_rate
+    )
+    with FS.put_to(save_path) as local_path:
+        image.save(local_path)
+    return local_path, seed
+
+
+def run():
+    parser = argparse.ArgumentParser(description='Argparser for Scepter:\n')
+    parser.add_argument('--instruction',
+                        dest='instruction',
+                        help='The instruction for editing or generating!',
+                        default="")
+    parser.add_argument('--output_h',
+                        dest='output_h',
+                        help='The height of output image for generation tasks!',
+                        type=int,
+                        default=1024)
+    parser.add_argument('--output_w',
+                        dest='output_w',
+                        help='The width of output image for generation tasks!',
+                        type=int,
+                        default=1024)
+    parser.add_argument('--input_reference_image',
+                        dest='input_reference_image',
+                        help='The input reference image!',
+                        default=None
+                        )
+    parser.add_argument('--input_image',
+                        dest='input_image',
+                        help='The input image!',
+                        default=None
+                        )
+    parser.add_argument('--input_mask',
+                        dest='input_mask',
+                        help='The input mask!',
+                        default=None
+                        )
+    parser.add_argument('--save_path',
+                        dest='save_path',
+                        help='The save path for output image!',
+                        default='examples/output_images/output.png'
+                        )
+    parser.add_argument('--seed',
+                        dest='seed',
+                        help='The seed for generation!',
+                        type=int,
+                        default=-1)
+
+    parser.add_argument('--step',
+                        dest='step',
+                        help='The sample step for generation!',
+                        type=int,
+                        default=None)
+
+    parser.add_argument('--guide_scale',
+                        dest='guide_scale',
+                        help='The guide scale for generation!',
+                        type=int,
+                        default=None)
+
+    parser.add_argument('--repainting_scale',
+                        dest='repainting_scale',
+                        help='The repainting scale for content filling generation!',
+                        type=int,
+                        default=None)
+
+    cfg = Config(load=True, parser_ins=parser)
+    model_cfg = Config(load=True, cfg_file="config/ace_plus_fft.yaml")
+    pipe = INFERENCES.build(model_cfg)
+
+
+    if cfg.args.instruction == "" and cfg.args.input_image is None and cfg.args.input_reference_image is None:
+        params = {
+            "output_h": cfg.args.output_h,
+            "output_w": cfg.args.output_w,
+            "sample_steps": cfg.args.step,
+            "guide_scale": cfg.args.guide_scale
+        }
+        # run examples
+
+        for example in all_examples:
+            example.update(params)
+            local_path, seed = run_one_case(pipe, **example)
+
+    else:
+        params = {
+            "input_image": cfg.args.input_image,
+            "input_mask": cfg.args.input_mask,
+            "input_reference_image": cfg.args.input_reference_image,
+            "save_path": cfg.args.save_path,
+            "instruction": cfg.args.instruction,
+            "output_h": cfg.args.output_h,
+            "output_w": cfg.args.output_w,
+            "sample_steps": cfg.args.step,
+            "guide_scale": cfg.args.guide_scale,
+            "repainting_scale": cfg.args.repainting_scale,
+        }
+        local_path, seed = run_one_case(pipe, **params)
+        print(local_path, seed)
+
+if __name__ == '__main__':
+    run()
+

ACE_plus/infer_lora.py

@@ -0,0 +1,228 @@
+# -*- coding: utf-8 -*-
+# Copyright (c) Alibaba, Inc. and its affiliates.
+import argparse
+import glob
+import io
+import os
+
+from PIL import Image
+from scepter.modules.transform.io import pillow_convert
+from scepter.modules.utils.config import Config
+from scepter.modules.utils.file_system import FS
+
+from examples.examples import all_examples
+from inference.ace_plus_diffusers import ACEPlusDiffuserInference
+inference_dict = {
+    "ACE_DIFFUSER_PLUS": ACEPlusDiffuserInference
+}
+
+fs_list = [
+    Config(cfg_dict={"NAME": "HuggingfaceFs", "TEMP_DIR": "./cache"}, load=False),
+    Config(cfg_dict={"NAME": "ModelscopeFs", "TEMP_DIR": "./cache"}, load=False),
+    Config(cfg_dict={"NAME": "HttpFs", "TEMP_DIR": "./cache"}, load=False),
+    Config(cfg_dict={"NAME": "LocalFs", "TEMP_DIR": "./cache"}, load=False),
+]
+
+for one_fs in fs_list:
+    FS.init_fs_client(one_fs)
+
+
+def run_one_case(pipe,
+                input_image = None,
+                input_mask = None,
+                input_reference_image = None,
+                save_path = "examples/output/example.png",
+                instruction = "",
+                output_h = 1024,
+                output_w = 1024,
+                seed = -1,
+                sample_steps = None,
+                guide_scale = None,
+                repainting_scale = None,
+                model_path = None,
+                **kwargs):
+    if input_image is not None:
+        input_image = Image.open(io.BytesIO(FS.get_object(input_image)))
+        input_image = pillow_convert(input_image, "RGB")
+    if input_mask is not None:
+        input_mask = Image.open(io.BytesIO(FS.get_object(input_mask)))
+        input_mask = pillow_convert(input_mask, "L")
+    if input_reference_image is not None:
+        input_reference_image = Image.open(io.BytesIO(FS.get_object(input_reference_image)))
+        input_reference_image = pillow_convert(input_reference_image, "RGB")
+
+    image, seed = pipe(
+        reference_image=input_reference_image,
+        edit_image=input_image,
+        edit_mask=input_mask,
+        prompt=instruction,
+        output_height=output_h,
+        output_width=output_w,
+        sampler='flow_euler',
+        sample_steps=sample_steps or pipe.input.get("sample_steps", 28),
+        guide_scale=guide_scale or pipe.input.get("guide_scale", 50),
+        seed=seed,
+        repainting_scale=repainting_scale or pipe.input.get("repainting_scale", 1.0),
+        lora_path = model_path
+    )
+    with FS.put_to(save_path) as local_path:
+        image.save(local_path)
+    return local_path, seed
+
+
+def run():
+    parser = argparse.ArgumentParser(description='Argparser for Scepter:\n')
+    parser.add_argument('--instruction',
+                        dest='instruction',
+                        help='The instruction for editing or generating!',
+                        default="")
+    parser.add_argument('--output_h',
+                        dest='output_h',
+                        help='The height of output image for generation tasks!',
+                        type=int,
+                        default=1024)
+    parser.add_argument('--output_w',
+                        dest='output_w',
+                        help='The width of output image for generation tasks!',
+                        type=int,
+                        default=1024)
+    parser.add_argument('--input_reference_image',
+                        dest='input_reference_image',
+                        help='The input reference image!',
+                        default=None
+                        )
+    parser.add_argument('--input_image',
+                        dest='input_image',
+                        help='The input image!',
+                        default=None
+                        )
+    parser.add_argument('--input_mask',
+                        dest='input_mask',
+                        help='The input mask!',
+                        default=None
+                        )
+    parser.add_argument('--save_path',
+                        dest='save_path',
+                        help='The save path for output image!',
+                        default='examples/output_images/output.png'
+                        )
+    parser.add_argument('--seed',
+                        dest='seed',
+                        help='The seed for generation!',
+                        type=int,
+                        default=-1)
+
+    parser.add_argument('--step',
+                        dest='step',
+                        help='The sample step for generation!',
+                        type=int,
+                        default=None)
+
+    parser.add_argument('--guide_scale',
+                        dest='guide_scale',
+                        help='The guide scale for generation!',
+                        type=int,
+                        default=None)
+
+    parser.add_argument('--repainting_scale',
+                        dest='repainting_scale',
+                        help='The repainting scale for content filling generation!',
+                        type=int,
+                        default=None)
+
+    parser.add_argument('--task_type',
+                        dest='task_type',
+                        choices=['portrait', 'subject', 'local_editing'],
+                        help="Choose the task type.",
+                        default='')
+
+    parser.add_argument('--task_model',
+                        dest='task_model',
+                        help='The models list for different tasks!',
+                        default="./models/model_zoo.yaml")
+
+
+    parser.add_argument('--infer_type',
+                        dest='infer_type',
+                        choices=['diffusers'],
+                        default='diffusers',
+                        help="Choose the inference scripts. 'native' refers to using the official implementation of ace++, "
+                             "while 'diffusers' refers to using the adaptation for diffusers")
+
+    parser.add_argument('--cfg_folder',
+                        dest='cfg_folder',
+                        help='The inference config!',
+                        default="./config")
+
+    cfg = Config(load=True, parser_ins=parser)
+
+    model_yamls = glob.glob(os.path.join(cfg.args.cfg_folder, '*.yaml'))
+    model_choices = dict()
+    for i in model_yamls:
+        model_cfg = Config(load=True, cfg_file=i)
+        model_name = model_cfg.NAME
+        model_choices[model_name] = model_cfg
+
+    if cfg.args.infer_type == "native":
+        infer_name = "ace_plus_native_infer"
+    elif cfg.args.infer_type == "diffusers":
+        infer_name = "ace_plus_diffuser_infer"
+    else:
+        raise ValueError("infer_type should be native or diffusers")
+
+    assert infer_name in model_choices
+
+    # choose different model
+    task_model_cfg = Config(load=True, cfg_file=cfg.args.task_model)
+
+    task_model_dict = {}
+    for task_name, task_model in task_model_cfg.MODEL.items():
+        task_model_dict[task_name] = task_model
+
+
+    # choose the inference scripts.
+    pipe_cfg = model_choices[infer_name]
+    infer_name = pipe_cfg.get("INFERENCE_TYPE", "ACE_PLUS")
+    pipe = inference_dict[infer_name]()
+    pipe.init_from_cfg(pipe_cfg)
+
+    if cfg.args.instruction == "" and cfg.args.input_image is None and cfg.args.input_reference_image is None:
+        params = {
+            "output_h": cfg.args.output_h,
+            "output_w": cfg.args.output_w,
+            "sample_steps": cfg.args.step,
+            "guide_scale": cfg.args.guide_scale
+        }
+        # run examples
+
+        for example in all_examples:
+            example["model_path"] = FS.get_from(task_model_dict[example["task_type"].upper()]["MODEL_PATH"])
+            example.update(params)
+            if example["edit_type"] == "repainting":
+                example["repainting_scale"] = 1.0
+            else:
+                example["repainting_scale"] = task_model_dict[example["task_type"].upper()].get("REPAINTING_SCALE", 1.0)
+            print(example)
+            local_path, seed = run_one_case(pipe, **example)
+
+    else:
+        assert cfg.args.task_type.upper() in task_model_cfg
+        params = {
+            "input_image": cfg.args.input_image,
+            "input_mask": cfg.args.input_mask,
+            "input_reference_image": cfg.args.input_reference_image,
+            "save_path": cfg.args.save_path,
+            "instruction": cfg.args.instruction,
+            "output_h": cfg.args.output_h,
+            "output_w": cfg.args.output_w,
+            "sample_steps": cfg.args.step,
+            "guide_scale": cfg.args.guide_scale,
+            "repainting_scale": cfg.args.repainting_scale,
+            "model_path": FS.get_from(task_model_dict[cfg.args.task_type.upper()]["MODEL_PATH"])
+        }
+        local_path, seed = run_one_case(pipe, **params)
+        print(local_path, seed)
+
+if __name__ == '__main__':
+    run()
+

ACE_plus/install.py

@@ -0,0 +1,27 @@
+# -*- coding: utf-8 -*-
+# Copyright (c) Alibaba, Inc. and its affiliates.
+import os
+import shutil
+import sys
+
+
+if sys.argv[0] == 'install.py':
+    sys.path.append('.')
+
+current_dir = os.path.dirname(__file__)
+source_folder = os.path.join(current_dir, "workflow/ComfyUI-ACE_Plus")
+target_folder = os.path.join(os.path.dirname(current_dir), "ComfyUI-ACE_Plus")
+
+if not os.path.exists(target_folder):
+    shutil.move(source_folder, target_folder)
+    print(f"{os.path.abspath(source_folder)} copy to {os.path.abspath(target_folder)} success!")
+else:
+    print(f"{os.path.abspath(target_folder)} exist.")
+
+try:
+    if os.path.exists(current_dir):
+        shutil.rmtree(current_dir)
+    else:
+        print(f"source_folder: '{current_dir}' not exist.")
+except Exception as e:
+    print(f"delete source_folder error: {e}")

ACE_plus/model_convert.py

@@ -0,0 +1,194 @@
+# Copy from https://gist.github.com/Stella2211/10f5bd870387ec1ddb9932235321068e
+# This is a great work.
+import json
+from pathlib import Path
+import torch
+from tqdm import tqdm
+import struct
+from typing import Dict, Any
+import sys
+
+# input file
+if(len(sys.argv) < 3):
+    print("Usage: mem_eff_fp8_convert.py {fp16 model path} {output path}")
+    sys.exit(1)
+
+path = sys.argv[1]
+output =sys.argv[2]
+model_file = Path(path)
+
+class MemoryEfficientSafeOpen:
+    # does not support metadata loading
+    def __init__(self, filename):
+        self.filename = filename
+        self.header, self.header_size = self._read_header()
+        self.file = open(filename, "rb")
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        self.file.close()
+
+    def keys(self):
+        return [k for k in self.header.keys() if k != "__metadata__"]
+
+    def get_tensor(self, key):
+        if key not in self.header:
+            raise KeyError(f"Tensor '{key}' not found in the file")
+
+        metadata = self.header[key]
+        offset_start, offset_end = metadata["data_offsets"]
+
+        if offset_start == offset_end:
+            tensor_bytes = None
+        else:
+            # adjust offset by header size
+            self.file.seek(self.header_size + 8 + offset_start)
+            tensor_bytes = self.file.read(offset_end - offset_start)
+
+        return self._deserialize_tensor(tensor_bytes, metadata)
+
+    def _read_header(self):
+        with open(self.filename, "rb") as f:
+            header_size = struct.unpack("<Q", f.read(8))[0]
+            header_json = f.read(header_size).decode("utf-8")
+            return json.loads(header_json), header_size
+
+    def _deserialize_tensor(self, tensor_bytes, metadata):
+        dtype = self._get_torch_dtype(metadata["dtype"])
+        shape = metadata["shape"]
+
+        if tensor_bytes is None:
+            byte_tensor = torch.empty(0, dtype=torch.uint8)
+        else:
+            tensor_bytes = bytearray(tensor_bytes)  # make it writable
+            byte_tensor = torch.frombuffer(tensor_bytes, dtype=torch.uint8)
+
+        # process float8 types
+        if metadata["dtype"] in ["F8_E5M2", "F8_E4M3"]:
+            return self._convert_float8(byte_tensor, metadata["dtype"], shape)
+
+        # convert to the target dtype and reshape
+        return byte_tensor.view(dtype).reshape(shape)
+
+    @staticmethod
+    def _get_torch_dtype(dtype_str):
+        dtype_map = {
+            "F64": torch.float64,
+            "F32": torch.float32,
+            "F16": torch.float16,
+            "BF16": torch.bfloat16,
+            "I64": torch.int64,
+            "I32": torch.int32,
+            "I16": torch.int16,
+            "I8": torch.int8,
+            "U8": torch.uint8,
+            "BOOL": torch.bool,
+        }
+        # add float8 types if available
+        if hasattr(torch, "float8_e5m2"):
+            dtype_map["F8_E5M2"] = torch.float8_e5m2
+        if hasattr(torch, "float8_e4m3fn"):
+            dtype_map["F8_E4M3"] = torch.float8_e4m3fn
+        return dtype_map.get(dtype_str)
+
+    @staticmethod
+    def _convert_float8(byte_tensor, dtype_str, shape):
+        if dtype_str == "F8_E5M2" and hasattr(torch, "float8_e5m2"):
+            return byte_tensor.view(torch.float8_e5m2).reshape(shape)
+        elif dtype_str == "F8_E4M3" and hasattr(torch, "float8_e4m3fn"):
+            return byte_tensor.view(torch.float8_e4m3fn).reshape(shape)
+        else:
+            # # convert to float16 if float8 is not supported
+            # print(f"Warning: {dtype_str} is not supported in this PyTorch version. Converting to float16.")
+            # return byte_tensor.view(torch.uint8).to(torch.float16).reshape(shape)
+            raise ValueError(f"Unsupported float8 type: {dtype_str} (upgrade PyTorch to support float8 types)")
+
+
+def mem_eff_save_file(tensors: Dict[str, torch.Tensor], filename: str, metadata: Dict[str, Any] = None):
+    _TYPES = {
+        torch.float64: "F64",
+        torch.float32: "F32",
+        torch.float16: "F16",
+        torch.bfloat16: "BF16",
+        torch.int64: "I64",
+        torch.int32: "I32",
+        torch.int16: "I16",
+        torch.int8: "I8",
+        torch.uint8: "U8",
+        torch.bool: "BOOL",
+        getattr(torch, "float8_e5m2", None): "F8_E5M2",
+        getattr(torch, "float8_e4m3fn", None): "F8_E4M3",
+    }
+    _ALIGN = 256
+
+    def validate_metadata(metadata: Dict[str, Any]) -> Dict[str, str]:
+        validated = {}
+        for key, value in metadata.items():
+            if not isinstance(key, str):
+                raise ValueError(f"Metadata key must be a string, got {type(key)}")
+            if not isinstance(value, str):
+                print(f"Warning: Metadata value for key '{key}' is not a string. Converting to string.")
+                validated[key] = str(value)
+            else:
+                validated[key] = value
+        return validated
+
+    header = {}
+    offset = 0
+    if metadata:
+        header["__metadata__"] = validate_metadata(metadata)
+    for k, v in tensors.items():
+        if v.numel() == 0:  # empty tensor
+            header[k] = {"dtype": _TYPES[v.dtype], "shape": list(v.shape), "data_offsets": [offset, offset]}
+        else:
+            size = v.numel() * v.element_size()
+            header[k] = {"dtype": _TYPES[v.dtype], "shape": list(v.shape), "data_offsets": [offset, offset + size]}
+            offset += size
+
+    hjson = json.dumps(header).encode("utf-8")
+    hjson += b" " * (-(len(hjson) + 8) % _ALIGN)
+
+    with open(filename, "wb") as f:
+        f.write(struct.pack("<Q", len(hjson)))
+        f.write(hjson)
+
+        for k, v in tensors.items():
+            if v.numel() == 0:
+                continue
+            if v.is_cuda:
+                # Direct GPU to disk save
+                with torch.cuda.device(v.device):
+                    if v.dim() == 0:  # if scalar, need to add a dimension to work with view
+                        v = v.unsqueeze(0)
+                    tensor_bytes = v.contiguous().view(torch.uint8)
+                    tensor_bytes.cpu().numpy().tofile(f)
+            else:
+                # CPU tensor save
+                if v.dim() == 0:  # if scalar, need to add a dimension to work with view
+                    v = v.unsqueeze(0)
+                v.contiguous().view(torch.uint8).numpy().tofile(f)
+
+
+# read safetensors metadata
+def read_safetensors_metadata(path: str):
+    with open(path, 'rb') as f:
+        header_size = int.from_bytes(f.read(8), 'little')
+        header_json = f.read(header_size).decode('utf-8')
+        header = json.loads(header_json)
+        metadata = header.get('__metadata__', {})
+        return metadata
+
+metadata = read_safetensors_metadata(path)
+print(json.dumps(metadata, indent=4)) #show metadata
+
+sd_pruned = dict() #initialize empty dict
+
+with MemoryEfficientSafeOpen(path) as reader:
+    keys = reader.keys()
+    for key in tqdm(keys): #for each key in the safetensors file
+        sd_pruned[key] = reader.get_tensor(key).to(torch.float8_e4m3fn) #convert to fp8
+
+# save the pruned safetensors file
+mem_eff_save_file(sd_pruned, output, metadata={"format": "pt", **metadata})

ACE_plus/pyproject.toml

@@ -0,0 +1,15 @@
+[project]
+name = "ace_plus"
+description = "Custom nodes for various visual generation and editing tasks using ACE_Plus FFT Model."
+version = "1.0.0"
+license = {file = "LICENSE"}
+dependencies = []
+
+[project.urls]
+Repository = "https://github.com/ali-vilab/ACE_plus"
+#  Used by Comfy Registry https://comfyregistry.org
+
+[tool.comfy]
+PublisherId = ""
+DisplayName = "ACE_plus"
+Icon = ""

ACE_plus/repo_requirements.txt

@@ -0,0 +1,8 @@
+huggingface_hub
+diffusers
+transformers
+torch>=2.4.1
+xformers>=0.0.27.post2
+gradio>=4.44.1
+scepter
+ms_swift

ACE_plus/run_train.py

@@ -0,0 +1,70 @@
+# -*- coding: utf-8 -*-
+# Copyright (c) Alibaba, Inc. and its affiliates.
+import argparse
+import importlib
+import os
+import sys
+from datetime import datetime
+sys.dont_write_bytecode = True
+from scepter.modules.solver.registry import SOLVERS
+from scepter.modules.utils.config import Config
+from scepter.modules.utils.distribute import we
+from scepter.modules.utils.file_system import FS
+from scepter.modules.utils.logger import get_logger
+
+if os.path.exists('__init__.py'):
+    package_name = 'scepter_ext'
+    spec = importlib.util.spec_from_file_location(package_name, '__init__.py')
+    package = importlib.util.module_from_spec(spec)
+    sys.modules[package_name] = package
+    spec.loader.exec_module(package)
+
+def run_task(cfg):
+    std_logger = get_logger(name='scepter')
+    solver = SOLVERS.build(cfg.SOLVER, logger=std_logger)
+    solver.set_up_pre()
+    solver.set_up()
+    if we.rank == 0:
+        FS.put_object_from_local_file(cfg.args.cfg_file, os.path.join(solver.work_dir, "train.yaml"))
+    if cfg.args.stage == "train":
+        solver.solve()
+    elif cfg.args.stage == "eval":
+        solver.run_eval()
+
+
+def update_config(cfg):
+    if hasattr(cfg.args, 'learning_rate') and cfg.args.learning_rate:
+        print(
+            f'learning_rate change from {cfg.SOLVER.OPTIMIZER.LEARNING_RATE} to {cfg.args.learning_rate}'
+        )
+        cfg.SOLVER.OPTIMIZER.LEARNING_RATE = float(cfg.args.learning_rate)
+    if hasattr(cfg.args, 'max_steps') and cfg.args.max_steps:
+        print(
+            f'max_steps change from {cfg.SOLVER.MAX_STEPS} to {cfg.args.max_steps}'
+        )
+        cfg.SOLVER.MAX_STEPS = int(cfg.args.max_steps)
+    cfg.SOLVER.WORK_DIR = os.path.join(cfg.SOLVER.WORK_DIR, "{0:%Y%m%d%H%M%S}".format(datetime.now()))
+    return cfg
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(description='Argparser for Scepter:\n')
+    parser.add_argument(
+        "--stage",
+        dest="stage",
+        help="Running stage!",
+        default="train",
+        choices=["train", "eval"]
+    )
+    parser.add_argument('--learning_rate',
+                        dest='learning_rate',
+                        help='The learning rate for our network!',
+                        default=None)
+    parser.add_argument('--max_steps',
+                        dest='max_steps',
+                        help='The max steps for training!',
+                        default=None)
+
+    cfg = Config(load=True, parser_ins=parser)
+    cfg = update_config(cfg)
+    we.init_env(cfg, logger=None, fn=run_task)

Reference_models/ae.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:afc8e28272cd15db3919bacdb6918ce9c1ed22e96cb12c4d5ed0fba823529e38
+size 335304388

Reference_models/clip_l/pytorch_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:660c6f5b1abae9dc498ac2d21e1347d2abdb0cf6c0c0c8576cd796491d9a6cdd
+size 246144152

Reference_models/comfyui_local_lora16.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a935ef5dd61e54e014375cd6ac05e1433a22bcfa8e7a04bbcc1fff17cf4a4873
+size 153268392

Reference_models/comfyui_subject_lora16.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dbb24ec6fe7d84514e22ca19fd571a1168fd107d7160dc7728a94777bf1a8707
+size 153268392

Reference_models/t5_xxl/config.json

@@ -0,0 +1,28 @@
+{
+  "_name_or_path": "./",
+  "architectures": [
+    "T5ForConditionalGeneration"
+  ],
+  "d_ff": 10240,
+  "d_kv": 64,
+  "d_model": 4096,
+  "decoder_start_token_id": 0,
+  "dropout_rate": 0.1,
+  "eos_token_id": 1,
+  "feed_forward_proj": "gated-gelu",
+  "initializer_factor": 1.0,
+  "is_encoder_decoder": true,
+  "layer_norm_epsilon": 1e-06,
+  "model_type": "t5",
+  "num_decoder_layers": 24,
+  "num_heads": 64,
+  "num_layers": 24,
+  "output_past": true,
+  "pad_token_id": 0,
+  "relative_attention_num_buckets": 32,
+  "tie_word_embeddings": false,
+  "torch_dtype": "float32",
+  "transformers_version": "4.13.0.dev0",
+  "use_cache": true,
+  "vocab_size": 32128
+}

Reference_models/t5_xxl/generation_config.json

@@ -0,0 +1,7 @@
+{
+  "_from_model_config": true,
+  "decoder_start_token_id": 0,
+  "eos_token_id": 1,
+  "pad_token_id": 0,
+  "transformers_version": "4.27.0.dev0"
+}

Reference_models/t5_xxl/special_tokens_map.json

@@ -0,0 +1 @@
+{"eos_token": "</s>", "unk_token": "<unk>", "pad_token": "<pad>", "additional_special_tokens": ["<extra_id_0>", "<extra_id_1>", "<extra_id_2>", "<extra_id_3>", "<extra_id_4>", "<extra_id_5>", "<extra_id_6>", "<extra_id_7>", "<extra_id_8>", "<extra_id_9>", "<extra_id_10>", "<extra_id_11>", "<extra_id_12>", "<extra_id_13>", "<extra_id_14>", "<extra_id_15>", "<extra_id_16>", "<extra_id_17>", "<extra_id_18>", "<extra_id_19>", "<extra_id_20>", "<extra_id_21>", "<extra_id_22>", "<extra_id_23>", "<extra_id_24>", "<extra_id_25>", "<extra_id_26>", "<extra_id_27>", "<extra_id_28>", "<extra_id_29>", "<extra_id_30>", "<extra_id_31>", "<extra_id_32>", "<extra_id_33>", "<extra_id_34>", "<extra_id_35>", "<extra_id_36>", "<extra_id_37>", "<extra_id_38>", "<extra_id_39>", "<extra_id_40>", "<extra_id_41>", "<extra_id_42>", "<extra_id_43>", "<extra_id_44>", "<extra_id_45>", "<extra_id_46>", "<extra_id_47>", "<extra_id_48>", "<extra_id_49>", "<extra_id_50>", "<extra_id_51>", "<extra_id_52>", "<extra_id_53>", "<extra_id_54>", "<extra_id_55>", "<extra_id_56>", "<extra_id_57>", "<extra_id_58>", "<extra_id_59>", "<extra_id_60>", "<extra_id_61>", "<extra_id_62>", "<extra_id_63>", "<extra_id_64>", "<extra_id_65>", "<extra_id_66>", "<extra_id_67>", "<extra_id_68>", "<extra_id_69>", "<extra_id_70>", "<extra_id_71>", "<extra_id_72>", "<extra_id_73>", "<extra_id_74>", "<extra_id_75>", "<extra_id_76>", "<extra_id_77>", "<extra_id_78>", "<extra_id_79>", "<extra_id_80>", "<extra_id_81>", "<extra_id_82>", "<extra_id_83>", "<extra_id_84>", "<extra_id_85>", "<extra_id_86>", "<extra_id_87>", "<extra_id_88>", "<extra_id_89>", "<extra_id_90>", "<extra_id_91>", "<extra_id_92>", "<extra_id_93>", "<extra_id_94>", "<extra_id_95>", "<extra_id_96>", "<extra_id_97>", "<extra_id_98>", "<extra_id_99>"]}

Reference_models/t5_xxl/spiece.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d60acb128cf7b7f2536e8f38a5b18a05535c9e14c7a355904270e15b0945ea86
+size 791656

Reference_models/t5_xxl/tokenizer_config.json

@@ -0,0 +1 @@
+{"eos_token": "</s>", "unk_token": "<unk>", "pad_token": "<pad>", "extra_ids": 100, "additional_special_tokens": ["<extra_id_0>", "<extra_id_1>", "<extra_id_2>", "<extra_id_3>", "<extra_id_4>", "<extra_id_5>", "<extra_id_6>", "<extra_id_7>", "<extra_id_8>", "<extra_id_9>", "<extra_id_10>", "<extra_id_11>", "<extra_id_12>", "<extra_id_13>", "<extra_id_14>", "<extra_id_15>", "<extra_id_16>", "<extra_id_17>", "<extra_id_18>", "<extra_id_19>", "<extra_id_20>", "<extra_id_21>", "<extra_id_22>", "<extra_id_23>", "<extra_id_24>", "<extra_id_25>", "<extra_id_26>", "<extra_id_27>", "<extra_id_28>", "<extra_id_29>", "<extra_id_30>", "<extra_id_31>", "<extra_id_32>", "<extra_id_33>", "<extra_id_34>", "<extra_id_35>", "<extra_id_36>", "<extra_id_37>", "<extra_id_38>", "<extra_id_39>", "<extra_id_40>", "<extra_id_41>", "<extra_id_42>", "<extra_id_43>", "<extra_id_44>", "<extra_id_45>", "<extra_id_46>", "<extra_id_47>", "<extra_id_48>", "<extra_id_49>", "<extra_id_50>", "<extra_id_51>", "<extra_id_52>", "<extra_id_53>", "<extra_id_54>", "<extra_id_55>", "<extra_id_56>", "<extra_id_57>", "<extra_id_58>", "<extra_id_59>", "<extra_id_60>", "<extra_id_61>", "<extra_id_62>", "<extra_id_63>", "<extra_id_64>", "<extra_id_65>", "<extra_id_66>", "<extra_id_67>", "<extra_id_68>", "<extra_id_69>", "<extra_id_70>", "<extra_id_71>", "<extra_id_72>", "<extra_id_73>", "<extra_id_74>", "<extra_id_75>", "<extra_id_76>", "<extra_id_77>", "<extra_id_78>", "<extra_id_79>", "<extra_id_80>", "<extra_id_81>", "<extra_id_82>", "<extra_id_83>", "<extra_id_84>", "<extra_id_85>", "<extra_id_86>", "<extra_id_87>", "<extra_id_88>", "<extra_id_89>", "<extra_id_90>", "<extra_id_91>", "<extra_id_92>", "<extra_id_93>", "<extra_id_94>", "<extra_id_95>", "<extra_id_96>", "<extra_id_97>", "<extra_id_98>", "<extra_id_99>"], "sp_model_kwargs": {}, "model_max_length": 512, "name_or_path": "./", "special_tokens_map_file": "/home/patrick/.cache/huggingface/transformers/5995b096c7f019eb696851d1af1c82271c3083fac6538e159d9b07ed718214d0.c94798918c92ded6aeef2d2f0e666d2cc4145eca1aa6e1336fde07f2e13e2f46", "tokenizer_class": "T5Tokenizer"}