Duplicate from DiffSynth-Studio/Qwen-Image-Edit-F2P

Co-authored-by: kelseye.xh <kelseye@users.noreply.huggingface.co>

.gitattributes

@@ -0,0 +1,38 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+assets/qwen_woman_0.jpg filter=lfs diff=lfs merge=lfs -text
+assets/qwen_woman_2.jpg filter=lfs diff=lfs merge=lfs -text
+assets/qwen_woman_3.jpg filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,104 @@
+---
+license: apache-2.0
+pipeline_tag: image-to-image
+library_name: diffusers
+tags:
+- lora
+---
+# Qwen-Image-Edit Face Generation Image Model
+## Model Introduction
+
+This model is based on the [Qwen-Image-Edit](https://www.modelscope.cn/models/Qwen/Qwen-Image-Edit) face-controlled image generation model. Given a cropped facial image as input, it generates full portrait images of the same person.
+
+## Result Demonstration
+
+|Face|Generated Image 1|Generated Image 2|Generated Image 3|Generated Image 4|
+|-|-|-|-|-|
+|![](./assets/qwen_woman_face_crop.png)|![](./assets/qwen_woman_0.jpg)|![](./assets/qwen_woman_1.jpg)|![](./assets/qwen_woman_2.jpg)|![](./assets/qwen_woman_3.jpg)|
+
+
+
+## Inference Code
+```
+git clone https://github.com/modelscope/DiffSynth-Studio.git  
+cd DiffSynth-Studio
+pip install -e .
+```
+
+```python
+from diffsynth.pipelines.qwen_image import QwenImagePipeline, ModelConfig
+import torch
+from modelscope import snapshot_download, dataset_snapshot_download
+from PIL import Image
+
+pipe = QwenImagePipeline.from_pretrained(
+    torch_dtype=torch.bfloat16,
+    device="cuda",
+    model_configs=[
+        ModelConfig(model_id="Qwen/Qwen-Image-Edit", origin_file_pattern="transformer/diffusion_pytorch_model*.safetensors"),
+        ModelConfig(model_id="Qwen/Qwen-Image", origin_file_pattern="text_encoder/model*.safetensors"),
+        ModelConfig(model_id="Qwen/Qwen-Image", origin_file_pattern="vae/diffusion_pytorch_model.safetensors"),
+    ],
+    tokenizer_config=None,
+    processor_config=ModelConfig(model_id="Qwen/Qwen-Image-Edit", origin_file_pattern="processor/"),
+)
+snapshot_download("DiffSynth-Studio/Qwen-Image-Edit-F2P", local_dir="models/DiffSynth-Studio/Qwen-Image-Edit-F2P", allow_file_pattern="model.safetensors")
+pipe.load_lora(pipe.dit, "models/DiffSynth-Studio/Qwen-Image-Edit-F2P/model.safetensors")
+dataset_snapshot_download(
+    dataset_id="DiffSynth-Studio/example_image_dataset",
+    local_dir="./data/example_image_dataset",
+    allow_file_pattern="f2p/qwen_woman_face_crop.png"
+)
+face_image = Image.open("data/example_image_dataset/f2p/qwen_woman_face_crop.png").convert("RGB")
+```
+
+```python
+prompt = "Photography. A young woman wearing a yellow dress stands in a flower field, with a background of colorful flowers and green grass."
+image = pipe(prompt, edit_image=face_image, seed=42, num_inference_steps=40, height=1152, width=864)
+image.save(f"image.jpg")
+```
+
+Face Auto-Cropping
+```python
+import torch
+from PIL import Image
+import numpy as np
+from insightface.app import FaceAnalysis
+import cv2
+
+class FaceDetector(torch.nn.Module):
+
+    def __init__(self):
+        super().__init__()
+        providers = ["CUDAExecutionProvider", "CPUExecutionProvider"]
+        provider_options = [{"device_id": 0}, {}]
+        self.app_640 = FaceAnalysis(name='antelopev2', providers=providers, provider_options=provider_options)
+        self.app_640.prepare(ctx_id=0, det_size=(640, 640))
+        self.app_320 = FaceAnalysis(name='antelopev2', providers=providers, provider_options=provider_options)
+        self.app_320.prepare(ctx_id=0, det_size=(320, 320))
+        self.app_160 = FaceAnalysis(name='antelopev2', providers=providers, provider_options=provider_options)
+        self.app_160.prepare(ctx_id=0, det_size=(160, 160))
+
+    def _detect_face(self, id_image_cv2):
+        face_info = self.app_640.get(id_image_cv2)
+        if len(face_info) > 0:
+            return face_info
+        face_info = self.app_320.get(id_image_cv2)
+        if len(face_info) > 0:
+            return face_info
+        face_info = self.app_160.get(id_image_cv2)
+        return face_info
+
+    def crop_face(self, id_image):
+        face_info = self._detect_face(cv2.cvtColor(np.array(id_image), cv2.COLOR_RGB2BGR))
+        if len(face_info) == 0:
+            return None
+        else:
+            bbox = sorted(face_info, key=lambda x: (x['bbox'][2] - x['bbox'][0]) * (x['bbox'][3] - x['bbox'][1]))[-1]['bbox']
+            return id_image.crop(list(map(int, bbox)))
+
+
+face_detector = FaceDetector()
+face_image = face_detector.crop_face(Image.open("image_2.jpg"))
+face_image.save("face_crop.jpg")
+```

README_from_modelscope.md

@@ -0,0 +1,107 @@
+---
+frameworks:
+- Pytorch
+license: Apache License 2.0
+tags: []
+tasks:
+- image-to-image
+base_model:
+  - Qwen/Qwen-Image-Edit
+base_model_relation: adapter
+---
+
+# Qwen-Image-Edit 人脸生成图像模型
+## 模型介绍
+
+本模型是基于 [Qwen-Image-Edit](https://www.modelscope.cn/models/Qwen/Qwen-Image-Edit) 人脸控制图像生成模型。输入裁剪下的人脸图像，输出该人的人像图片。
+
+## 效果展示
+
+|人脸|生成图1|生成图2|生成图3|生成图4|
+|-|-|-|-|-|
+|![](./assets/qwen_woman_face_crop.png)|![](./assets/qwen_woman_0.jpg)|![](./assets/qwen_woman_1.jpg)|![](./assets/qwen_woman_2.jpg)|![](./assets/qwen_woman_3.jpg)|
+
+
+
+## 推理代码
+```
+git clone https://github.com/modelscope/DiffSynth-Studio.git  
+cd DiffSynth-Studio
+pip install -e .
+```
+
+```python
+from diffsynth.pipelines.qwen_image import QwenImagePipeline, ModelConfig
+import torch
+from modelscope import snapshot_download, dataset_snapshot_download
+from PIL import Image
+
+pipe = QwenImagePipeline.from_pretrained(
+    torch_dtype=torch.bfloat16,
+    device="cuda",
+    model_configs=[
+        ModelConfig(model_id="Qwen/Qwen-Image-Edit", origin_file_pattern="transformer/diffusion_pytorch_model*.safetensors"),
+        ModelConfig(model_id="Qwen/Qwen-Image", origin_file_pattern="text_encoder/model*.safetensors"),
+        ModelConfig(model_id="Qwen/Qwen-Image", origin_file_pattern="vae/diffusion_pytorch_model.safetensors"),
+    ],
+    tokenizer_config=None,
+    processor_config=ModelConfig(model_id="Qwen/Qwen-Image-Edit", origin_file_pattern="processor/"),
+)
+snapshot_download("DiffSynth-Studio/Qwen-Image-Edit-F2P", local_dir="models/DiffSynth-Studio/Qwen-Image-Edit-F2P", allow_file_pattern="model.safetensors")
+pipe.load_lora(pipe.dit, "models/DiffSynth-Studio/Qwen-Image-Edit-F2P/model.safetensors")
+dataset_snapshot_download(
+    dataset_id="DiffSynth-Studio/example_image_dataset",
+    local_dir="./data/example_image_dataset",
+    allow_file_pattern="f2p/qwen_woman_face_crop.png"
+)
+face_image = Image.open("data/example_image_dataset/f2p/qwen_woman_face_crop.png").convert("RGB")
+
+prompt = "摄影。一个年轻女性穿着黄色连衣裙，站在花田中，背景是五颜六色的花朵和绿色的草地。"
+image = pipe(prompt, edit_image=face_image, seed=42, num_inference_steps=40, height=1152, width=864)
+image.save(f"image.jpg")
+```
+人脸自动裁剪
+```python
+import torch
+from PIL import Image
+import numpy as np
+from insightface.app import FaceAnalysis
+import cv2
+
+class FaceDetector(torch.nn.Module):
+
+    def __init__(self):
+        super().__init__()
+        providers = ["CUDAExecutionProvider", "CPUExecutionProvider"]
+        provider_options = [{"device_id": 0}, {}]
+        self.app_640 = FaceAnalysis(name='antelopev2', providers=providers, provider_options=provider_options)
+        self.app_640.prepare(ctx_id=0, det_size=(640, 640))
+        self.app_320 = FaceAnalysis(name='antelopev2', providers=providers, provider_options=provider_options)
+        self.app_320.prepare(ctx_id=0, det_size=(320, 320))
+        self.app_160 = FaceAnalysis(name='antelopev2', providers=providers, provider_options=provider_options)
+        self.app_160.prepare(ctx_id=0, det_size=(160, 160))
+
+    def _detect_face(self, id_image_cv2):
+        face_info = self.app_640.get(id_image_cv2)
+        if len(face_info) > 0:
+            return face_info
+        face_info = self.app_320.get(id_image_cv2)
+        if len(face_info) > 0:
+            return face_info
+        face_info = self.app_160.get(id_image_cv2)
+        return face_info
+
+    def crop_face(self, id_image):
+        face_info = self._detect_face(cv2.cvtColor(np.array(id_image), cv2.COLOR_RGB2BGR))
+        if len(face_info) == 0:
+            return None
+        else:
+            bbox = sorted(face_info, key=lambda x: (x['bbox'][2] - x['bbox'][0]) * (x['bbox'][3] - x['bbox'][1]))[-1]['bbox']
+            return id_image.crop(list(map(int, bbox)))
+
+
+face_detector = FaceDetector()
+face_image = face_detector.crop_face(Image.open("image_2.jpg"))
+face_image.save("face_crop.jpg")
+
+```

configuration.json

@@ -0,0 +1 @@
+{"framework":"Pytorch","task":"image-to-image"}

edit_0917.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3eaec9705c770f3453a802ff317f5095c806cc27b069b760015f71bd26179f2e
+size 472047152

edit_0922_lora_step13000.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:da99840137731e62a6cd74f9b98e42da2d62ec7011927f199b8d9bb2ba7ed23f
+size 472047184

edit_0928_lora_step40000.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2766a84f5d01b14b172c0307ef69c077fe9c399d304359eadb5fa5819594a175
+size 472047184