Update pipeline.py

pipeline.py

@@ -17,6 +17,7 @@
 # Here is the AGPL-3.0 license https://github.com/AUTOMATIC1111/stable-diffusion-webui/blob/master/LICENSE.txt
 
 import inspect
+from pathlib import Path
 from typing import Any, Callable, Dict, List, Optional, Union
 
 import paddle
@@ -30,18 +31,25 @@ from ppdiffusers.pipelines.stable_diffusion.safety_checker import (
     StableDiffusionSafetyChecker,
 )
 from ppdiffusers.schedulers import KarrasDiffusionSchedulers
-from ppdiffusers.utils import logging, randn_tensor, safetensors_load, torch_load, smart_load
-
-from pathlib import Path
+from ppdiffusers.utils import (
+    logging,
+    randn_tensor,
+    safetensors_load,
+    smart_load,
+    torch_load,
+)
 
 logger = logging.get_logger(__name__)  # pylint: disable=invalid-name
 
+
 @paddle.no_grad()
-def load_lora(pipeline,
-                 state_dict: dict,
-                 LORA_PREFIX_UNET: str = "lora_unet",
-                 LORA_PREFIX_TEXT_ENCODER: str = "lora_te",
-                 ratio: float = 1.0):
+def load_lora(
+    pipeline,
+    state_dict: dict,
+    LORA_PREFIX_UNET: str = "lora_unet",
+    LORA_PREFIX_TEXT_ENCODER: str = "lora_te",
+    ratio: float = 1.0,
+):
     ratio = float(ratio)
     visited = []
     for key in state_dict:
@@ -49,8 +57,7 @@ def load_lora(pipeline,
             continue
 
         if "text" in key:
-            tmp_layer_infos = key.split(".")[0].split(
-                LORA_PREFIX_TEXT_ENCODER + "_")[-1].split("_")
+            tmp_layer_infos = key.split(".")[0].split(LORA_PREFIX_TEXT_ENCODER + "_")[-1].split("_")
             hf_to_ppnlp = {
                 "encoder": "transformer",
                 "fc1": "linear1",
@@ -58,12 +65,12 @@ def load_lora(pipeline,
             }
             layer_infos = []
             for layer_info in tmp_layer_infos:
-                if layer_info == "mlp": continue
+                if layer_info == "mlp":
+                    continue
                 layer_infos.append(hf_to_ppnlp.get(layer_info, layer_info))
             curr_layer: paddle.nn.Linear = pipeline.text_encoder
         else:
-            layer_infos = key.split(".")[0].split(LORA_PREFIX_UNET +
-                                                  "_")[-1].split("_")
+            layer_infos = key.split(".")[0].split(LORA_PREFIX_UNET + "_")[-1].split("_")
             curr_layer: paddle.nn.Linear = pipeline.unet
 
         temp_name = layer_infos.pop(0)
@@ -82,14 +89,9 @@ def load_lora(pipeline,
                 else:
                     temp_name = layer_infos.pop(0)
 
-        triplet_keys = [
-            key,
-            key.replace("lora_down", "lora_up"),
-            key.replace("lora_down.weight", "alpha")
-        ]
+        triplet_keys = [key, key.replace("lora_down", "lora_up"), key.replace("lora_down.weight", "alpha")]
         dtype: paddle.dtype = curr_layer.weight.dtype
-        weight_down: paddle.Tensor = state_dict[triplet_keys[0]].cast(
-            dtype)
+        weight_down: paddle.Tensor = state_dict[triplet_keys[0]].cast(dtype)
         weight_up: paddle.Tensor = state_dict[triplet_keys[1]].cast(dtype)
         rank: float = float(weight_down.shape[0])
         if triplet_keys[2] in state_dict:
@@ -100,31 +102,37 @@ def load_lora(pipeline,
 
         if not hasattr(curr_layer, "backup_weights"):
             curr_layer.backup_weights = curr_layer.weight.clone()
-            
+
         if len(weight_down.shape) == 4:
             if weight_down.shape[2:4] == [1, 1]:
                 # conv2d 1x1
                 curr_layer.weight.copy_(
-                    curr_layer.weight +
-                    ratio * paddle.matmul(weight_up.squeeze(
-                        [-1, -2]), weight_down.squeeze([-1, -2])).unsqueeze(
-                            [-1, -2]) * scale, True)
+                    curr_layer.weight
+                    + ratio
+                    * paddle.matmul(weight_up.squeeze([-1, -2]), weight_down.squeeze([-1, -2])).unsqueeze([-1, -2])
+                    * scale,
+                    True,
+                )
             else:
                 # conv2d 3x3
                 curr_layer.weight.copy_(
-                    curr_layer.weight + ratio * paddle.nn.functional.conv2d(
-                        weight_down.transpose([1, 0, 2, 3]),
-                        weight_up).transpose([1, 0, 2, 3]) * scale, True)
+                    curr_layer.weight
+                    + ratio
+                    * paddle.nn.functional.conv2d(weight_down.transpose([1, 0, 2, 3]), weight_up).transpose(
+                        [1, 0, 2, 3]
+                    )
+                    * scale,
+                    True,
+                )
         else:
             # linear
-            curr_layer.weight.copy_(
-                curr_layer.weight +
-                ratio * paddle.matmul(weight_up, weight_down).T * scale, True)
+            curr_layer.weight.copy_(curr_layer.weight + ratio * paddle.matmul(weight_up, weight_down).T * scale, True)
 
         # update visited list
         visited.extend(triplet_keys)
     return pipeline
 
+
 class WebUIStableDiffusionPipeline(DiffusionPipeline):
     r"""
     Pipeline for text-to-image generation using Stable Diffusion.
@@ -399,7 +407,7 @@ class WebUIStableDiffusionPipeline(DiffusionPipeline):
         callback: Optional[Callable[[int, int, paddle.Tensor], None]] = None,
         callback_steps: Optional[int] = 1,
         cross_attention_kwargs: Optional[Dict[str, Any]] = None,
-        clip_skip: int = 0,
+        clip_skip: int = 1,
         lora_dir: str = "./loras",
     ):
         r"""
@@ -452,7 +460,9 @@ class WebUIStableDiffusionPipeline(DiffusionPipeline):
                 `self.processor` in
                 [diffusers.cross_attention](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/cross_attention.py).
             clip_skip (`int`, *optional*, defaults to 0):
-                CLIP_stop_at_last_layers, if clip_skip < 1, we will use the last_hidden_state from text_encoder.
+                CLIP_stop_at_last_layers, if clip_skip <= 1, we will use the last_hidden_state from text_encoder.
+            lora_dir (`str`, *optional*):
+                Path to lora which we want to load.
         Examples:
 
         Returns:
@@ -554,7 +564,9 @@ class WebUIStableDiffusionPipeline(DiffusionPipeline):
                             cross_attention_kwargs=cross_attention_kwargs,
                         ).sample
                         noise_pred_uncond, noise_pred_text = noise_pred.chunk(2)
-                        noise_pred = noise_pred_uncond + weight * guidance_scale * (noise_pred_text - noise_pred_uncond)
+                        noise_pred = noise_pred_uncond + weight * guidance_scale * (
+                            noise_pred_text - noise_pred_uncond
+                        )
                     else:
                         noise_pred = self.unet(
                             latent_model_input,
@@ -616,6 +628,7 @@ class WebUIStableDiffusionPipeline(DiffusionPipeline):
                         sub_layer.weight.copy_(sub_layer.backup_weights, True)
                 self.weights_has_changed = False
 
+
 # clip.py
 import math
 from collections import namedtuple