Spaces:
Running on Zero
Fix runtime LoRA loading on torchao-quantized pipe (bias + activation + cleanup)
Browse filesThree bugs seen in production logs after the peft 0.20.0 upgrade:
1. 'TorchaoLoraLinear does not support lora_bias yet, set it to False' —
diffusers auto-detects lora_bias=True purely from lora_B.*.bias keys in
the LoRA state dict (peft_utils.py), and peft's TorchaoLoraLinear
hard-rejects it on torchao-quantized layers. load_lora_to_pipe now loads
via the dict path (safetensors.torch.load_file) and strips lora_B bias
keys first, so such LoRAs (e.g. Ultimate Deepthroat) load successfully.
2. Adapter-name collision cascade ('Adapter name lora_0_high already in
use') — a partial failure left lora_loaded=False so unload_lora was
skipped and stale adapters leaked into the next task. load_lora_to_pipe
now unloads all adapters on failure and re-raises; app.py also unloads in
the except path, keeping adapter names reusable across tasks.
3. Latent multi-LoRA activation bug — each load_lora_weights leaves only the
last-loaded adapter active, so multiple auto-activated LoRAs were never
applied together (and a pair only had its LOW half active). load_lora_to_pipe
now returns the registered adapter names, routes HIGH->transformer and
LOW->transformer_2 (the Wan 2.2 high/low-noise stage split, mirroring the
validated startup load), and app.py calls pipe.set_adapters once with all
names at weight 1.0 so every selected LoRA is active simultaneously.
- app.py +11 -2
- lora_loader.py +49 -14
|
@@ -459,15 +459,24 @@ def run_inference(
|
|
| 459 |
start = time.time()
|
| 460 |
|
| 461 |
lora_loaded = False
|
|
|
|
| 462 |
if lora_groups:
|
| 463 |
try:
|
| 464 |
for idx, name in enumerate(lora_groups):
|
| 465 |
if name and name != "(None)":
|
| 466 |
-
lora_loader.load_lora_to_pipe(
|
| 467 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 468 |
print(f"LoRA loaded: {lora_groups}")
|
| 469 |
except Exception as e:
|
| 470 |
print(f"LoRA warning: {e}")
|
|
|
|
| 471 |
|
| 472 |
# LoRA prompt + user prompt combined — lora triggers first for activation
|
| 473 |
try:
|
|
|
|
| 459 |
start = time.time()
|
| 460 |
|
| 461 |
lora_loaded = False
|
| 462 |
+
lora_adapter_names = []
|
| 463 |
if lora_groups:
|
| 464 |
try:
|
| 465 |
for idx, name in enumerate(lora_groups):
|
| 466 |
if name and name != "(None)":
|
| 467 |
+
lora_adapter_names += lora_loader.load_lora_to_pipe(
|
| 468 |
+
pipe, name, adapter_name=f"lora_{idx}"
|
| 469 |
+
)
|
| 470 |
+
# Activate ALL loaded adapters together (HIGH in transformer, LOW in
|
| 471 |
+
# transformer_2); each load_lora_weights alone would leave only the
|
| 472 |
+
# last adapter active.
|
| 473 |
+
if lora_adapter_names:
|
| 474 |
+
pipe.set_adapters(lora_adapter_names, adapter_weights=[1.0] * len(lora_adapter_names))
|
| 475 |
+
lora_loaded = True
|
| 476 |
print(f"LoRA loaded: {lora_groups}")
|
| 477 |
except Exception as e:
|
| 478 |
print(f"LoRA warning: {e}")
|
| 479 |
+
lora_loader.unload_lora(pipe)
|
| 480 |
|
| 481 |
# LoRA prompt + user prompt combined — lora triggers first for activation
|
| 482 |
try:
|
|
@@ -14,6 +14,7 @@ import os
|
|
| 14 |
import urllib.parse
|
| 15 |
import re
|
| 16 |
import math
|
|
|
|
| 17 |
from huggingface_hub import hf_hub_download
|
| 18 |
|
| 19 |
LORA_REPO = "lkzd7/WAN2.2_LoraSet_NSFW"
|
|
@@ -997,22 +998,56 @@ def download_lora(group_name):
|
|
| 997 |
return high_path, low_path
|
| 998 |
|
| 999 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1000 |
def load_lora_to_pipe(pipe, group_name, adapter_name="lora"):
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1001 |
high_path, low_path = download_lora(group_name)
|
| 1002 |
-
if high_path
|
| 1003 |
-
|
| 1004 |
-
|
| 1005 |
-
|
| 1006 |
-
|
| 1007 |
-
|
| 1008 |
-
|
| 1009 |
-
|
| 1010 |
-
|
| 1011 |
-
|
| 1012 |
-
|
| 1013 |
-
|
| 1014 |
-
|
| 1015 |
-
|
| 1016 |
|
| 1017 |
|
| 1018 |
def unload_lora(pipe):
|
|
|
|
| 14 |
import urllib.parse
|
| 15 |
import re
|
| 16 |
import math
|
| 17 |
+
import safetensors.torch
|
| 18 |
from huggingface_hub import hf_hub_download
|
| 19 |
|
| 20 |
LORA_REPO = "lkzd7/WAN2.2_LoraSet_NSFW"
|
|
|
|
| 998 |
return high_path, low_path
|
| 999 |
|
| 1000 |
|
| 1001 |
+
def _strip_lora_bias(state_dict):
|
| 1002 |
+
"""Drop lora_B.*.bias keys that peft's TorchaoLoraLinear cannot support.
|
| 1003 |
+
|
| 1004 |
+
diffusers auto-detects lora_bias=True purely from the presence of
|
| 1005 |
+
"lora_B" ... ".bias" state-dict keys (src/diffusers/utils/peft_utils.py),
|
| 1006 |
+
and peft's TorchaoLoraLinear.__init__ hard-rejects lora_bias=True on
|
| 1007 |
+
torchao-quantized layers. Removing the keys makes lora_bias=False and the
|
| 1008 |
+
adapter loads fine; only the (small) learned bias delta is discarded.
|
| 1009 |
+
"""
|
| 1010 |
+
return {
|
| 1011 |
+
k: v for k, v in state_dict.items()
|
| 1012 |
+
if not ("lora_B" in k and k.endswith(".bias"))
|
| 1013 |
+
}
|
| 1014 |
+
|
| 1015 |
+
|
| 1016 |
+
def _load_wan_lora(pipe, path, adapter_name, load_into_transformer_2=False):
|
| 1017 |
+
"""Load a single Wan LoRA .safetensors via the dict path so we can strip
|
| 1018 |
+
lora_B bias keys first. HIGH files go to `transformer`, LOW files to
|
| 1019 |
+
`transformer_2` (the Wan 2.2 high/low-noise stage split), mirroring the
|
| 1020 |
+
validated startup load in app.py."""
|
| 1021 |
+
state_dict = safetensors.torch.load_file(path)
|
| 1022 |
+
state_dict = _strip_lora_bias(state_dict)
|
| 1023 |
+
kwargs = {"load_into_transformer_2": True} if load_into_transformer_2 else {}
|
| 1024 |
+
pipe.load_lora_weights(state_dict, adapter_name=adapter_name, **kwargs)
|
| 1025 |
+
|
| 1026 |
+
|
| 1027 |
def load_lora_to_pipe(pipe, group_name, adapter_name="lora"):
|
| 1028 |
+
"""Load a LoRA group into the pipe.
|
| 1029 |
+
|
| 1030 |
+
HIGH -> transformer, LOW -> transformer_2 (Wan 2.2 stage split). Returns
|
| 1031 |
+
the list of adapter names registered ([] if nothing was loaded). Callers
|
| 1032 |
+
must pipe.set_adapters(names, weights) afterwards to activate them. On any
|
| 1033 |
+
failure all adapters are unloaded (names become reusable) and the error
|
| 1034 |
+
re-raised.
|
| 1035 |
+
"""
|
| 1036 |
high_path, low_path = download_lora(group_name)
|
| 1037 |
+
if not (high_path or low_path):
|
| 1038 |
+
return []
|
| 1039 |
+
names = []
|
| 1040 |
+
try:
|
| 1041 |
+
if high_path:
|
| 1042 |
+
_load_wan_lora(pipe, high_path, f"{adapter_name}_high")
|
| 1043 |
+
names.append(f"{adapter_name}_high")
|
| 1044 |
+
if low_path:
|
| 1045 |
+
_load_wan_lora(pipe, low_path, f"{adapter_name}_low", load_into_transformer_2=True)
|
| 1046 |
+
names.append(f"{adapter_name}_low")
|
| 1047 |
+
return names
|
| 1048 |
+
except Exception:
|
| 1049 |
+
pipe.unload_lora_weights() # drop partial/ghost adapters; names reusable
|
| 1050 |
+
raise
|
| 1051 |
|
| 1052 |
|
| 1053 |
def unload_lora(pipe):
|