observantdistressed commited on
Commit
bf209fb
·
1 Parent(s): 7119a13

Fix runtime LoRA loading on torchao-quantized pipe (bias + activation + cleanup)

Browse files

Three bugs seen in production logs after the peft 0.20.0 upgrade:

1. 'TorchaoLoraLinear does not support lora_bias yet, set it to False' —
diffusers auto-detects lora_bias=True purely from lora_B.*.bias keys in
the LoRA state dict (peft_utils.py), and peft's TorchaoLoraLinear
hard-rejects it on torchao-quantized layers. load_lora_to_pipe now loads
via the dict path (safetensors.torch.load_file) and strips lora_B bias
keys first, so such LoRAs (e.g. Ultimate Deepthroat) load successfully.

2. Adapter-name collision cascade ('Adapter name lora_0_high already in
use') — a partial failure left lora_loaded=False so unload_lora was
skipped and stale adapters leaked into the next task. load_lora_to_pipe
now unloads all adapters on failure and re-raises; app.py also unloads in
the except path, keeping adapter names reusable across tasks.

3. Latent multi-LoRA activation bug — each load_lora_weights leaves only the
last-loaded adapter active, so multiple auto-activated LoRAs were never
applied together (and a pair only had its LOW half active). load_lora_to_pipe
now returns the registered adapter names, routes HIGH->transformer and
LOW->transformer_2 (the Wan 2.2 high/low-noise stage split, mirroring the
validated startup load), and app.py calls pipe.set_adapters once with all
names at weight 1.0 so every selected LoRA is active simultaneously.

Files changed (2) hide show
  1. app.py +11 -2
  2. lora_loader.py +49 -14
app.py CHANGED
@@ -459,15 +459,24 @@ def run_inference(
459
  start = time.time()
460
 
461
  lora_loaded = False
 
462
  if lora_groups:
463
  try:
464
  for idx, name in enumerate(lora_groups):
465
  if name and name != "(None)":
466
- lora_loader.load_lora_to_pipe(pipe, name, adapter_name=f"lora_{idx}")
467
- lora_loaded = True
 
 
 
 
 
 
 
468
  print(f"LoRA loaded: {lora_groups}")
469
  except Exception as e:
470
  print(f"LoRA warning: {e}")
 
471
 
472
  # LoRA prompt + user prompt combined — lora triggers first for activation
473
  try:
 
459
  start = time.time()
460
 
461
  lora_loaded = False
462
+ lora_adapter_names = []
463
  if lora_groups:
464
  try:
465
  for idx, name in enumerate(lora_groups):
466
  if name and name != "(None)":
467
+ lora_adapter_names += lora_loader.load_lora_to_pipe(
468
+ pipe, name, adapter_name=f"lora_{idx}"
469
+ )
470
+ # Activate ALL loaded adapters together (HIGH in transformer, LOW in
471
+ # transformer_2); each load_lora_weights alone would leave only the
472
+ # last adapter active.
473
+ if lora_adapter_names:
474
+ pipe.set_adapters(lora_adapter_names, adapter_weights=[1.0] * len(lora_adapter_names))
475
+ lora_loaded = True
476
  print(f"LoRA loaded: {lora_groups}")
477
  except Exception as e:
478
  print(f"LoRA warning: {e}")
479
+ lora_loader.unload_lora(pipe)
480
 
481
  # LoRA prompt + user prompt combined — lora triggers first for activation
482
  try:
lora_loader.py CHANGED
@@ -14,6 +14,7 @@ import os
14
  import urllib.parse
15
  import re
16
  import math
 
17
  from huggingface_hub import hf_hub_download
18
 
19
  LORA_REPO = "lkzd7/WAN2.2_LoraSet_NSFW"
@@ -997,22 +998,56 @@ def download_lora(group_name):
997
  return high_path, low_path
998
 
999
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1000
  def load_lora_to_pipe(pipe, group_name, adapter_name="lora"):
 
 
 
 
 
 
 
 
1001
  high_path, low_path = download_lora(group_name)
1002
- if high_path and low_path:
1003
- pipe.load_lora_weights(high_path, adapter_name=f"{adapter_name}_high")
1004
- pipe.load_lora_weights(low_path, adapter_name=f"{adapter_name}_low")
1005
- print(f"Loaded LoRA pair: {group_name}")
1006
- return True
1007
- elif high_path:
1008
- pipe.load_lora_weights(high_path, adapter_name=adapter_name)
1009
- print(f"Loaded LoRA: {group_name}")
1010
- return True
1011
- elif low_path:
1012
- pipe.load_lora_weights(low_path, adapter_name=adapter_name)
1013
- print(f"Loaded LoRA (low): {group_name}")
1014
- return True
1015
- return False
1016
 
1017
 
1018
  def unload_lora(pipe):
 
14
  import urllib.parse
15
  import re
16
  import math
17
+ import safetensors.torch
18
  from huggingface_hub import hf_hub_download
19
 
20
  LORA_REPO = "lkzd7/WAN2.2_LoraSet_NSFW"
 
998
  return high_path, low_path
999
 
1000
 
1001
+ def _strip_lora_bias(state_dict):
1002
+ """Drop lora_B.*.bias keys that peft's TorchaoLoraLinear cannot support.
1003
+
1004
+ diffusers auto-detects lora_bias=True purely from the presence of
1005
+ "lora_B" ... ".bias" state-dict keys (src/diffusers/utils/peft_utils.py),
1006
+ and peft's TorchaoLoraLinear.__init__ hard-rejects lora_bias=True on
1007
+ torchao-quantized layers. Removing the keys makes lora_bias=False and the
1008
+ adapter loads fine; only the (small) learned bias delta is discarded.
1009
+ """
1010
+ return {
1011
+ k: v for k, v in state_dict.items()
1012
+ if not ("lora_B" in k and k.endswith(".bias"))
1013
+ }
1014
+
1015
+
1016
+ def _load_wan_lora(pipe, path, adapter_name, load_into_transformer_2=False):
1017
+ """Load a single Wan LoRA .safetensors via the dict path so we can strip
1018
+ lora_B bias keys first. HIGH files go to `transformer`, LOW files to
1019
+ `transformer_2` (the Wan 2.2 high/low-noise stage split), mirroring the
1020
+ validated startup load in app.py."""
1021
+ state_dict = safetensors.torch.load_file(path)
1022
+ state_dict = _strip_lora_bias(state_dict)
1023
+ kwargs = {"load_into_transformer_2": True} if load_into_transformer_2 else {}
1024
+ pipe.load_lora_weights(state_dict, adapter_name=adapter_name, **kwargs)
1025
+
1026
+
1027
  def load_lora_to_pipe(pipe, group_name, adapter_name="lora"):
1028
+ """Load a LoRA group into the pipe.
1029
+
1030
+ HIGH -> transformer, LOW -> transformer_2 (Wan 2.2 stage split). Returns
1031
+ the list of adapter names registered ([] if nothing was loaded). Callers
1032
+ must pipe.set_adapters(names, weights) afterwards to activate them. On any
1033
+ failure all adapters are unloaded (names become reusable) and the error
1034
+ re-raised.
1035
+ """
1036
  high_path, low_path = download_lora(group_name)
1037
+ if not (high_path or low_path):
1038
+ return []
1039
+ names = []
1040
+ try:
1041
+ if high_path:
1042
+ _load_wan_lora(pipe, high_path, f"{adapter_name}_high")
1043
+ names.append(f"{adapter_name}_high")
1044
+ if low_path:
1045
+ _load_wan_lora(pipe, low_path, f"{adapter_name}_low", load_into_transformer_2=True)
1046
+ names.append(f"{adapter_name}_low")
1047
+ return names
1048
+ except Exception:
1049
+ pipe.unload_lora_weights() # drop partial/ghost adapters; names reusable
1050
+ raise
1051
 
1052
 
1053
  def unload_lora(pipe):