Spaces:
Running on Zero
Running on Zero
feat(lora): add ComfyUI MiniMax-H3 LoRA conversion
Browse filesImplement automatic remapping of ComfyUI-format MiniMax-H3 Turbo LoRAs to diffusers module names.
- Added `_is_comfyui_lora` to detect ComfyUI naming conventions.
- Added `_convert_comfyui_lora` to handle:
- Block prefix remapping (`blocks` -> `transformer_blocks`).
- Token refiner path updates.
- Splitting fused QKV projections into separate `to_q`, `to_k`, and `to_v` weights.
- Remapping MLP and output projection keys.
- Integrated conversion into `apply_loras` to support Turbo LoRAs without manual conversion.
app.py
CHANGED
|
@@ -349,11 +349,80 @@ def _lora_prefix(state_dict) -> str | None:
|
|
| 349 |
return None
|
| 350 |
|
| 351 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 352 |
def apply_loras(transformer, loras) -> list[str]:
|
| 353 |
"""Attach `loras` (local path, strength) to `transformer` and give each its strength, replacing whatever was on it.
|
| 354 |
|
| 355 |
Every adapter already on the model is removed first, so a request is never affected by the one before it — which
|
| 356 |
-
matters when a worker is reused rather than forked fresh.
|
|
|
|
| 357 |
"""
|
| 358 |
import torch
|
| 359 |
|
|
@@ -365,6 +434,8 @@ def apply_loras(transformer, loras) -> list[str]:
|
|
| 365 |
names, scales = [], []
|
| 366 |
for index, (path, scale) in enumerate(loras):
|
| 367 |
state_dict = load_file(path)
|
|
|
|
|
|
|
| 368 |
name = f"lora{index}"
|
| 369 |
transformer.load_lora_adapter(state_dict, adapter_name=name, prefix=_lora_prefix(state_dict))
|
| 370 |
names.append(name)
|
|
|
|
| 349 |
return None
|
| 350 |
|
| 351 |
|
| 352 |
+
def _is_comfyui_lora(state_dict) -> bool:
|
| 353 |
+
"""Whether a LoRA state dict is in ComfyUI's MiniMax-H3 naming rather than diffusers'.
|
| 354 |
+
|
| 355 |
+
ComfyUI names the block stack `blocks.N.*` and the token refiner `token_refiner.blocks.N.*`; diffusers names them
|
| 356 |
+
`transformer_blocks.N.*` and `token_refiner.refiner_blocks.N.*`. A key starting with `blocks.` is the tell.
|
| 357 |
+
"""
|
| 358 |
+
for key in state_dict:
|
| 359 |
+
if key.startswith(("blocks.", "token_refiner.blocks.", "final_layer.")):
|
| 360 |
+
return True
|
| 361 |
+
return False
|
| 362 |
+
|
| 363 |
+
|
| 364 |
+
def _convert_comfyui_lora(state_dict) -> dict:
|
| 365 |
+
"""Remap a ComfyUI-format MiniMax-H3 Turbo LoRA to the diffusers `transformer_ref` module names.
|
| 366 |
+
|
| 367 |
+
The Turbo LoRA ([`larryvrh/MiniMax-H3-Turbo-Lora`](https://huggingface.co/larryvrh/MiniMax-H3-Turbo-Lora)) is trained
|
| 368 |
+
against the ComfyUI checkpoint, whose module names differ from diffusers' in four ways:
|
| 369 |
+
|
| 370 |
+
* the block stack is `blocks.N` in ComfyUI but `transformer_blocks.N` in diffusers,
|
| 371 |
+
* the token refiner is `token_refiner.blocks.N` but `token_refiner.refiner_blocks.N`,
|
| 372 |
+
* the final AdaLN is `final_layer.adaln_proj.linear` but `norm_out.linear`,
|
| 373 |
+
* attention QKV is one fused `attn.qkv_proj` in ComfyUI but three separate `attn.to_q` / `to_k` / `to_v` in
|
| 374 |
+
diffusers, and the output projection is `attn.out_proj` but `attn.to_out.0`,
|
| 375 |
+
* the feed-forward is `mlp.fc1` / `mlp.fc2` but `ff.fc1` / `ff.fc2`.
|
| 376 |
+
|
| 377 |
+
The fused QKV `lora_B` is `[3 * inner_dim, rank]`; splitting it into three along dim 0 gives the three separate
|
| 378 |
+
`lora_B` matrices, and `lora_A` (which is `[rank, hidden_size]`) is shared verbatim across the three. The metadata
|
| 379 |
+
says `W_eff = W + lora_B @ lora_A` with alpha = rank, so the scaling is 1.0 and no alpha key is added.
|
| 380 |
+
"""
|
| 381 |
+
import torch
|
| 382 |
+
|
| 383 |
+
converted = {}
|
| 384 |
+
for key, value in state_dict.items():
|
| 385 |
+
# `blocks.N.` -> `transformer_blocks.N.`
|
| 386 |
+
if key.startswith("blocks."):
|
| 387 |
+
new_key = "transformer_blocks." + key[len("blocks."):]
|
| 388 |
+
elif key.startswith("token_refiner.blocks."):
|
| 389 |
+
new_key = "token_refiner.refiner_blocks." + key[len("token_refiner.blocks."):]
|
| 390 |
+
elif key.startswith("final_layer.adaln_proj.linear."):
|
| 391 |
+
new_key = "norm_out.linear." + key[len("final_layer.adaln_proj.linear."):]
|
| 392 |
+
else:
|
| 393 |
+
converted[key] = value
|
| 394 |
+
continue
|
| 395 |
+
|
| 396 |
+
# At this point `new_key` is a diffusers block path. Remap the leaf module names.
|
| 397 |
+
if ".attn.qkv_proj." in new_key:
|
| 398 |
+
# Fused QKV: split `lora_B` along dim 0 into q/k/v, duplicate `lora_A` verbatim.
|
| 399 |
+
leaf = new_key.split(".attn.qkv_proj.")[-1] # `lora_A.weight` or `lora_B.weight`
|
| 400 |
+
stem = new_key[: new_key.index(".attn.qkv_proj.")]
|
| 401 |
+
if leaf == "lora_A.weight":
|
| 402 |
+
for proj in ("to_q", "to_k", "to_v"):
|
| 403 |
+
converted[f"{stem}.attn.{proj}.lora_A.weight"] = value
|
| 404 |
+
else: # lora_B.weight
|
| 405 |
+
q_b, k_b, v_b = value.chunk(3, dim=0)
|
| 406 |
+
converted[f"{stem}.attn.to_q.lora_B.weight"] = q_b
|
| 407 |
+
converted[f"{stem}.attn.to_k.lora_B.weight"] = k_b
|
| 408 |
+
converted[f"{stem}.attn.to_v.lora_B.weight"] = v_b
|
| 409 |
+
elif ".attn.out_proj." in new_key:
|
| 410 |
+
converted[new_key.replace(".attn.out_proj.", ".attn.to_out.0.")] = value
|
| 411 |
+
elif ".mlp." in new_key:
|
| 412 |
+
converted[new_key.replace(".mlp.", ".ff.")] = value
|
| 413 |
+
else:
|
| 414 |
+
# `adaln_proj.linear` and the token refiner's attention/ff already match diffusers' names after the
|
| 415 |
+
# block-prefix rename above.
|
| 416 |
+
converted[new_key] = value
|
| 417 |
+
return converted
|
| 418 |
+
|
| 419 |
+
|
| 420 |
def apply_loras(transformer, loras) -> list[str]:
|
| 421 |
"""Attach `loras` (local path, strength) to `transformer` and give each its strength, replacing whatever was on it.
|
| 422 |
|
| 423 |
Every adapter already on the model is removed first, so a request is never affected by the one before it — which
|
| 424 |
+
matters when a worker is reused rather than forked fresh. A LoRA in ComfyUI's MiniMax-H3 naming is remapped to
|
| 425 |
+
diffusers' module names on the fly, so the Turbo LoRA works without a separate conversion step.
|
| 426 |
"""
|
| 427 |
import torch
|
| 428 |
|
|
|
|
| 434 |
names, scales = [], []
|
| 435 |
for index, (path, scale) in enumerate(loras):
|
| 436 |
state_dict = load_file(path)
|
| 437 |
+
if _is_comfyui_lora(state_dict):
|
| 438 |
+
state_dict = _convert_comfyui_lora(state_dict)
|
| 439 |
name = f"lora{index}"
|
| 440 |
transformer.load_lora_adapter(state_dict, adapter_name=name, prefix=_lora_prefix(state_dict))
|
| 441 |
names.append(name)
|