mpasila commited on
Commit
b3babf9
·
verified ·
1 Parent(s): e7c3a50

feat(lora): add ComfyUI MiniMax-H3 LoRA conversion

Browse files

Implement automatic remapping of ComfyUI-format MiniMax-H3 Turbo LoRAs to diffusers module names.

- Added `_is_comfyui_lora` to detect ComfyUI naming conventions.
- Added `_convert_comfyui_lora` to handle:
- Block prefix remapping (`blocks` -> `transformer_blocks`).
- Token refiner path updates.
- Splitting fused QKV projections into separate `to_q`, `to_k`, and `to_v` weights.
- Remapping MLP and output projection keys.
- Integrated conversion into `apply_loras` to support Turbo LoRAs without manual conversion.

Files changed (1) hide show
  1. app.py +72 -1
app.py CHANGED
@@ -349,11 +349,80 @@ def _lora_prefix(state_dict) -> str | None:
349
  return None
350
 
351
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
352
  def apply_loras(transformer, loras) -> list[str]:
353
  """Attach `loras` (local path, strength) to `transformer` and give each its strength, replacing whatever was on it.
354
 
355
  Every adapter already on the model is removed first, so a request is never affected by the one before it — which
356
- matters when a worker is reused rather than forked fresh.
 
357
  """
358
  import torch
359
 
@@ -365,6 +434,8 @@ def apply_loras(transformer, loras) -> list[str]:
365
  names, scales = [], []
366
  for index, (path, scale) in enumerate(loras):
367
  state_dict = load_file(path)
 
 
368
  name = f"lora{index}"
369
  transformer.load_lora_adapter(state_dict, adapter_name=name, prefix=_lora_prefix(state_dict))
370
  names.append(name)
 
349
  return None
350
 
351
 
352
+ def _is_comfyui_lora(state_dict) -> bool:
353
+ """Whether a LoRA state dict is in ComfyUI's MiniMax-H3 naming rather than diffusers'.
354
+
355
+ ComfyUI names the block stack `blocks.N.*` and the token refiner `token_refiner.blocks.N.*`; diffusers names them
356
+ `transformer_blocks.N.*` and `token_refiner.refiner_blocks.N.*`. A key starting with `blocks.` is the tell.
357
+ """
358
+ for key in state_dict:
359
+ if key.startswith(("blocks.", "token_refiner.blocks.", "final_layer.")):
360
+ return True
361
+ return False
362
+
363
+
364
+ def _convert_comfyui_lora(state_dict) -> dict:
365
+ """Remap a ComfyUI-format MiniMax-H3 Turbo LoRA to the diffusers `transformer_ref` module names.
366
+
367
+ The Turbo LoRA ([`larryvrh/MiniMax-H3-Turbo-Lora`](https://huggingface.co/larryvrh/MiniMax-H3-Turbo-Lora)) is trained
368
+ against the ComfyUI checkpoint, whose module names differ from diffusers' in four ways:
369
+
370
+ * the block stack is `blocks.N` in ComfyUI but `transformer_blocks.N` in diffusers,
371
+ * the token refiner is `token_refiner.blocks.N` but `token_refiner.refiner_blocks.N`,
372
+ * the final AdaLN is `final_layer.adaln_proj.linear` but `norm_out.linear`,
373
+ * attention QKV is one fused `attn.qkv_proj` in ComfyUI but three separate `attn.to_q` / `to_k` / `to_v` in
374
+ diffusers, and the output projection is `attn.out_proj` but `attn.to_out.0`,
375
+ * the feed-forward is `mlp.fc1` / `mlp.fc2` but `ff.fc1` / `ff.fc2`.
376
+
377
+ The fused QKV `lora_B` is `[3 * inner_dim, rank]`; splitting it into three along dim 0 gives the three separate
378
+ `lora_B` matrices, and `lora_A` (which is `[rank, hidden_size]`) is shared verbatim across the three. The metadata
379
+ says `W_eff = W + lora_B @ lora_A` with alpha = rank, so the scaling is 1.0 and no alpha key is added.
380
+ """
381
+ import torch
382
+
383
+ converted = {}
384
+ for key, value in state_dict.items():
385
+ # `blocks.N.` -> `transformer_blocks.N.`
386
+ if key.startswith("blocks."):
387
+ new_key = "transformer_blocks." + key[len("blocks."):]
388
+ elif key.startswith("token_refiner.blocks."):
389
+ new_key = "token_refiner.refiner_blocks." + key[len("token_refiner.blocks."):]
390
+ elif key.startswith("final_layer.adaln_proj.linear."):
391
+ new_key = "norm_out.linear." + key[len("final_layer.adaln_proj.linear."):]
392
+ else:
393
+ converted[key] = value
394
+ continue
395
+
396
+ # At this point `new_key` is a diffusers block path. Remap the leaf module names.
397
+ if ".attn.qkv_proj." in new_key:
398
+ # Fused QKV: split `lora_B` along dim 0 into q/k/v, duplicate `lora_A` verbatim.
399
+ leaf = new_key.split(".attn.qkv_proj.")[-1] # `lora_A.weight` or `lora_B.weight`
400
+ stem = new_key[: new_key.index(".attn.qkv_proj.")]
401
+ if leaf == "lora_A.weight":
402
+ for proj in ("to_q", "to_k", "to_v"):
403
+ converted[f"{stem}.attn.{proj}.lora_A.weight"] = value
404
+ else: # lora_B.weight
405
+ q_b, k_b, v_b = value.chunk(3, dim=0)
406
+ converted[f"{stem}.attn.to_q.lora_B.weight"] = q_b
407
+ converted[f"{stem}.attn.to_k.lora_B.weight"] = k_b
408
+ converted[f"{stem}.attn.to_v.lora_B.weight"] = v_b
409
+ elif ".attn.out_proj." in new_key:
410
+ converted[new_key.replace(".attn.out_proj.", ".attn.to_out.0.")] = value
411
+ elif ".mlp." in new_key:
412
+ converted[new_key.replace(".mlp.", ".ff.")] = value
413
+ else:
414
+ # `adaln_proj.linear` and the token refiner's attention/ff already match diffusers' names after the
415
+ # block-prefix rename above.
416
+ converted[new_key] = value
417
+ return converted
418
+
419
+
420
  def apply_loras(transformer, loras) -> list[str]:
421
  """Attach `loras` (local path, strength) to `transformer` and give each its strength, replacing whatever was on it.
422
 
423
  Every adapter already on the model is removed first, so a request is never affected by the one before it — which
424
+ matters when a worker is reused rather than forked fresh. A LoRA in ComfyUI's MiniMax-H3 naming is remapped to
425
+ diffusers' module names on the fly, so the Turbo LoRA works without a separate conversion step.
426
  """
427
  import torch
428
 
 
434
  names, scales = [], []
435
  for index, (path, scale) in enumerate(loras):
436
  state_dict = load_file(path)
437
+ if _is_comfyui_lora(state_dict):
438
+ state_dict = _convert_comfyui_lora(state_dict)
439
  name = f"lora{index}"
440
  transformer.load_lora_adapter(state_dict, adapter_name=name, prefix=_lora_prefix(state_dict))
441
  names.append(name)