"""Convert IFM/PAN2 (https://huggingface.co/IFM/PAN2) into single-file models for ComfyUI. Makes up to six files under --out: diffusion_models/PAN2_int8_convrot.safetensors 39.8 GiB int8 ConvRot, the fp32 norm layers kept as fp32 diffusion_models/PAN2_int8_convrot_full.safetensors 27.4 GiB int8 ConvRot, those layers quantized as well diffusion_models/PAN2_nvfp4.safetensors 15.5 GiB NVFP4 (4-bit) text_encoders/PAN2_qwen3_5_int8_convrot.safetensors 8.4 GiB Qwen3.5-9B, int8 ConvRot, embedding table bf16 text_encoders/PAN2_qwen3_5_nvfp4.safetensors 4.6 GiB Qwen3.5-9B, NVFP4, embedding table plain int8 vae/PAN2_vae_fp16.safetensors 1.3 GiB the fp32 VAE cast to fp16 Usage: python convert_pan2.py --out D:/ComfyUI/models # download what is needed, make all six python convert_pan2.py --out D:/ComfyUI/models --only transformer_nvfp4 te_nvfp4 vae_fp16 python convert_pan2.py --out D:/ComfyUI/models --src D:/PAN2 # a local copy with transformer/, text_encoder/, vae/ Needs: torch, safetensors, numpy, and huggingface_hub when --src is not given. Formats written (what ComfyUI's quantized-weight loader reads): int8 ConvRot .weight int8 [out, in] = round((W @ H) / scale), H = block Hadamard of size 256 .weight_scale fp32 [out, 1], one scale per output row .comfy_quant json {"format": "int8_tensorwise", "convrot": true, "convrot_groupsize": 256} NVFP4 .weight uint8 [out, in/2], two 4-bit e2m1 codes per byte .weight_scale fp8 e4m3 [out, in/16], one scale per block of 16, in cuBLAS block order .weight_scale_2 fp32 scalar .comfy_quant json {"format": "nvfp4"} Every other tensor is copied unchanged. Files are written tensor by tensor, one tensor at a time. """ import argparse import collections import glob import json import math import os import re import struct import time import torch from safetensors import safe_open REPO = "IFM/PAN2" GROUP = 256 ST_DTYPE = {torch.float32: "F32", torch.bfloat16: "BF16", torch.float16: "F16", torch.int8: "I8", torch.uint8: "U8"} ITEM = {"F8_E4M3": 1, "F32": 4, "BF16": 2, "F16": 2, "I8": 1, "U8": 1, "F64": 8, "I64": 8, "I32": 4} F4_E2M1_MAX, F8_E4M3_MAX = 6.0, 448.0 E2M1_VALUES = torch.tensor([0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0]) TRANSFORMER = {"folder": "transformer", "source": "IFM/PAN2 transformer", "prefixes": ("transformer_blocks.", "context_refiner.")} TEXT_ENCODER = {"folder": "text_encoder", "source": "IFM/PAN2 text_encoder", "prefixes": ("layers.",)} TARGETS = { "transformer_int8_convrot": dict(TRANSFORMER, out="diffusion_models/PAN2_int8_convrot.safetensors", format="int8_convrot"), "transformer_int8_convrot_full": dict(TRANSFORMER, out="diffusion_models/PAN2_int8_convrot_full.safetensors", format="int8_convrot", quant_f32=True), "transformer_nvfp4": dict(TRANSFORMER, out="diffusion_models/PAN2_nvfp4.safetensors", format="nvfp4", quant_f32=True), "te_int8_convrot": dict(TEXT_ENCODER, out="text_encoders/PAN2_qwen3_5_int8_convrot.safetensors", format="int8_convrot"), # the text encoder's matrix products stay in full precision, and its embedding table (a lookup, not a product) # is stored as plain row-wise int8 "te_nvfp4": dict(TEXT_ENCODER, out="text_encoders/PAN2_qwen3_5_nvfp4.safetensors", format="nvfp4", embed_int8="embed_tokens.weight", full_precision_mm=True), "vae_fp16": {"folder": "vae", "out": "vae/PAN2_vae_fp16.safetensors", "format": "fp16"}, } # ---------------------------------------------------------------- int8 ConvRot def hadamard(size): h4 = torch.tensor([[1, 1, 1, -1], [1, 1, -1, 1], [1, -1, 1, 1], [-1, 1, 1, 1]], dtype=torch.float32) h, cur = h4, 4 while cur < size: h = torch.kron(h, h4) cur *= 4 return h / (size ** 0.5) def quantize_int8_convrot(w, h): w = w.float() out_f, in_f = w.shape rot = (w.reshape(out_f, in_f // GROUP, GROUP) @ h.T).reshape(out_f, in_f) scale = (rot.abs().amax(dim=1, keepdim=True) / 127.0).clamp(min=1e-12) q = torch.round(rot / scale).clamp(-128, 127).to(torch.int8) err = ((q.float() * scale - rot).norm() / rot.norm().clamp(min=1e-30)).item() return q, scale, err # ---------------------------------------------------------------- NVFP4 def _ceil_div(a, b): return -(-a // b) def to_blocked(m): """Block scales in the order cuBLAS expects (128x4 tiles, each stored as 32x16).""" rows, cols = m.shape nr, nc = _ceil_div(rows, 128), _ceil_div(cols, 4) pr, pc = nr * 128, nc * 4 if (rows, cols) != (pr, pc): padded = torch.zeros((pr, pc), device=m.device, dtype=m.dtype) padded[:rows, :cols] = m m = padded blocks = m.view(nr, 128, nc, 4).permute(0, 2, 1, 3) return blocks.reshape(-1, 4, 32, 4).transpose(1, 2).reshape(-1, 32, 16).reshape(pr, pc) def from_blocked(b, rows, cols): nr, nc = _ceil_div(rows, 128), _ceil_div(cols, 4) pr, pc = nr * 128, nc * 4 s = b.reshape(-1, 32, 16).reshape(-1, 32, 4, 4).transpose(1, 2) s = s.reshape(nr, nc, 4, 32, 4).reshape(nr, nc, 128, 4).permute(0, 2, 1, 3) return s.reshape(pr, pc)[:rows, :cols] def _e2m1_codes(x): """fp32 values already scaled to [-6, 6] -> 4-bit codes (sign in bit 3), round to nearest, ties to even.""" sign = (x < 0).to(torch.uint8) a = x.abs() vals = E2M1_VALUES.to(x.device) mids = (vals[:-1] + vals[1:]) / 2 code = torch.bucketize(a, mids, right=False).to(torch.uint8) tie = torch.isin(a, mids) code = torch.where(tie & (code % 2 == 1), code + 1, code).clamp(max=7) return code | (sign << 3) def quantize_nvfp4(w): """Returns (packed uint8 [rows, cols/2], fp8 block scales in cuBLAS order, fp32 per-tensor scale).""" w = w.float() rows, cols = w.shape pr, pc = _ceil_div(rows, 16) * 16, _ceil_div(cols, 16) * 16 if (pr, pc) != (rows, cols): w = torch.nn.functional.pad(w, (0, pc - cols, 0, pr - rows)) per_tensor = (w.abs().amax() / (F8_E4M3_MAX * F4_E2M1_MAX)).to(torch.float32) x = w.reshape(pr, -1, 16) block = x.abs().amax(dim=-1) / F4_E2M1_MAX scaled_fp8 = (block / per_tensor).clamp(max=F8_E4M3_MAX).to(torch.float8_e4m3fn) total = per_tensor * scaled_fp8.float() safe = torch.where(total == 0, torch.ones_like(total), total) data = (x / safe.unsqueeze(-1)).clamp(-F4_E2M1_MAX, F4_E2M1_MAX) data = torch.where((total == 0).unsqueeze(-1), torch.zeros_like(data), data).reshape(pr, pc) codes = _e2m1_codes(data).reshape(-1) packed = (codes[::2] << 4 | codes[1::2]).reshape(pr, pc // 2) return packed, to_blocked(scaled_fp8), per_tensor def dequantize_nvfp4(packed, block_scales, per_tensor, shape): pr, pc = packed.shape[0], packed.shape[1] * 2 lut = torch.cat([E2M1_VALUES, -E2M1_VALUES]).to(packed.device) codes = torch.stack([packed >> 4, packed & 0x0F], dim=-1).reshape(pr, pc) vals = lut[codes.int()].reshape(pr, pc // 16, 16) scales = from_blocked(block_scales, pr, pc // 16).float() * per_tensor.float() return (vals * scales.unsqueeze(-1)).reshape(pr, pc)[: shape[0], : shape[1]] # ---------------------------------------------------------------- converters def eligible(name, dtype, shape, quant_f32, prefixes): """Large 2-D weights of the blocks. The transformer keeps some of them in fp32 (the norm1.linear layers).""" if not name.endswith(".weight") or len(shape) != 2 or min(shape) < 1024 or shape[1] % GROUP: return False if not name.startswith(prefixes): return False return dtype == "BF16" or (quant_f32 and dtype == "F32") def convert_quantized(src, dst, fmt, prefixes, source, quant_f32=False, embed_int8=None, full_precision_mm=False, device="cpu", **_): shards = sorted(glob.glob(os.path.join(src, "*.safetensors"))) assert shards, f"no .safetensors files in {src}" where, info = {}, {} for p in shards: with open(p, "rb") as f: n = struct.unpack(" {dst}") folder = os.path.join(src, t["folder"]) if t["format"] == "fp16": convert_vae(folder, dst) else: convert_quantized(folder, dst, t["format"], device=args.device, **{k: v for k, v in t.items() if k != "format"}) if __name__ == "__main__": main()