FLUX.2-klein-4B-1step-RDM-bf16 / make_rdm_snapshot.py
xocialize's picture
FLUX.2 klein 4B 1-step RDM (epfl-vita, Apache-2.0) as a diffusers tree for the Swift/MLX port
2037b5c verified
Raw History Blame Contribute Delete
5.32 kB
"""Build a Klein snapshot whose DiT is FLUX.2 Klein 1-step RDM (epfl-vita/flux2-klein-1step-rdm, Apache-2.0).
RDM's model.safetensors holds the DiT in BFL's native key layout under the training adapter's `model.` prefix; the
Klein package loads diffusers' layout. This maps BFL -> diffusers (qkv split into q/k/v, the final adaLN's
(shift, scale) halves swapped to diffusers' (scale, shift)), then proves the mapping: RDM is a full fine-tune of
Klein 4B, so every mapped tensor must stay close to the Klein tensor of the same name (cosine reported; a wrong
split or a missed swap shows up as a low cosine).
text_encoder / tokenizer / vae / scheduler / model_index are symlinks to the shipped mlx-community Klein 4B bf16
snapshot.
usage: .venv/bin/python -I make_rdm_snapshot.py <rdm model.safetensors> <klein snapshot> <out dir>
"""
import os
import re
import sys
import torch
from safetensors import safe_open
from safetensors.torch import save_file
FIXED = {
"img_in.weight": "x_embedder.weight",
"txt_in.weight": "context_embedder.weight",
"time_in.in_layer.weight": "time_guidance_embed.timestep_embedder.linear_1.weight",
"time_in.out_layer.weight": "time_guidance_embed.timestep_embedder.linear_2.weight",
"double_stream_modulation_img.lin.weight": "double_stream_modulation_img.linear.weight",
"double_stream_modulation_txt.lin.weight": "double_stream_modulation_txt.linear.weight",
"single_stream_modulation.lin.weight": "single_stream_modulation.linear.weight",
"final_layer.linear.weight": "proj_out.weight",
}
DOUBLE = {
"img_attn.norm.query_norm.scale": "attn.norm_q.weight",
"img_attn.norm.key_norm.scale": "attn.norm_k.weight",
"img_attn.proj.weight": "attn.to_out.0.weight",
"img_mlp.0.weight": "ff.linear_in.weight",
"img_mlp.2.weight": "ff.linear_out.weight",
"txt_attn.norm.query_norm.scale": "attn.norm_added_q.weight",
"txt_attn.norm.key_norm.scale": "attn.norm_added_k.weight",
"txt_attn.proj.weight": "attn.to_add_out.weight",
"txt_mlp.0.weight": "ff_context.linear_in.weight",
"txt_mlp.2.weight": "ff_context.linear_out.weight",
}
SINGLE = {
"linear1.weight": "attn.to_qkv_mlp_proj.weight",
"linear2.weight": "attn.to_out.weight",
"norm.query_norm.scale": "attn.norm_q.weight",
"norm.key_norm.scale": "attn.norm_k.weight",
}
def convert(bfl):
out = {}
for k, v in bfl.items():
if k in FIXED:
out[FIXED[k]] = v
elif k.startswith("final_layer.adaLN_modulation.") and k.endswith(".weight"):
shift, scale = v.chunk(2, dim=0)
out["norm_out.linear.weight"] = torch.cat([scale, shift], 0)
elif m := re.match(r"double_blocks\.(\d+)\.(.+)", k):
i, rest = m.groups()
p = f"transformer_blocks.{i}."
if rest == "img_attn.qkv.weight":
for n, t in zip(("to_q", "to_k", "to_v"), v.chunk(3, 0)):
out[p + f"attn.{n}.weight"] = t
elif rest == "txt_attn.qkv.weight":
for n, t in zip(("add_q_proj", "add_k_proj", "add_v_proj"), v.chunk(3, 0)):
out[p + f"attn.{n}.weight"] = t
else:
out[p + DOUBLE[rest]] = v
elif m := re.match(r"single_blocks\.(\d+)\.(.+)", k):
i, rest = m.groups()
out[f"single_transformer_blocks.{i}." + SINGLE[rest]] = v
else:
raise KeyError(k)
return out
def main():
rdm, klein, out = sys.argv[1:4]
os.makedirs(os.path.join(out, "transformer"), exist_ok=True)
for name in ("text_encoder", "tokenizer", "vae", "scheduler", "model_index.json"):
dst = os.path.join(out, name)
if not os.path.lexists(dst):
os.symlink(os.path.join(klein, name), dst)
with safe_open(rdm, "pt") as f:
bfl = {k[len("model."):]: f.get_tensor(k) for k in f.keys() if k.startswith("model.")}
sd = convert(bfl)
ref_path = os.path.join(klein, "transformer", "diffusion_pytorch_model.safetensors")
worst = []
with safe_open(ref_path, "pt") as f:
ref_keys = set(f.keys())
missing, extra = sorted(ref_keys - set(sd)), sorted(set(sd) - ref_keys)
print(f"klein keys {len(ref_keys)} mapped {len(sd)} missing {len(missing)} extra {len(extra)}")
if missing or extra:
print("missing", missing[:5], "\nextra", extra[:5])
sys.exit(1)
for k in sorted(ref_keys):
r = f.get_tensor(k).float().flatten()
v = sd[k].float().flatten()
assert r.shape == v.shape, (k, r.shape, v.shape)
worst.append((torch.nn.functional.cosine_similarity(r, v, dim=0).item(), k))
worst.sort()
print("lowest cosines vs Klein 4B:", [(round(c, 4), k) for c, k in worst[:6]])
print(f"median cosine {worst[len(worst) // 2][0]:.4f}")
if worst[0][0] < 0.5:
print("a mapped tensor is far from Klein's: mapping wrong")
sys.exit(1)
save_file({k: v.contiguous() for k, v in sd.items()}, os.path.join(out, "transformer", "diffusion_pytorch_model.safetensors"))
cfg = os.path.join(out, "transformer", "config.json")
if not os.path.lexists(cfg):
os.symlink(os.path.join(klein, "transformer", "config.json"), cfg)
print("wrote", out)
if __name__ == "__main__":
main()