File size: 5,323 Bytes
2037b5c
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
"""Build a Klein snapshot whose DiT is FLUX.2 Klein 1-step RDM (epfl-vita/flux2-klein-1step-rdm, Apache-2.0).

RDM's model.safetensors holds the DiT in BFL's native key layout under the training adapter's `model.` prefix; the
Klein package loads diffusers' layout. This maps BFL -> diffusers (qkv split into q/k/v, the final adaLN's
(shift, scale) halves swapped to diffusers' (scale, shift)), then proves the mapping: RDM is a full fine-tune of
Klein 4B, so every mapped tensor must stay close to the Klein tensor of the same name (cosine reported; a wrong
split or a missed swap shows up as a low cosine).

text_encoder / tokenizer / vae / scheduler / model_index are symlinks to the shipped mlx-community Klein 4B bf16
snapshot.

usage: .venv/bin/python -I make_rdm_snapshot.py <rdm model.safetensors> <klein snapshot> <out dir>
"""
import os
import re
import sys

import torch
from safetensors import safe_open
from safetensors.torch import save_file

FIXED = {
    "img_in.weight": "x_embedder.weight",
    "txt_in.weight": "context_embedder.weight",
    "time_in.in_layer.weight": "time_guidance_embed.timestep_embedder.linear_1.weight",
    "time_in.out_layer.weight": "time_guidance_embed.timestep_embedder.linear_2.weight",
    "double_stream_modulation_img.lin.weight": "double_stream_modulation_img.linear.weight",
    "double_stream_modulation_txt.lin.weight": "double_stream_modulation_txt.linear.weight",
    "single_stream_modulation.lin.weight": "single_stream_modulation.linear.weight",
    "final_layer.linear.weight": "proj_out.weight",
}
DOUBLE = {
    "img_attn.norm.query_norm.scale": "attn.norm_q.weight",
    "img_attn.norm.key_norm.scale": "attn.norm_k.weight",
    "img_attn.proj.weight": "attn.to_out.0.weight",
    "img_mlp.0.weight": "ff.linear_in.weight",
    "img_mlp.2.weight": "ff.linear_out.weight",
    "txt_attn.norm.query_norm.scale": "attn.norm_added_q.weight",
    "txt_attn.norm.key_norm.scale": "attn.norm_added_k.weight",
    "txt_attn.proj.weight": "attn.to_add_out.weight",
    "txt_mlp.0.weight": "ff_context.linear_in.weight",
    "txt_mlp.2.weight": "ff_context.linear_out.weight",
}
SINGLE = {
    "linear1.weight": "attn.to_qkv_mlp_proj.weight",
    "linear2.weight": "attn.to_out.weight",
    "norm.query_norm.scale": "attn.norm_q.weight",
    "norm.key_norm.scale": "attn.norm_k.weight",
}


def convert(bfl):
    out = {}
    for k, v in bfl.items():
        if k in FIXED:
            out[FIXED[k]] = v
        elif k.startswith("final_layer.adaLN_modulation.") and k.endswith(".weight"):
            shift, scale = v.chunk(2, dim=0)
            out["norm_out.linear.weight"] = torch.cat([scale, shift], 0)
        elif m := re.match(r"double_blocks\.(\d+)\.(.+)", k):
            i, rest = m.groups()
            p = f"transformer_blocks.{i}."
            if rest == "img_attn.qkv.weight":
                for n, t in zip(("to_q", "to_k", "to_v"), v.chunk(3, 0)):
                    out[p + f"attn.{n}.weight"] = t
            elif rest == "txt_attn.qkv.weight":
                for n, t in zip(("add_q_proj", "add_k_proj", "add_v_proj"), v.chunk(3, 0)):
                    out[p + f"attn.{n}.weight"] = t
            else:
                out[p + DOUBLE[rest]] = v
        elif m := re.match(r"single_blocks\.(\d+)\.(.+)", k):
            i, rest = m.groups()
            out[f"single_transformer_blocks.{i}." + SINGLE[rest]] = v
        else:
            raise KeyError(k)
    return out


def main():
    rdm, klein, out = sys.argv[1:4]
    os.makedirs(os.path.join(out, "transformer"), exist_ok=True)
    for name in ("text_encoder", "tokenizer", "vae", "scheduler", "model_index.json"):
        dst = os.path.join(out, name)
        if not os.path.lexists(dst):
            os.symlink(os.path.join(klein, name), dst)
    with safe_open(rdm, "pt") as f:
        bfl = {k[len("model."):]: f.get_tensor(k) for k in f.keys() if k.startswith("model.")}
    sd = convert(bfl)
    ref_path = os.path.join(klein, "transformer", "diffusion_pytorch_model.safetensors")
    worst = []
    with safe_open(ref_path, "pt") as f:
        ref_keys = set(f.keys())
        missing, extra = sorted(ref_keys - set(sd)), sorted(set(sd) - ref_keys)
        print(f"klein keys {len(ref_keys)}  mapped {len(sd)}  missing {len(missing)}  extra {len(extra)}")
        if missing or extra:
            print("missing", missing[:5], "\nextra", extra[:5])
            sys.exit(1)
        for k in sorted(ref_keys):
            r = f.get_tensor(k).float().flatten()
            v = sd[k].float().flatten()
            assert r.shape == v.shape, (k, r.shape, v.shape)
            worst.append((torch.nn.functional.cosine_similarity(r, v, dim=0).item(), k))
    worst.sort()
    print("lowest cosines vs Klein 4B:", [(round(c, 4), k) for c, k in worst[:6]])
    print(f"median cosine {worst[len(worst) // 2][0]:.4f}")
    if worst[0][0] < 0.5:
        print("a mapped tensor is far from Klein's: mapping wrong")
        sys.exit(1)
    save_file({k: v.contiguous() for k, v in sd.items()}, os.path.join(out, "transformer", "diffusion_pytorch_model.safetensors"))
    cfg = os.path.join(out, "transformer", "config.json")
    if not os.path.lexists(cfg):
        os.symlink(os.path.join(klein, "transformer", "config.json"), cfg)
    print("wrote", out)


if __name__ == "__main__":
    main()