# Copyright (C) 2025 the HuggingFace Inc. team. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from __future__ import annotations from collections import defaultdict from copy import deepcopy from typing import TYPE_CHECKING from .core_model_loading import ( Chunk, Concatenate, ErnieFuseAndSplitTextVisionExperts, GroupWeightRename, Inkling2IndexOffsetWeightRename, Interleave, MergeModulelist, PermuteForRope, PrefixChange, Split, Transpose, WeightConverter, WeightRenaming, WeightTransform, ) if TYPE_CHECKING: from .modeling_utils import PreTrainedModel from .quantizers import HfQuantizer _MODEL_TO_CONVERSION_PATTERN = { # Mixtral-style MoE "minimax": "mixtral", "minimax_m2": "mixtral", # Qwen2-style MoE "Dots3NoteTextModel": "qwen2_moe", "afmoe": "qwen2_moe", "deepseek_v2": "qwen2_moe", "deepseek_v3": "qwen2_moe", "deepseek_v32": "qwen2_moe", "dots1": "qwen2_moe", "ernie4_5_moe": "qwen2_moe", "glm4_moe": "qwen2_moe", "glm4_moe_lite": "qwen2_moe", "glm_moe_dsa": "qwen2_moe", "glm4v_moe": "qwen2_moe", "longcat_flash": "qwen2_moe", "solar_open": "qwen2_moe", "mellum": "qwen2_moe", "qwen3_moe": "qwen2_moe", "beam": "qwen2_moe", "qwen3_omni_moe": "qwen2_moe", "qwen3_omni_moe_thinker": "qwen2_moe", "qwen3_omni_moe_text": "qwen2_moe", "qwen3_next": "qwen2_moe", "hunyuan_v1_moe": "qwen2_moe", "flex_olmo": "qwen2_moe", "olmoe": "qwen2_moe", "exaone_moe": "qwen2_moe", "cohere2_moe": "qwen2_moe", "rt_detr_v2": "rt_detr", "pp_doclayout_v2": "rt_detr", "pp_doclayout_v3": "rt_detr", "sam3_tracker_video": "sam3_tracker", "AltCLIPVisionModel": "CLIPVisionModel", "ChineseCLIPVisionModel": "CLIPVisionModel", "CLIPSegVisionModel": "CLIPVisionModel", "MetaClip2VisionModel": "CLIPVisionModel", "mlcd_vision": "CLIPVisionModel", "mlcd": "CLIPVisionModel", "SiglipVisionModel": "CLIPVisionModel", "Siglip2VisionModel": "CLIPVisionModel", "xCLIPVisionModel": "CLIPVisionModel", "CLIPSegTextModel": "CLIPTextModel", "MetaClip2TextModel": "CLIPTextModel", "SiglipTextModel": "CLIPTextModel", "Siglip2TextModel": "CLIPTextModel", "xCLIPTextModel": "CLIPTextModel", "aria": "llava", "paligemma": "llava", "aya_vision": "llava", "got_ocr2": "llava", "gemma3": "llava", "internvl": "llava", "vipllava": "llava", "mistral3": "llava", "pp_chart2table": "llava", "voxtral": "qwen2_audio", "voxtral_realtime": "qwen2_audio", "audioflamingo3": "qwen2_audio", "glmasr": "qwen2_audio", "musicflamingo": "qwen2_audio", "granite_speech_plus": "granite_speech", "granitemoeshared": "granitemoe", "granitemoehybrid": "granitemoe", "gemma3n_text": "qwen3_5_text", "glm5_next_text": "glm5_next", "hy_v4_vl_text": "hy_v4", "qwen3_5_moe_text": "qwen3_5_text", "llava_next_video": "llava_next", "llava_onevision": "llava_next", # class-based mappings "AriaModel": "LlavaModel", "PaliGemmaModel": "LlavaModel", "AyaVisionModel": "LlavaModel", "GotOcr2Model": "LlavaModel", "Gemma3Model": "LlavaModel", "InternVLModel": "LlavaModel", "VipLlavaModel": "LlavaModel", "Mistral3Model": "LlavaModel", "PPChart2TableModel": "LlavaModel", "LlavaNextModel": "LlavaModel", "LlavaNextVideoModel": "LlavaModel", "LlavaOnevisionModel": "LlavaModel", "FuyuModel": "LlavaModel", "MllamaModel": "LlavaModel", "VoxtralModel": "Qwen2AudioModel", "VoxtralRealtimeModel": "Qwen2AudioModel", "AudioFlamingo3Model": "Qwen2AudioModel", "GlmAsrModel": "Qwen2AudioModel", "MusicFlamingoModel": "Qwen2AudioModel", "GraniteSpeechPlusModel": "GraniteSpeechModel", "MaskFormerDetrDecoder": "DetrModel", "Qwen2_5_VLModel": "Qwen2VLModel", "Qwen2_5_VLForConditionalGeneration": "Qwen2VLForConditionalGeneration", "qwen2_5_omni_thinker": "Qwen2_5OmniThinkerForConditionalGeneration", "Qwen3OmniMoeThinkerModel": "Qwen2_5OmniThinkerModel", "Qwen3OmniMoeThinkerForConditionalGeneration": "Qwen2_5OmniThinkerForConditionalGeneration", "Tipsv2VisionBackbone": "Tipsv2VisionModel", "UnlimitedOcrTextModel": "qwen2_moe", # ViT-style vision models (old HuggingFace checkpoint format → new modular format) "ASTModel": "ViTModel", "BeitModel": "ViTModel", "DeiTModel": "ViTModel", "IJepaModel": "ViTModel", "ViTMAEModel": "ViTModel", "ViTMSNModel": "ViTModel", "VivitModel": "ViTModel", "Dinov2Backbone": "Dinov2Model", "Dinov2WithRegistersBackbone": "Dinov2WithRegistersModel", } def _build_checkpoint_conversion_mapping(): mapping = { "hy_v4": [ # General HC prefix which is dropped WeightRenaming(r"\.hc_pre\.hc_", ".hc_"), # Attn gating + sinks WeightRenaming(r"\.learnable_sink_param", ".sinks"), WeightRenaming(r"\.linear_gate", ".gate_proj"), # Follow DSv4 HC standards WeightRenaming(r"\.hc_attn_layer\.hc_fn", ".attn_hc.fn"), WeightRenaming(r"\.hc_attn_layer\.hc_base", ".attn_hc.base"), WeightRenaming(r"\.hc_attn_layer\.hc_scale", ".attn_hc.scale"), WeightRenaming(r"\.hc_mlp_layer\.hc_fn", ".ffn_hc.fn"), WeightRenaming(r"\.hc_mlp_layer\.hc_base", ".ffn_hc.base"), WeightRenaming(r"\.hc_mlp_layer\.hc_scale", ".ffn_hc.scale"), WeightRenaming(r"\.hc_head_fn", ".hc_fn"), WeightRenaming(r"\.hc_head_base", ".hc_base"), WeightRenaming(r"\.hc_head_scale", ".hc_scale"), ], # HY4-VL: text renames come from the scoped `hy_v4_vl_text` mapping after module placement. # Vision renames follow `hunyuan_vl`, with only `before_rms` and no image begin/end/sep embeddings. "hy_v4_vl": [ # -- vision tower: Megatron-style MLP and pre/post norm names -- WeightRenaming(source_patterns="mlp.dense_h_to_4h", target_patterns="mlp.fc1"), WeightRenaming(source_patterns="mlp.dense_4h_to_h", target_patterns="mlp.fc2"), WeightRenaming( source_patterns=r"^vit\.layers\.(\d+)\.input_layernorm\.", target_patterns=r"vit.layers.\1.layer_norm1.", ), WeightRenaming( source_patterns=r"^vit\.layers\.(\d+)\.post_attention_layernorm\.", target_patterns=r"vit.layers.\1.layer_norm2.", ), # -- projector: normalize the RMS name and give the convolutions explicit module names -- WeightRenaming(source_patterns=r"(perceive|patch_merger)\.norm\.", target_patterns=r"\1.before_rms."), WeightRenaming(source_patterns=r"perceive\.proj\.0\.", target_patterns="perceive.proj_conv."), WeightRenaming(source_patterns=r"perceive\.proj\.2\.", target_patterns="perceive.proj_out."), WeightRenaming(source_patterns=r"perceive\.", target_patterns="patch_merger."), # -- module placement: text under `language_model`, tower under `vision_tower` -- WeightRenaming( source_patterns=r"^model(?!\.(language_model|vit|vision_tower))", target_patterns="model.language_model", ), WeightRenaming(source_patterns=r"^vit", target_patterns="model.vision_tower"), ], # HY4-Flash: the same vision layout as HY4-VL, down to the projector's two convolutions. The text # renames sit here too rather than under the text sub-config's type, so they apply whatever a # released config calls its text tower. "hy_v4_flash": [ # -- text tower: the modules HY4 renames, with every rule spelled out. The mapping is reversed # when saving, which turns each target into a regex, and HY4's short targets (`.hc_`, # `.hc_scale`) would then also match inside this model's `unified_per_hc_scales`. WeightRenaming(r"\.self_attn\.learnable_sink_param", ".self_attn.sinks"), WeightRenaming(r"\.self_attn\.linear_gate\.weight", ".self_attn.gate_proj.weight"), # Follow DSv4 HC standards, dropping the `hc_pre` level the checkpoint carries WeightRenaming(r"\.hc_attn_layer\.hc_pre\.hc_fn", ".attn_hc.fn"), WeightRenaming(r"\.hc_attn_layer\.hc_pre\.hc_base", ".attn_hc.base"), WeightRenaming(r"\.hc_attn_layer\.hc_pre\.hc_scale", ".attn_hc.scale"), WeightRenaming(r"\.hc_mlp_layer\.hc_pre\.hc_fn", ".ffn_hc.fn"), WeightRenaming(r"\.hc_mlp_layer\.hc_pre\.hc_base", ".ffn_hc.base"), WeightRenaming(r"\.hc_mlp_layer\.hc_pre\.hc_scale", ".ffn_hc.scale"), WeightRenaming(r"\.hc_head\.hc_head_fn", ".hc_head.hc_fn"), WeightRenaming(r"\.hc_head\.hc_head_base", ".hc_head.hc_base"), WeightRenaming(r"\.hc_head\.hc_head_scale", ".hc_head.hc_scale"), # -- vision tower: Megatron-style MLP and pre/post norm names -- WeightRenaming(source_patterns="mlp.dense_h_to_4h", target_patterns="mlp.fc1"), WeightRenaming(source_patterns="mlp.dense_4h_to_h", target_patterns="mlp.fc2"), WeightRenaming( source_patterns=r"^vit\.layers\.(\d+)\.input_layernorm\.", target_patterns=r"vit.layers.\1.layer_norm1.", ), WeightRenaming( source_patterns=r"^vit\.layers\.(\d+)\.post_attention_layernorm\.", target_patterns=r"vit.layers.\1.layer_norm2.", ), # -- projector: normalize the RMS name and give the convolutions explicit module names -- WeightRenaming(source_patterns=r"(perceive|patch_merger)\.norm\.", target_patterns=r"\1.before_rms."), WeightRenaming(source_patterns=r"perceive\.proj\.0\.", target_patterns="perceive.proj_conv."), WeightRenaming(source_patterns=r"perceive\.proj\.2\.", target_patterns="perceive.proj_out."), WeightRenaming(source_patterns=r"perceive\.", target_patterns="patch_merger."), # -- module placement: text under `language_model`, tower under `vision_tower` -- WeightRenaming( source_patterns=r"^model(?!\.(language_model|vit|vision_tower))", target_patterns="model.language_model", ), WeightRenaming(source_patterns=r"^vit", target_patterns="model.vision_tower"), ], # Cosmos3 Edge's composite checkpoint stores its dense reasoner text tower as conventional attention + MLP # blocks. The visual/projector tensors already use their native module names and intentionally need no mapping. "cosmos3_edge": [ WeightRenaming(r"^embed_tokens\.", "model.language_model.embed_tokens."), WeightRenaming(r"^norm\.", "model.language_model.norm."), WeightRenaming(r"^layers\.", "model.language_model.layers."), WeightRenaming(r"\.self_attn\.to_q\.", ".self_attn.q_proj."), WeightRenaming(r"\.self_attn\.to_k\.", ".self_attn.k_proj."), WeightRenaming(r"\.self_attn\.to_v\.", ".self_attn.v_proj."), WeightRenaming(r"\.self_attn\.to_out\.", ".self_attn.o_proj."), WeightRenaming(r"\.mlp\.up_proj\.", ".mlp.fc1."), WeightRenaming(r"\.mlp\.down_proj\.", ".mlp.fc2."), ], "inkling_mm_model": [ WeightRenaming(source_patterns=r"model\.llm\.layers", target_patterns=r"model.language_model.layers"), WeightRenaming( source_patterns=r"model\.llm\.embed_norm\.weight", target_patterns=r"model.language_model.embed_tokens.embed_norm.weight", ), WeightRenaming( source_patterns=r"model\.llm\.embed\.weight", target_patterns=r"model.language_model.embed_tokens.weight", ), WeightRenaming( source_patterns=r"model\.llm\.norm\.weight", target_patterns=r"model.language_model.norm.weight" ), WeightRenaming(source_patterns=r"model\.llm\.unembed\.weight", target_patterns=r"lm_head.weight"), WeightRenaming(source_patterns=r"model\.audio\.", target_patterns=r"model.audio_tower."), WeightRenaming(source_patterns=r"model\.visual", target_patterns=r"model.vision_tower"), # Vision encoder internals (run after the tower namespace renames) WeightRenaming( source_patterns=r"vision_tower.layers.linear_(\d+)", target_patterns=r"vision_tower.encoder_layers.\1.projection", ), WeightRenaming( source_patterns=r"vision_tower.layers.norm_(\d+)", target_patterns=r"vision_tower.encoder_layers.\1.layer_norm", ), # Audio tower internals. These run AFTER the generic `model.audio.` -> `model.audio_tower.` # rename above and substring-match its output. The audio dMel embedding moved from # `audio.encoder` to an `InklingAudioModelEmbeddings` submodule, and `final_norm` was # renamed to `norm`. WeightRenaming( source_patterns=r"audio_tower.encoder.weight", target_patterns=r"audio_tower.embed_audio_tokens.embed_audio_tokens.weight", ), WeightRenaming( source_patterns=r"audio_tower.final_norm.weight", target_patterns=r"audio_tower.norm.weight" ), # MoE and MLP # no Transpose ops here: the TP loader shards the raw tensor but validates the shard # shape on the target param, so dim-permuting conversions break sharded loads cc @cyril WeightConverter( source_patterns="shared_w13_weight", target_patterns=["gate_proj", "up_proj"], operations=[Interleave(dim=1), Chunk(dim=1)], ), WeightRenaming(source_patterns=r"shared_w2_weight", target_patterns=r"down_proj"), WeightConverter( source_patterns="mlp.experts.w13_weight", target_patterns=["mlp.experts.gate_up_proj"], operations=[Interleave(dim=1)], ), WeightRenaming(source_patterns=r"mlp.experts.w2_weight", target_patterns=r"mlp.experts.down_proj"), WeightConverter( source_patterns="mlp.w13_dn.weight", target_patterns=["mlp.gate_proj.weight", "mlp.up_proj.weight"], operations=[Interleave(dim=0), Chunk(dim=0)], ), WeightRenaming(source_patterns=r"mlp.w2_md.weight", target_patterns=r"mlp.down_proj.weight"), WeightRenaming(source_patterns=r"mlp.gate.bias", target_patterns=r"mlp.gate.e_score_correction_bias"), # Attn WeightRenaming(source_patterns=r"attn\.wq_du", target_patterns=r"self_attn.q_proj"), WeightRenaming(source_patterns=r"attn\.wk_dv", target_patterns=r"self_attn.k_proj"), WeightRenaming(source_patterns=r"attn\.wv_dv", target_patterns=r"self_attn.v_proj"), WeightRenaming(source_patterns=r"attn\.wr_du", target_patterns=r"self_attn.r_proj"), WeightRenaming(source_patterns=r"attn\.wo_ud", target_patterns=r"self_attn.o_proj"), WeightRenaming(source_patterns=r"\.attn\.q_norm", target_patterns=r".self_attn.q_norm"), WeightRenaming(source_patterns=r"\.attn\.k_norm", target_patterns=r".self_attn.k_norm"), WeightRenaming(source_patterns=r"\.attn\.k_sconv", target_patterns=r".self_attn.k_sconv.conv1d"), WeightRenaming(source_patterns=r"\.attn\.v_sconv", target_patterns=r".self_attn.v_sconv.conv1d"), WeightRenaming(source_patterns=r"\.attn\.rel_logits_proj", target_patterns=r".self_attn.rel_logits_proj"), WeightRenaming(source_patterns=r"attn_sconv\.weight", target_patterns=r"attn_sconv.conv1d.weight"), WeightRenaming(source_patterns=r"mlp_sconv\.weight", target_patterns=r"mlp_sconv.conv1d.weight"), WeightRenaming(source_patterns=r"mlp_norm", target_patterns=r"post_attention_layernorm"), WeightRenaming(source_patterns=r"attn_norm", target_patterns=r"input_layernorm"), ], "inkling2": [ WeightRenaming(source_patterns=r"model\.llm\.unembed\.weight", target_patterns=r"lm_head.weight"), WeightRenaming(source_patterns=r"model\.llm\.", target_patterns=r"model.language_model."), WeightRenaming( source_patterns=r"model\.visual\.backbone\.embeddings\.", target_patterns=r"model.vision_tower.embeddings.", ), WeightRenaming( source_patterns=r"vision_tower\.embeddings\.position_embedding", target_patterns=r"vision_tower.embeddings.position_embedding.weight", ), WeightRenaming( source_patterns=r"model\.visual\.backbone\.ln_pre\.", target_patterns=r"model.vision_tower.pre_layernorm.", ), WeightRenaming( source_patterns=r"model\.visual\.backbone\.layers\.", target_patterns=r"model.vision_tower.layers." ), WeightRenaming( source_patterns=r"model\.visual\.proj_in\.", target_patterns=r"model.vision_tower.merger.linear_1." ), WeightRenaming( source_patterns=r"model\.visual\.proj_out\.", target_patterns=r"model.vision_tower.merger.linear_2." ), WeightRenaming( source_patterns=r"(vision_tower\.layers\.\d+)\.ln_1\.", target_patterns=r"\1.layernorm_before." ), WeightRenaming( source_patterns=r"(vision_tower\.layers\.\d+)\.ln_2\.", target_patterns=r"\1.layernorm_after." ), WeightRenaming( source_patterns=r"(vision_tower\.layers\.\d+)\.mlp\.c_fc\.", target_patterns=r"\1.mlp.fc1." ), WeightRenaming( source_patterns=r"(vision_tower\.layers\.\d+)\.mlp\.c_proj\.", target_patterns=r"\1.mlp.fc2." ), WeightRenaming( source_patterns=r"(vision_tower\.layers\.\d+)\.ls_1\.gamma", target_patterns=r"\1.lambda_1" ), WeightRenaming( source_patterns=r"(vision_tower\.layers\.\d+)\.ls_2\.gamma", target_patterns=r"\1.lambda_2" ), ], "inkling2_text": [ WeightRenaming(source_patterns=r"model\.llm\.unembed\.weight", target_patterns=r"lm_head.weight"), WeightRenaming(source_patterns=r"model\.llm\.", target_patterns=r"model."), WeightRenaming(source_patterns=r"embed\.weight", target_patterns=r"embed_tokens.weight"), WeightRenaming(source_patterns=r"embed_norm\.", target_patterns=r"embed_tokens.embed_norm."), WeightConverter( source_patterns=r"mlp\.experts\.w13_weight", target_patterns=r"mlp.experts.gate_up_proj", operations=[Interleave(dim=1)], ), WeightConverter( source_patterns=r"mlp\.experts\.w13_weight_scale", target_patterns=r"mlp.experts.gate_up_proj_scale_inv", operations=[Interleave(dim=1)], ), WeightRenaming(source_patterns=r"mlp\.experts\.w2_weight", target_patterns=r"mlp.experts.down_proj"), WeightRenaming( source_patterns=r"mlp\.experts\.w2_weight_scale", target_patterns=r"mlp.experts.down_proj_scale_inv" ), WeightRenaming(source_patterns=r"mlp.gate.bias", target_patterns=r"mlp.gate.e_score_correction_bias"), WeightRenaming(source_patterns=r"attn\.wq_du", target_patterns=r"self_attn.q_proj"), WeightRenaming(source_patterns=r"attn\.wk_dv", target_patterns=r"self_attn.k_proj"), WeightRenaming(source_patterns=r"attn\.wv_dv", target_patterns=r"self_attn.v_proj"), WeightRenaming(source_patterns=r"attn\.wr_du", target_patterns=r"self_attn.r_proj"), WeightRenaming(source_patterns=r"attn\.wo_ud", target_patterns=r"self_attn.o_proj"), WeightRenaming(source_patterns=r"attn\.gate_attn", target_patterns=r"self_attn.gate_proj"), WeightRenaming(source_patterns=r"attn\.attn_sink_proj", target_patterns=r"self_attn.sink_proj"), WeightRenaming( source_patterns=r"\.attn\.(q_l2_clip|k_l2_clip|rel_l2_clip|k_sconv|v_sconv|rel_logits_proj)\.", target_patterns=r".self_attn.\1.", ), WeightRenaming(source_patterns=r"attn_sconv\.", target_patterns=r"post_attention_convolution."), WeightRenaming(source_patterns=r"mlp_sconv\.", target_patterns=r"post_mlp_convolution."), WeightRenaming(source_patterns=r"(convolution|sconv)\.weight(?!_)", target_patterns=r"\1.conv1d.weight"), WeightRenaming(source_patterns=r"(convolution|sconv)\.static_w", target_patterns=r"\1.static_weight"), WeightRenaming( source_patterns=r"(convolution|sconv)\.weight_proj", target_patterns=r"\1.weight_proj.weight" ), WeightRenaming(source_patterns=r"mlp_norm", target_patterns=r"post_attention_layernorm"), WeightRenaming(source_patterns=r"attn_norm", target_patterns=r"input_layernorm"), WeightRenaming(source_patterns=r"(q|k|v)_input_norm", target_patterns=r"\1_input_layernorm"), Inkling2IndexOffsetWeightRename( source_patterns=r"attn_res\.attn_queries\.(\d+)", target_patterns=r"layers.\1.attention_input.query", offset=1, ), WeightRenaming( source_patterns=r"attn_res\.mlp_queries\.(\d+)", target_patterns=r"layers.\1.mlp_input.query" ), WeightRenaming(source_patterns=r"attn_res\.output_query", target_patterns=r"final_input.query"), ], "GPTNeoXForCausalLM": [ WeightRenaming(source_patterns=r"^embed_out\.", target_patterns="lm_head."), ], "axk2": [ # The A.X-K2 hub checkpoints store the routed experts as individual `experts.{i}` projections # (DeepSeek-V3 layout); the model packs them into the stacked expert tensors. WeightConverter( source_patterns=[ "mlp.experts.*.gate_proj.weight", "mlp.experts.*.up_proj.weight", ], target_patterns="mlp.experts.gate_up_proj", operations=[MergeModulelist(dim=0), Concatenate(dim=1)], ), WeightConverter( source_patterns="mlp.experts.*.down_proj.weight", target_patterns="mlp.experts.down_proj", operations=[MergeModulelist(dim=0)], ), # The gated norms store their low-rank gate MLP as `W_down` / `W_up`; the model reuses `CLIPMLP` # (`fc1` / `fc2`). Bridge the names on load (and back on save). Patterns are unanchored (no # surrounding dots) so the same rename also normalizes the bare module names in an fp8 checkpoint's # `modules_to_not_convert` (`W_down`/`W_up` -> `mlp.fc1`/`mlp.fc2`); a dot-anchored pattern would # leave those bare names untouched and the gate MLP would be quantized by mistake. WeightRenaming(source_patterns=r"W_down", target_patterns="mlp.fc1"), WeightRenaming(source_patterns=r"W_up", target_patterns="mlp.fc2"), # The released checkpoints fuse the query up-projection and the attention output gate into a # single `q_b_proj` (vLLM layout); the model keeps it fused under the clearer name `q_gate_proj` # and splits the activation in the forward. A plain rename (kept fp8-safe: no weight split, and # the prefix also renames the `weight_scale_inv` companion). WeightRenaming(source_patterns=r"self_attn\.q_b_proj\.", target_patterns="self_attn.q_gate_proj."), ], "gemma4_unified": [ WeightRenaming(source_patterns=r"vision_embedder\.patch_ln1", target_patterns="embed_vision.patch_ln1"), WeightRenaming( source_patterns=r"vision_embedder\.patch_dense", target_patterns="embed_vision.patch_dense" ), WeightRenaming(source_patterns=r"vision_embedder\.patch_ln2", target_patterns="embed_vision.patch_ln2"), WeightRenaming( source_patterns=r"vision_embedder\.pos_embedding", target_patterns="embed_vision.pos_embedding" ), WeightRenaming(source_patterns=r"vision_embedder\.pos_norm", target_patterns="embed_vision.pos_norm"), WeightRenaming( source_patterns=r"embed_vision\.embedding_projection", target_patterns="embed_vision.multimodal_embedder.embedding_projection", ), ], "radio": [ WeightRenaming("radio_model.model.patch_generator.video_embedder", "embeddings.video_patch_projection"), WeightRenaming("radio_model.model.patch_generator.embedder", "embeddings.patch_projection"), WeightRenaming("radio_model.model.patch_generator.pos_embed", "embeddings.position_embedding"), WeightRenaming("radio_model.model.patch_generator.cls_token.token", "embeddings.cls_register_token"), WeightRenaming("radio_model.model.blocks", "encoder.layer"), WeightRenaming("attn.proj", "attention.o_proj"), WeightRenaming("radio_model.input_conditioner", "input_conditioner"), WeightRenaming("radio_model.summary_idxs", "summary_idxs"), WeightConverter( source_patterns="attn.qkv", target_patterns=["attention.q_proj", "attention.k_proj", "attention.v_proj"], operations=[Chunk(dim=0)], ), ], "NemotronH_Omni_Reasoning_V3": [ WeightRenaming(r"^mlp1\.0\.", r"multi_modal_projector\.layer_norm\."), WeightRenaming(r"^mlp1\.1\.", r"multi_modal_projector\.linear_1\."), WeightRenaming(r"^mlp1\.3\.", r"multi_modal_projector\.linear_2\."), WeightRenaming(r"^sound_encoder\.encoder\.", r"audio_tower\."), WeightRenaming(r"^sound_projection\.norm\.", r"embed_audio\.layer_norm\."), WeightRenaming(r"^sound_projection\.linear1\.", r"embed_audio\.linear_1\."), WeightRenaming(r"^sound_projection\.linear2\.", r"embed_audio\.linear_2\."), ], "hrm_text": [ WeightConverter( source_patterns="mlp.gate_up_proj.weight", target_patterns=["mlp.gate_proj.weight", "mlp.up_proj.weight"], operations=[Chunk(dim=0)], ), WeightConverter( source_patterns="attn.gqkv_proj.weight", target_patterns=[ "self_attn.gate_proj.weight", "self_attn.q_proj.weight", "self_attn.k_proj.weight", "self_attn.v_proj.weight", ], operations=[Chunk(dim=0)], ), WeightRenaming(source_patterns=r"\.attn\.o_proj\.", target_patterns=".self_attn.o_proj."), ], "hunyuan_vl": [ WeightRenaming(source_patterns="mlp.dense_h_to_4h", target_patterns="mlp.fc1"), WeightRenaming(source_patterns="mlp.dense_4h_to_h", target_patterns="mlp.fc2"), WeightRenaming( source_patterns=r"^vit\.layers\.(\d+)\.input_layernorm\.", target_patterns=r"vit.layers.\1.layer_norm1.", ), WeightRenaming( source_patterns=r"^vit\.layers\.(\d+)\.post_attention_layernorm\.", target_patterns=r"vit.layers.\1.layer_norm2.", ), WeightRenaming(source_patterns=r"perceive\.proj\.0\.", target_patterns="perceive.proj_conv."), WeightRenaming(source_patterns=r"perceive\.proj\.2\.", target_patterns="perceive.proj_out."), WeightRenaming(source_patterns=r"perceive\.", target_patterns="patch_merger."), WeightRenaming( source_patterns=r"^model(?!\.(language_model|vit|vision_tower))", target_patterns="model.language_model", ), WeightRenaming(source_patterns=r"^model\.vit", target_patterns="model.vision_tower"), WeightRenaming(source_patterns=r"^vit", target_patterns="model.vision_tower"), ], "ViTModel": [ WeightRenaming(r"encoder\.layer\.", "layers."), WeightRenaming("attention.query", "q_proj"), WeightRenaming("attention.key", "k_proj"), WeightRenaming("attention.value", "v_proj"), WeightRenaming("attention.output.dense", "attention.o_proj"), WeightRenaming("intermediate.dense", "mlp.fc1"), WeightRenaming("output.dense", "mlp.fc2"), ], "Dinov2Model": [ WeightRenaming("attention.attention.query", "attention.q_proj"), WeightRenaming("attention.attention.key", "attention.k_proj"), WeightRenaming("attention.attention.value", "attention.v_proj"), WeightRenaming("attention.output.dense", "attention.o_proj"), WeightConverter( source_patterns="mlp.weights_in.weight", target_patterns=["mlp.gate_proj.weight", "mlp.up_proj.weight"], operations=[Chunk(dim=0)], ), WeightConverter( source_patterns="mlp.weights_in.bias", target_patterns=["mlp.gate_proj.bias", "mlp.up_proj.bias"], operations=[Chunk(dim=0)], ), WeightRenaming("mlp.weights_out", "mlp.down_proj"), ], "Dinov2WithRegistersModel": [ WeightRenaming("attention.attention.query", "attention.q_proj"), WeightRenaming("attention.attention.key", "attention.k_proj"), WeightRenaming("attention.attention.value", "attention.v_proj"), WeightRenaming("attention.output.dense", "attention.o_proj"), WeightConverter( source_patterns="mlp.weights_in.weight", target_patterns=["mlp.gate_proj.weight", "mlp.up_proj.weight"], operations=[Chunk(dim=0)], ), WeightConverter( source_patterns="mlp.weights_in.bias", target_patterns=["mlp.gate_proj.bias", "mlp.up_proj.bias"], operations=[Chunk(dim=0)], ), WeightRenaming("mlp.weights_out", "mlp.down_proj"), ], "ViTMSNForImageClassification": [ WeightRenaming(r"^encoder\.", "vit.encoder."), ], "ViTMAEForPreTraining": [ WeightRenaming("attention.query", "q_proj"), WeightRenaming("attention.key", "k_proj"), WeightRenaming("attention.value", "v_proj"), WeightRenaming("attention.output.dense", "attention.o_proj"), WeightRenaming("intermediate.dense", "mlp.fc1"), WeightRenaming("output.dense", "mlp.fc2"), ], "BeitBackbone": [ WeightRenaming(r"^fpn1\.0\.", "fpn.fpn1.conv_transpose1."), WeightRenaming(r"^fpn1\.1\.", "fpn.fpn1.normalization."), WeightRenaming(r"^fpn1\.3\.", "fpn.fpn1.conv_transpose2."), WeightRenaming(r"^fpn2\.0\.", "fpn.fpn2."), WeightRenaming(r"^encoder\.layer\.", "beit.encoder.layer."), WeightRenaming(r"^embeddings\.", "beit.embeddings."), WeightRenaming("attention.query", "q_proj"), WeightRenaming("attention.key", "k_proj"), WeightRenaming("attention.value", "v_proj"), WeightRenaming("attention.output.dense", "attention.o_proj"), WeightRenaming("intermediate.dense", "mlp.fc1"), WeightRenaming("output.dense", "mlp.fc2"), ], "BeitForSemanticSegmentation": [ WeightRenaming(r"(? `.self_attn.` # would otherwise also match the text decoder's `language_model.layers.*.self_attn.*` keys. "step3p5_vision": [ WeightRenaming(source_patterns=r"^conv1\.weight", target_patterns="embeddings.patch_embedding.weight"), WeightRenaming( source_patterns=r"^positional_embedding", target_patterns="embeddings.position_embedding.weight", ), WeightRenaming(source_patterns=r"^transformer\.resblocks\.", target_patterns="layers."), WeightRenaming(source_patterns=r"^ln_pre\.", target_patterns="pre_layernorm."), WeightRenaming(source_patterns=r"^vit_downsampler1\.", target_patterns="downsampler1."), WeightRenaming(source_patterns=r"^vit_downsampler2\.", target_patterns="downsampler2."), WeightRenaming(source_patterns=r"\.ls_1\.gamma", target_patterns=".lambda_1"), WeightRenaming(source_patterns=r"\.ls_2\.gamma", target_patterns=".lambda_2"), WeightRenaming(source_patterns=r"\.mlp\.c_fc\.", target_patterns=".mlp.fc1."), WeightRenaming(source_patterns=r"\.mlp\.c_proj\.", target_patterns=".mlp.fc2."), WeightRenaming(source_patterns=r"\.attn\.", target_patterns=".self_attn."), WeightRenaming(source_patterns=r"\.ln_1\.", target_patterns=".layernorm_before."), WeightRenaming(source_patterns=r"\.ln_2\.", target_patterns=".layernorm_after."), # Unfuse qkv and apply rope permutation. WeightConverter( source_patterns=r"attn\.in_proj_weight", target_patterns=[ r"attn.q_proj.weight", r"attn.k_proj.weight", r"attn.v_proj.weight", ], operations=[ Chunk(dim=0), PermuteForRope(subconfig_key="vision_config", permute_layer_names=["q_proj", "k_proj"]), ], ), WeightConverter( source_patterns=r"attn\.in_proj_bias", target_patterns=[ r"attn.q_proj.bias", r"attn.k_proj.bias", r"attn.v_proj.bias", ], operations=[ Chunk(dim=0), PermuteForRope(subconfig_key="vision_config", permute_layer_names=["q_proj", "k_proj"]), ], ), ], "colqwen2": [PrefixChange(prefix_to_remove="model", model_prefix="vlm")], "shieldgemma2": [PrefixChange(prefix_to_add="model", model_prefix="model")], "timm_wrapper": [PrefixChange(prefix_to_add="timm_model")], "pi0": [ PrefixChange(prefix_to_remove="model"), WeightRenaming(source_patterns=r"state_proj", target_patterns="embed_action_time.state_proj"), WeightRenaming(source_patterns=r"action_in_proj", target_patterns="embed_action_time.action_in_proj"), WeightRenaming( source_patterns=r"action_time_mlp_in", target_patterns="embed_action_time.action_time_mlp_in" ), WeightRenaming( source_patterns=r"action_time_mlp_out", target_patterns="embed_action_time.action_time_mlp_out" ), WeightRenaming(source_patterns=r"^paligemma_with_expert.paligemma.model", target_patterns="model.vlm"), WeightRenaming(source_patterns=r"^paligemma_with_expert.gemma_expert.model", target_patterns="model.dit"), # Weight on the hub have only `lm_head` saved, but PI0 doesn't create any lm-head initialized! WeightRenaming( source_patterns=r"^paligemma_with_expert.gemma_expert.lm_head", target_patterns="model.dit.embed_tokens", ), WeightRenaming( source_patterns=r"^paligemma_with_expert.paligemma.lm_head", target_patterns="model.vlm.language_model.embed_tokens", ), ], "esm": [ WeightRenaming( "encoder.layer.*.attention.self.rotary_embeddings.inv_freq", "rotary_embeddings.inv_freq", ), ], "dinov3_convnext": [WeightRenaming(r"(? we end up with 2 tensors Concatenate(dim=1), # each process has 2 tensors, gate and up, we concat them into gate_up ], # we want the loading to add this shard operation here. Though we can't shard after concats and merge, needs to be first ), WeightConverter( source_patterns=[ ".experts.*.w2.weight", ], target_patterns=".experts.down_proj", # target key gets the list of two tensors operations=[ MergeModulelist( dim=0 ), # each process has two lists of tensors, we cat each list. -> we end up with 2 tensors ], # we want the loading to add this shard operation here. Though we can't shard after concats and merge, needs to be first ), ], "qwen2_moe": [ WeightConverter( source_patterns=[ "mlp.experts.*.gate_proj.weight", "mlp.experts.*.up_proj.weight", ], target_patterns="mlp.experts.gate_up_proj", operations=[MergeModulelist(dim=0), Concatenate(dim=1)], ), WeightConverter( source_patterns="mlp.experts.*.down_proj.weight", target_patterns="mlp.experts.down_proj", operations=[MergeModulelist(dim=0)], ), ], "dots3_note_audio_encoder": [ WeightRenaming(r"^audio_adapter\.", "adapter."), WeightRenaming(r"^adapter\.proj\.0\.", "adapter.norm."), WeightRenaming(r"^adapter\.proj\.1\.", "adapter.fc1."), WeightRenaming(r"^adapter\.proj\.3\.", "adapter.fc2."), WeightRenaming(r"dots_encoder\.speech_encoder\.", "speech_encoder."), WeightRenaming(r"speech_encoder\.(conv2d[123]|conv_out)\.", r"speech_encoder.conv_stem.\1."), WeightRenaming(r"conv_stem\.conv2d1\.", "conv_stem.convs.0."), WeightRenaming(r"conv_stem\.conv2d2\.", "conv_stem.convs.1."), WeightRenaming(r"conv_stem\.conv2d3\.", "conv_stem.convs.2."), WeightRenaming(r"(speech_encoder\.layers\.\d+\.self_attn)\.out_proj\.", r"\1.o_proj."), WeightRenaming(r"\.self_attn_layer_norm\.", ".input_layernorm."), WeightRenaming(r"\.final_layer_norm\.", ".post_attention_layernorm."), WeightRenaming(r"(speech_encoder\.layers\.\d+)\.fc1\.", r"\1.mlp.gate_up_proj."), WeightRenaming(r"(speech_encoder\.layers\.\d+)\.fc2\.", r"\1.mlp.down_proj."), WeightRenaming(r"^speech_encoder\.(conv_stem|layers|layer_norm)\.", r"\1."), ], "dots3_note_vision_encoder": [ WeightRenaming(r"^post_trunk_norm\.", "norm."), WeightRenaming(r"^adapter\.", "merger."), WeightRenaming(r"(blocks\.\d+)\.norm_1\.", r"\1.norm1."), WeightRenaming(r"(blocks\.\d+)\.norm_2\.", r"\1.norm2."), WeightRenaming(r"(blocks\.\d+\.mlp)\.fc1\.", r"\1.gate_proj."), WeightRenaming(r"(blocks\.\d+\.mlp)\.fc2\.", r"\1.down_proj."), WeightRenaming(r"(blocks\.\d+\.mlp)\.fc3\.", r"\1.up_proj."), WeightRenaming(r"\.mlp\.gate_weight", ".mlp.gate.weight"), WeightRenaming(r"\.mlp\.router_bias", ".mlp.e_score_correction_bias"), WeightConverter( source_patterns=[".mlp.experts.*.fc1.weight", ".mlp.experts.*.fc3.weight"], target_patterns=".mlp.experts.gate_up_proj", operations=[MergeModulelist(dim=0), Concatenate(dim=1)], ), WeightConverter( source_patterns=".mlp.experts.*.fc2.weight", target_patterns=".mlp.experts.down_proj", operations=[MergeModulelist(dim=0)], ), ], "Dots3NoteForConditionalGeneration": [ WeightRenaming(r"^audio_encoder\.audio_adapter\.", "audio_adapter."), WeightRenaming(r"^(vision_encoder|audio_encoder|audio_adapter)\.", r"model.\1."), ], "Dots3NoteModel": [ PrefixChange(prefix_to_remove="model"), WeightRenaming(r"^(embed_tokens|layers|norm)\.", r"language_model.\1."), WeightRenaming(r"^vision_encoder\.", "vision_tower."), WeightRenaming(r"^audio_encoder\.", "audio_tower."), WeightRenaming(r"^audio_adapter\.", "audio_tower.audio_adapter."), ], "qwen3_vl_moe": [ WeightConverter( source_patterns="mlp.experts.gate_up_proj", target_patterns="mlp.experts.gate_up_proj", operations=[Transpose(1, 2, check_dims=True)], ), WeightConverter( source_patterns="mlp.experts.down_proj", target_patterns="mlp.experts.down_proj", operations=[Transpose(1, 2, check_dims=True)], ), ], "cosmos3_omni": [ # Cosmos3 unified checkpoints store the Reasoner LLM with the old `model.` prefix # stripped off and the ViT under flat `blocks.*` / `merger.*` / `patch_embed.*` / # `pos_embed.*` / `deepstack_merger_list.*`. Re-target both to the nested Qwen3-VL # layout (`model.language_model.*` and `model.visual.*`). Newer checkpoints also # use Diffusers-style attention names; map them back to Qwen3-VL module names. WeightRenaming( source_patterns=r"^(layers\.|embed_tokens\.|norm\.)", target_patterns=r"model.language_model.\1", ), WeightRenaming( source_patterns=r"^(blocks\.|merger\.|patch_embed\.|pos_embed\.|deepstack_merger_list\.)", target_patterns=r"model.visual.\1", ), WeightRenaming(source_patterns=r"\.self_attn\.to_q\.", target_patterns=".self_attn.q_proj."), WeightRenaming(source_patterns=r"\.self_attn\.to_k\.", target_patterns=".self_attn.k_proj."), WeightRenaming(source_patterns=r"\.self_attn\.to_v\.", target_patterns=".self_attn.v_proj."), WeightRenaming(source_patterns=r"\.self_attn\.to_out\.", target_patterns=".self_attn.o_proj."), WeightRenaming(source_patterns=r"\.self_attn\.norm_q\.", target_patterns=".self_attn.q_norm."), WeightRenaming(source_patterns=r"\.self_attn\.norm_k\.", target_patterns=".self_attn.k_norm."), ], "phimoe": [ WeightRenaming(".block_sparse_moe.", ".mlp."), WeightRenaming(".gate.weight", ".router.weight"), WeightConverter( source_patterns=[ ".experts.*.w1.weight", ".experts.*.w3.weight", ], target_patterns=".experts.gate_up_proj", operations=[MergeModulelist(dim=0), Concatenate(dim=1)], ), WeightConverter( source_patterns=".experts.*.w2.weight", target_patterns=".experts.down_proj", operations=[MergeModulelist(dim=0)], ), ], "lfm3": [ WeightRenaming(source_patterns=r"\.self_attn\.gate_proj\.", target_patterns=".self_attn.g_proj."), ], "lfm3_vl_vision": [ WeightRenaming(source_patterns="patchifier", target_patterns="patch_embed"), WeightRenaming(source_patterns="q_layernorm", target_patterns="q_norm"), WeightRenaming(source_patterns="k_layernorm", target_patterns="k_norm"), ], "lfm2_moe": [ WeightConverter( source_patterns=[ "feed_forward.experts.*.w1.weight", "feed_forward.experts.*.w3.weight", ], target_patterns="feed_forward.experts.gate_up_proj", operations=[MergeModulelist(dim=0), Concatenate(dim=1)], ), WeightConverter( source_patterns="feed_forward.experts.*.w2.weight", target_patterns="feed_forward.experts.down_proj", operations=[MergeModulelist(dim=0)], ), ], "ernie4_5_vl_moe": [ # vision WeightRenaming("vision_model", "vision_tower"), # resampler WeightRenaming("spatial_linear.0", "spatial_linear.fc1"), WeightRenaming("spatial_linear.2", "spatial_linear.fc2"), WeightRenaming("spatial_linear.3", "spatial_linear.ln"), WeightRenaming("temporal_linear.0", "temporal_linear.fc1"), WeightRenaming("temporal_linear.2", "temporal_linear.fc2"), WeightRenaming("temporal_linear.3", "temporal_linear.ln"), # language model WeightRenaming(r"(? self_attn rename. # Pack the official q/k/v kernels once at load time for a single runtime depthwise convolution. WeightConverter( source_patterns=[ "q_conv1d.weight", "k_conv1d.weight", "v_conv1d.weight", ], target_patterns="conv1d.weight", operations=[Concatenate(dim=0)], ), WeightRenaming(r"\.self_attn\.dense\.", ".self_attn.o_proj."), WeightRenaming(r"\.mlp\.gate\.expert_bias", ".mlp.gate.e_score_correction_bias"), ] mapping["ernie4_5_moe"] = mapping["qwen2_moe"].copy() mapping["ernie4_5_moe"] += [ WeightRenaming("mlp.moe_statics.e_score_correction_bias", "mlp.gate.moe_statics.e_score_correction_bias") ] mapping["BeitModel"] = mapping["ViTModel"].copy() + [ WeightRenaming("encoder.relative_position_bias", "shared_position_bias"), WeightRenaming( "attention.attention.relative_position_bias.relative_position_bias_table", "relative_position_bias.relative_position_bias_table", ), ] mapping["pixio"] = mapping["ViTModel"].copy() mapping["pixio"] += [ WeightRenaming("norm1", "layernorm_before"), WeightRenaming("norm2", "layernorm_after"), WeightRenaming(r"^encoder\.", "pixio."), ] mapping["minimax_m2"] = mapping["mixtral"].copy() mapping["minimax_m2"] += [ WeightRenaming(".block_sparse_moe.e_score_correction_bias", ".mlp.e_score_correction_bias"), ] mapping["exaone_moe"] = mapping["qwen2_moe"].copy() mapping["exaone_moe"] += [WeightRenaming("mlp.e_score_correction_bias", "mlp.gate.e_score_correction_bias")] mapping["kolibri1"] = mapping["qwen2_moe"].copy() mapping["kolibri1"] += [ WeightRenaming("moe.router.expert_bias", "mlp.gate.e_score_correction_bias"), WeightRenaming("post_attn_norm", "post_self_attn_layernorm"), WeightRenaming("post_ffn_norm", "post_mlp_layernorm"), ] mapping["mimo_v2_flash"] = mapping["qwen2_moe"].copy() mapping["mimo_v2_flash"] += [ WeightRenaming("self_attn.attention_sink_bias", "self_attn.sinks"), ] mapping["flint"] = mapping["qwen2_moe"].copy() mapping["flint"] += [ WeightRenaming("self_attn.softmax_offset", "self_attn.sinks"), ] # HYV3: qwen2_moe expert fusion + attribute renames for MiniMaxM2-style inheritance mapping["hy_v3"] = mapping["qwen2_moe"].copy() mapping["hy_v3"] += [ WeightRenaming(source_patterns=r"mlp\.router\.gate\.weight", target_patterns="mlp.gate.weight"), WeightRenaming(source_patterns=r"mlp\.expert_bias", target_patterns="mlp.e_score_correction_bias"), WeightRenaming(source_patterns=r"mlp\.shared_mlp\.", target_patterns="mlp.shared_experts."), ] mapping["qwen3_5_moe_text"] = mapping["qwen3_5_text"].copy() mapping["qwen3_5_moe_text"] += mapping["qwen2_moe"].copy() mapping["laguna"] = mapping["qwen2_moe"].copy() mapping["laguna"] += [ WeightRenaming("mlp.experts.e_score_correction_bias", "mlp.gate.e_score_correction_bias"), WeightRenaming("mlp.shared_expert.", "mlp.shared_experts."), ] # A.X-K1 checkpoints store the MoE output norm at the decoder-layer level. mapping["axk1"] = mapping["qwen2_moe"].copy() mapping["axk1"] += [ WeightRenaming("post_mlp_layernorm", "mlp.post_mlp_layernorm"), ] mapping["qwen4_exp_text"] = mapping["qwen3_5_moe_text"].copy() mapping["qwen4_exp_text"] += [ WeightConverter( source_patterns="ngram_embedding.shard_*.weight", target_patterns="ngram_embedding.weight", operations=[Concatenate(dim=0, num_shards_attribute="split_ngram_parts")], # The size of the embedding is ~95 GiB, so we cannot afford to perform the Cat on device, as it will need # a temporary memory buffer of the same size force_cpu=True, ), ] mapping["MtpModel"] = [ PrefixChange(prefix_to_remove="model"), PrefixChange(prefix_to_remove="mtp"), WeightRenaming(source_patterns=".shared_head.norm.", target_patterns=".post_norm."), WeightRenaming(source_patterns=".mtp_block.enorm.", target_patterns=".enorm."), WeightRenaming(source_patterns=".mtp_block.hnorm.", target_patterns=".hnorm."), WeightRenaming(source_patterns=".mtp_block.eh_proj.", target_patterns=".eh_proj."), WeightRenaming(source_patterns=".mtp_block.post_norm.", target_patterns=".post_norm."), # Inkling checkpoint layout: per-depth extras sit at `layers.{k}.` while the decoder block # is nested under `layers.{k}.transformer_block.`; the main-model conversions (applied after) # rename the block internals WeightRenaming(source_patterns=r"\.hidden_norm\.", target_patterns=r".hnorm."), WeightRenaming(source_patterns=r"\.embed_norm\.", target_patterns=r".enorm."), WeightRenaming(source_patterns=r"\.input_proj\.", target_patterns=r".eh_proj."), WeightRenaming(source_patterns=r"^chain_norm\.", target_patterns=r"shared_post_norm."), WeightRenaming(source_patterns=r"layers\.(\d+)\.transformer_block\.", target_patterns=r"layers.\1.mtp_block."), # Mellum checkpoint layout: single MTP layer with `fc`, `pre_fc_norm_*` and `norm` stored one level up. WeightRenaming( source_patterns=r"^layers\.(\d+)\.(?!(?:mtp_block|enorm|hnorm|eh_proj|post_norm)\.)", target_patterns=r"layers.\1.mtp_block.", ), WeightRenaming(source_patterns=r"^fc\.", target_patterns="layers.0.eh_proj."), WeightRenaming(source_patterns=r"^pre_fc_norm_embedding\.", target_patterns="layers.0.enorm."), WeightRenaming(source_patterns=r"^pre_fc_norm_hidden\.", target_patterns="layers.0.hnorm."), WeightRenaming(source_patterns=r"^norm\.", target_patterns="layers.0.post_norm."), ] mapping["kimi_linear"] = [ # Forget gate weights are attached to the forget gate module instead of the attention WeightRenaming(source_patterns=r"self_attn\.f_a_proj\.", target_patterns=r"self_attn.forget_gate.f_a_proj."), WeightRenaming(source_patterns=r"self_attn\.f_b_proj\.", target_patterns=r"self_attn.forget_gate.f_b_proj."), WeightRenaming(source_patterns=r"self_attn\.dt_bias", target_patterns=r"self_attn.forget_gate.dt_bias"), WeightRenaming(source_patterns=r"self_attn\.A_log", target_patterns=r"self_attn.forget_gate.A_log"), # Conv weights are stacked before runtime WeightConverter( source_patterns=[ "self_attn.q_conv1d.weight", "self_attn.k_conv1d.weight", "self_attn.v_conv1d.weight", ], target_patterns="self_attn.conv1d.weight", operations=[Concatenate(dim=0)], ), # Rename MoEs so they have the same prefix as the MLPs WeightRenaming(source_patterns=r"\.block_sparse_moe\.", target_patterns=r"\.mlp\."), # Concatenate w1 (gate) and w3 (up) weights into a single weight and merge across experts WeightConverter( source_patterns=[ r"\.experts.*.w1.weight", r"\.experts.*.w3.weight", ], target_patterns=r"\.experts.gate_up_proj", operations=[MergeModulelist(dim=0), Concatenate(dim=1)], ), # Merge w2 (down) weights across experts WeightConverter( source_patterns=r"\.experts.*.w2.weight", target_patterns=r"\.experts.down_proj", operations=[MergeModulelist(dim=0)], ), ] # The `readout.safetensors` of the decision models of vLLM (pplx-decider, d3): the weight of the head, no bias mapping["jev"] = [WeightRenaming(r"^weight", "proj.weight")] # Gradium's runtime checkpoint (`model.safetensors` next to the runtime's files). Its codec (`mimi.`) and its # training-only tensors are left unexpected: the codec is the processor's audio tokenizer. conditioners = r"^flow_lm\.condition_provider\.conditioners\." flow_layers = r"^flow_lm\.transformer\.layers\.(\d+)\." flow_net = r"^flow_lm\.flow_net\." mapping["gradium_phonon_tts"] = [ WeightRenaming(conditioners + r"transcript_in_segment\.embed\.", "model.embed_tokens."), WeightRenaming(conditioners + r"speaker_wavs\.output_proj\.", "model.audio_prompt_proj."), WeightRenaming(conditioners + r"num_speakers\.embed\.", "model.num_speakers_embed."), WeightRenaming(conditioners + r"num_speakers\.output_proj\.", "model.num_speakers_proj."), WeightRenaming(conditioners + r"duration_delta\.output_proj\.", "model.duration_delta_proj."), WeightRenaming(r"^flow_lm\.input_linear\.", "model.audio_latent_proj."), WeightRenaming(r"^flow_lm\.bos_emb", "model.bos_latent"), WeightRenaming(r"^flow_lm\.emb_mean", "latent_mean"), WeightRenaming(r"^flow_lm\.emb_std", "latent_std"), WeightRenaming(r"^flow_lm\.out_norm\.", "model.norm."), WeightRenaming(r"^flow_lm\.out_eos\.", "eos_head."), WeightRenaming(flow_layers + r"self_attn\.in_proj\.", r"model.layers.\1.self_attn.in_proj."), WeightRenaming(flow_layers + r"self_attn\.out_proj\.", r"model.layers.\1.self_attn.o_proj."), WeightRenaming(flow_layers + r"linear1\.", r"model.layers.\1.mlp.fc1."), WeightRenaming(flow_layers + r"linear2\.", r"model.layers.\1.mlp.fc2."), WeightRenaming(flow_layers + r"norm1\.", r"model.layers.\1.input_layernorm."), WeightRenaming(flow_layers + r"norm2\.", r"model.layers.\1.post_attention_layernorm."), WeightRenaming(flow_net + r"time_embed\.(\d+)\.freqs", r"flow_head.timestep_embeddings.\1.freqs"), WeightRenaming(flow_net + r"time_embed\.(\d+)\.mlp\.0\.", r"flow_head.timestep_embeddings.\1.fc1."), WeightRenaming(flow_net + r"time_embed\.(\d+)\.mlp\.2\.", r"flow_head.timestep_embeddings.\1.fc2."), WeightRenaming( flow_net + r"time_embed\.(\d+)\.mlp\.3\.alpha", r"flow_head.timestep_embeddings.\1.norm.weight" ), WeightRenaming(flow_net + r"cond_embed\.", "flow_head.cond_proj."), WeightRenaming(flow_net + r"input_proj\.", "flow_head.input_proj."), WeightRenaming(flow_net + r"res_blocks\.(\d+)\.in_ln\.", r"flow_head.layers.\1.norm."), WeightRenaming(flow_net + r"res_blocks\.(\d+)\.mlp\.0\.", r"flow_head.layers.\1.fc1."), WeightRenaming(flow_net + r"res_blocks\.(\d+)\.mlp\.2\.", r"flow_head.layers.\1.fc2."), WeightRenaming(flow_net + r"res_blocks\.(\d+)\.adaLN_modulation\.1\.", r"flow_head.layers.\1.adaln_proj."), WeightRenaming(flow_net + r"final_layer\.linear\.", "flow_head.final_layer.proj."), WeightRenaming(flow_net + r"final_layer\.adaLN_modulation\.1\.", "flow_head.final_layer.adaln_proj."), # The fused projection, its q/k rows permuted from interleaved RoPE pairs to the rotate-half layout (it also # splits the codec's, left unexpected) WeightConverter( source_patterns=r"self_attn\.in_proj\.weight", target_patterns=["self_attn.q_proj.weight", "self_attn.k_proj.weight", "self_attn.v_proj.weight"], operations=[Chunk(dim=0), PermuteForRope(permute_layer_names=["q_proj", "k_proj"])], ), ] # NVIDIA's NeMo export (`model.safetensors`, also loaded by NeMo and vLLM). Its multi-token prediction head, the # feature extractor constants, an unused speaker head, the TransformerEngine metadata and the word timestamp head's # constants are left unexpected (see `Nemotron3_5TranscribePreTrainedModel._keys_to_ignore_on_load_unexpected`). diarization = r"^perception\.encoder\.diarization_model\.sortformer_modules\." mapping["nemo_speechlm"] = [ WeightRenaming(r"^perception\.encoder\.asr_encoder\.", r"^model\.audio_tower\.asr_encoder\."), WeightRenaming( r"^perception\.encoder\.diarization_model\.encoder\.", r"^model\.audio_tower\.diarization_model\.model\.audio_tower\.", ), WeightRenaming(r"^ctc_timestamp\.decoder\.transformer\.", r"^model\.word_timestamp_encoder\."), WeightRenaming(r"^ctc_timestamp\.decoder\.upsample\.", r"^model\.word_timestamp_encoder\.upsampler\."), WeightRenaming(r"^ctc_timestamp\.decoder\.decoder_layers\.0\.", r"^word_timestamp_head\."), WeightRenaming(diarization + r"encoder_proj\.", r"^model\.audio_tower\.diarization_model\.model\.proj\."), WeightRenaming( diarization + r"subpixel_upsample\.", r"^model\.audio_tower\.diarization_model\.model\.upsampler\.conv\." ), WeightRenaming( diarization + r"first_hidden_to_hidden\.", r"^model\.audio_tower\.diarization_model\.classifier\.dense\." ), WeightRenaming( diarization + r"single_hidden_to_spks\.", r"^model\.audio_tower\.diarization_model\.classifier\.out_proj\." ), WeightRenaming(diarization + r"learnable_sil_emb", r"^model\.audio_tower\.diarization_model\.silence_embeds"), WeightRenaming(r"^perception\.encoder\.asr_norm\.", r"^model\.audio_tower\.asr_norm\."), WeightRenaming(r"^perception\.encoder\.diar_norm\.", r"^model\.audio_tower\.speaker_norm\."), WeightRenaming(r"^perception\.proj\.", r"^model\.multi_modal_projector\."), WeightRenaming(r"^llm\.model\.embed_tokens\.", r"^model\.language_model\.embeddings\."), WeightRenaming(r"^llm\.model\.norm\.", r"^model\.language_model\.norm_f\."), WeightRenaming(r"^llm\.model\.layers\.", r"^model\.language_model\.layers\."), WeightRenaming(r"^llm\.lm_head\.", r"^lm_head\."), # NeMo `TransformerEncoder` blocks: the speech recognition encoder, the diarization encoder and the word # timestamp head WeightRenaming(r"\.pre_encode\.proj\.", r"\.embedder\.projection\."), WeightRenaming(r"\.embed_norm\.", r"\.input_layer_norm\."), WeightRenaming(r"\.final_norm\.", r"\.layer_norm\."), WeightRenaming(r"\.norm1\.", r"\.layer_norm1\."), WeightRenaming(r"\.norm2\.", r"\.layer_norm2\."), WeightRenaming(r"\.attn\.out_proj\.", r"\.self_attn\.o_proj\."), WeightRenaming(r"\.attn\.q_norm\.", r"\.self_attn\.q_norm\."), WeightRenaming(r"\.attn\.k_norm\.", r"\.self_attn\.k_norm\."), WeightRenaming(r"\.ffn\.net\.0\.", r"\.mlp\.fc1\."), WeightRenaming(r"\.ffn\.net\.3\.", r"\.mlp\.fc2\."), # Mamba-2 parameters NeMo keeps in float32 under a separate module WeightRenaming(r"\.mixer\._fp32_params\.A_log", r"\.mixer\.A_log"), WeightRenaming(r"\.mixer\._fp32_params\.D", r"\.mixer\.D"), WeightRenaming(r"\.mixer\._fp32_params\.dt_bias", r"\.mixer\.dt_bias"), # the fused projection, rows ordered `[query | key | value]` WeightConverter( source_patterns=r"\.attn\.w_qkv\.weight", target_patterns=[ r"\.self_attn\.q_proj\.weight", r"\.self_attn\.k_proj\.weight", r"\.self_attn\.v_proj\.weight", ], operations=[Chunk(dim=0)], ), # routed experts stored as `(num_experts, in_features, out_features)` WeightConverter( source_patterns=r"\.mixer\.experts\.gate_and_up_projs", target_patterns=r"\.mixer\.experts\.up_proj", operations=[Transpose(1, 2)], ), WeightConverter( source_patterns=r"\.mixer\.experts\.down_projs", target_patterns=r"\.mixer\.experts\.down_proj", operations=[Transpose(1, 2)], ), ] mapping["mistral_large4"] = [ # Dispatch the root vision_encoder to the vision model WeightRenaming(r"^vision_encoder\.", "model.vision_tower."), # Dispatch the root language modules to the language model WeightRenaming(r"^tok_embeddings\.", "model.language_model.embed_tokens."), WeightRenaming(r"^lerope\.alpha", "model.language_model.rotary_emb.alpha"), WeightRenaming(r"^layers\.", "model.language_model.layers."), WeightRenaming(r"^norm\.", "model.language_model.norm."), # And the rest to the multi-modal projector WeightRenaming(r"^output\.", "lm_head."), WeightRenaming(r"^pre_mm_projector_norm\.", "model.multi_modal_projector.norm."), WeightRenaming(r"^patch_merger\.", "model.multi_modal_projector.patch_merger."), WeightRenaming(r"^vision_language_adapter\.w_in\.", "model.multi_modal_projector.linear_1."), WeightRenaming(r"^vision_language_adapter\.w_out\.", "model.multi_modal_projector.linear_2."), # Rename the attention projections shared by both towers: value and output WeightRenaming(r"\.wv\.", ".v_proj."), WeightRenaming(r"\.wo\.", ".o_proj."), # Same for up, down and gate projection weights (except for routed experts' which will be handled by converters) WeightRenaming(r"\.(feed_forward|shared_experts)\.w1\.", r".\1.gate_proj."), WeightRenaming(r"\.(feed_forward|shared_experts)\.w2\.", r".\1.down_proj."), WeightRenaming(r"\.(feed_forward|shared_experts)\.w3\.", r".\1.up_proj."), # Vision model Q and K projectors: saved for interleaved RoPE, Pixtral expects rotate-half WeightConverter( source_patterns=r"attention\.wq\.", target_patterns="attention.q_proj.", operations=[PermuteForRope(subconfig_key="vision_config")], ), WeightConverter( source_patterns=r"attention\.wk\.", target_patterns="attention.k_proj.", operations=[PermuteForRope(subconfig_key="vision_config")], ), ] mapping["mistral_large4_text"] = [ # Rename attention modules (the indexer keeps wk, wq_b, and weights_proj) WeightRenaming(r"\.attention\.", r"\.self_attn\."), WeightRenaming(r"\.wq\.", ".q_proj."), WeightRenaming(r"\.wkv_a_with_mqa\.", r"\.kv_a_proj_with_mqa\."), WeightRenaming(r"\.wkv_b\.", ".kv_b_proj."), # Rename the norms WeightRenaming(r"\.attention_norm\.", r"\.input_layernorm\."), WeightRenaming(r"\.ffn_norm\.", r"\.post_attention_layernorm\."), WeightRenaming(r"\.kv_a_norm\.", r"\.kv_a_layernorm\."), # Rename MLPs and MoEs to the "mlp" prefix WeightRenaming(r"\.feed_forward\.", r"\.mlp\."), WeightRenaming(r"\.gate\.", r"\.mlp.gate\."), WeightRenaming(r"\.experts\.", r"\.mlp.experts\."), WeightRenaming(r"\.shared_experts\.", r"\.mlp.shared_experts\."), # Rename softmax scaling module WeightRenaming(r"\.ssmax_scaling.s", r"\.softmax_scaling.scale"), WeightRenaming(r"\.ssmax_scaling.b", r"\.softmax_scaling.bias"), # Concatenate w1 (gate) and w3 (up) weights into a single weight and merge across experts WeightConverter( source_patterns=[r"\.experts.*.w1.weight", r"\.experts.*.w3.weight"], target_patterns=r"\.experts.gate_up_proj", operations=[MergeModulelist(dim=0), Concatenate(dim=1)], ), # Merge w2 (down) weights across experts WeightConverter( source_patterns=r"\.experts.*.w2.weight", target_patterns=r"\.experts.down_proj", operations=[MergeModulelist(dim=0)], ), ] for model_type, base_pattern in _MODEL_TO_CONVERSION_PATTERN.items(): if model_type in mapping: continue mapping[model_type] = mapping[base_pattern].copy() return mapping _checkpoint_conversion_mapping_cache = None def get_checkpoint_conversion_mapping(model_type): global _checkpoint_conversion_mapping_cache if _checkpoint_conversion_mapping_cache is None: _checkpoint_conversion_mapping_cache = _build_checkpoint_conversion_mapping() return deepcopy(_checkpoint_conversion_mapping_cache.get(model_type)) USER_REGISTERED_MAPPINGS = set() def register_checkpoint_conversion_mapping( model_type_or_class_name: str, mapping: list[WeightConverter | WeightRenaming], overwrite: bool = False, ) -> None: """ Register a conversion mapping for a model type string or a class name. Class names take priority over `model_type` strings during lookup (see `extract_weight_conversions_for_model`), making it possible to define task-head-specific or class-specific conversions that differ from the shared `model_type` baseline. """ global _checkpoint_conversion_mapping_cache if _checkpoint_conversion_mapping_cache is None: _checkpoint_conversion_mapping_cache = _build_checkpoint_conversion_mapping() if model_type_or_class_name in _checkpoint_conversion_mapping_cache and not overwrite: raise ValueError( f"Conversion mapping for '{model_type_or_class_name}' already exists. Pass overwrite=True to replace it." ) _checkpoint_conversion_mapping_cache[model_type_or_class_name] = mapping # Keep track of what was added manually by the user USER_REGISTERED_MAPPINGS.add(model_type_or_class_name) def extract_weight_conversions_for_model( model: PreTrainedModel, ) -> list[WeightTransform] | None: """ Return the registered conversion list for `model`, or `None` if none exists. Looks up by class name first (enables task-head-specific overrides), then falls back to `model.config.model_type`. Transforms are returned unmodified; the caller sets `scope_prefix` on each transform for sub-module isolation. """ class_name = type(model).__name__ model_type = model.config.model_type # Class name takes priority — allows ForXxx-specific overrides conversions = get_checkpoint_conversion_mapping(class_name) if conversions is None and model_type: conversions = get_checkpoint_conversion_mapping(model_type) return conversions def get_model_conversion_mapping( model: PreTrainedModel, key_mapping: dict[str, str] | None = None, hf_quantizer: HfQuantizer | None = None, add_legacy: bool = True, ) -> list[WeightTransform]: """ Collect the ordered list of weight transforms for `model` (used during loading and, when reversed, during saving). Each `PreTrainedModel` sub-module is looked up by class name then `model_type`. Root transforms are applied globally; sub-module transforms have their `scope_prefix` set so they only match keys under that prefix. After any sub-module is processed, both its class name and `model_type` are marked seen to prevent `XForY` / `XModel` pairs from applying the same mapping twice via different lookup paths. """ from .modeling_utils import PreTrainedModel # note: this function is used in PEFT, so changing the API requires coordination weight_conversions = [] # Load models with explicit, user-provided key mapping if key_mapping is not None: weight_conversions = [WeightRenaming(source_patterns=k, target_patterns=v) for k, v in key_mapping.items()] # Maps each identifier (class name or model_type) to the module paths that have # already claimed it. A later module is skipped only when one of those paths is # an ancestor of the current module path — siblings are never ancestors of each # other, so two sibling sub-models with the same model_type both get their own # scoped transforms. A child is an ancestor of everything nested under it, which # prevents a parent's transforms from being duplicated with a scoped copy for the child. seen_identifiers: defaultdict[str, list[str]] = defaultdict(list) for module_name, submodule in model.named_modules(): # Skip if it's not a submodel if not isinstance(submodule, PreTrainedModel): continue class_name = type(submodule).__name__ model_type = submodule.config.model_type # Skip it if it's custom code and it was NOT registered by the user directly: it may have the same `model_type`/ClassName # as a native model inside the library, but it uses custom modeling so it should not share the conversions if ( submodule.is_custom_code() and class_name not in USER_REGISTERED_MAPPINGS and model_type not in USER_REGISTERED_MAPPINGS ): continue # Skip if an ancestor already claimed this class (its unscoped transforms already cover this subtree). if any(seen == "" or module_name.startswith(seen + ".") for seen in seen_identifiers[class_name]): continue # Class name takes priority — a class-specific mapping bypasses the model_type # deduplication check (e.g. LlavaModel nested inside LlavaForConditionalGeneration # must still get its own scoped mapping even after "llava" is marked seen). conversions = get_checkpoint_conversion_mapping(class_name) found_via_class = conversions is not None if not found_via_class: # Same ancestor check as above, but via model_type for modules without a class-specific mapping. if model_type and any( seen == "" or module_name.startswith(seen + ".") for seen in seen_identifiers[model_type] ): continue if model_type is not None: conversions = get_checkpoint_conversion_mapping(model_type) if conversions is None: continue is_root_model = module_name == "" if not is_root_model: # Scope each transform so it only matches keys under this sub-module's prefix - but we still allow to # arbitrary add/remove base_model_prefix to load ForXXX model from BaseModel and the opposite # Note that we need 2 removeprefix calls here, as only one level of nesting would not have the ending dot to module_name scope_prefix = module_name.removeprefix(model.base_model_prefix) scope_prefix = scope_prefix.removeprefix(".") for transform in conversions: transform.scope_prefix = scope_prefix transform.base_model_prefix = model.base_model_prefix weight_conversions.extend(conversions) seen_identifiers[class_name].append(module_name) # Only record model_type when the hit was via model_type. When the hit was via # class name, other sub-modules sharing the same model_type but without a # class-specific mapping (e.g. DetrModel under DetrForSegmentation) must still # be reachable so their base transforms are picked up and scoped. if not found_via_class and model_type: seen_identifiers[model_type].append(module_name) if add_legacy: weight_conversions.extend(get_checkpoint_conversion_mapping("legacy")) if hf_quantizer is not None: weight_conversions = hf_quantizer.update_weight_conversions(weight_conversions, model) return weight_conversions