hy4-flash-vision-encoder / component.py
gatilin's picture
Add files using upload-large-folder tool
48e1b03 verified
Raw History Blame Contribute Delete
2.71 kB
"""Load extracted Hy4-flash vision tower plus visual projector.
Requires the matching upstream Transformers implementation referenced in README.
The checkpoint retains original `vit.*` keys; `convert_name` performs the same
vision-key renames as the pinned upstream Transformers conversion mapping.
"""
import json,re
from pathlib import Path
import torch
from safetensors.torch import load_file
def convert_name(key):
if not key.startswith('vit.'):
raise ValueError(f'Unexpected non-vision key: {key}')
name=key.removeprefix('vit.')
name=name.replace('mlp.dense_h_to_4h','mlp.fc1').replace('mlp.dense_4h_to_h','mlp.fc2')
name=re.sub(r'^(layers\.\d+)\.input_layernorm\.',r'\1.layer_norm1.',name)
name=re.sub(r'^(layers\.\d+)\.post_attention_layernorm\.',r'\1.layer_norm2.',name)
name=name.replace('perceive.norm.','perceive.before_rms.').replace('perceive.proj.0.','perceive.proj_conv.').replace('perceive.proj.2.','perceive.proj_out.')
name=name.replace('perceive.','patch_merger.')
return name
def load_model(root=None,*,device='cpu',dtype=None):
from transformers.models.hy_v4_flash.configuration_hy_v4_flash import HYV4FlashVisionConfig
from transformers.models.hy_v4_flash.modeling_hy_v4_flash import HYV4FlashVisionTransformer
root=Path(root or Path(__file__).resolve().parent)
raw=json.loads((root/'config.json').read_text())['vision_config']
# The released config contains extra training/runtime keys unused by the vision class.
keys=('hidden_act','hidden_size','intermediate_size','interpolate_mode','rms_norm_eps','attention_dropout','num_attention_heads','num_key_value_heads','num_channels','num_hidden_layers','out_hidden_size','patch_size','spatial_merge_size','max_image_size')
cfg=HYV4FlashVisionConfig(**{k:raw[k] for k in keys if k in raw});cfg._attn_implementation='sdpa'
with torch.device('meta'):
model=HYV4FlashVisionTransformer(cfg)
source=load_file(str(root/'model.safetensors'),device='cpu');state={}
for key,value in source.items():
name=convert_name(key)
if name in state:raise ValueError('Duplicate target key: '+name)
state[name]=value
model.load_state_dict(state,strict=True,assign=True)
if dtype is not None:model=model.to(dtype=dtype)
return model.to(device=device).eval().requires_grad_(False)
def load_processor(root=None):
from transformers.models.hy_v4_flash.image_processing_hy_v4_flash import HYV4FlashImageProcessor
root=Path(root or Path(__file__).resolve().parent)
cfg=json.loads((root/'processor_config.json').read_text())['image_processor'];cfg.pop('image_processor_type',None)
return HYV4FlashImageProcessor(**cfg)