Download native_runtime/warm_model.py from tinnel123/VisionJEV-4B: direct link, hf CLI and curl.
- Browser
- Download file 8.43 kB
-
https://huggingface.co/tinnel123/VisionJEV-4B/resolve/main/native_runtime/warm_model.py
- Command line
-
hf download hf://tinnel123/VisionJEV-4B/native_runtime/warm_model.py
-
curl -L -o warm_model.py https://huggingface.co/tinnel123/VisionJEV-4B/resolve/main/native_runtime/warm_model.py
8.43 kB
| """Full official Flash warmstart. No raw-base or fresh classifier initialization. | |
| The actual frozen author reference runs before FP32 promotion/FSDP on the same | |
| single loaded body. The training wrapper gathers the exact native marker and | |
| retains saved temperature; last-token pooling is not silently substituted. | |
| """ | |
| from pathlib import Path | |
| import json,hashlib | |
| from contract import need,TEMPERATURE,REVISION | |
| def install_qualified_mlu_conv(): | |
| from fla.modules.convolution import causal_conv1d | |
| from transformers.models.qwen3_5 import modeling_qwen3_5 | |
| def accelerated_conv(x,weight,bias=None,activation=None,**kwargs): | |
| output,_=causal_conv1d(x.transpose(1,2).contiguous(),weight,bias=bias,activation=activation) | |
| return output.transpose(1,2) | |
| modeling_qwen3_5.causal_conv1d_fn=accelerated_conv | |
| def load_native_reference(cfg,device,final=False): | |
| install_qualified_mlu_conv() | |
| import torch | |
| from transformers import AutoProcessor | |
| from transformers.models.qwen3_5.modeling_qwen3_5 import Qwen3_5Model | |
| from safetensors.torch import load_file | |
| from flash_author.model import Decider,enable_noncausal_full_attention | |
| from flash_author.protocol import Encoder,MIN_PIXELS | |
| checkpoint=Path(cfg['staged_parent']);saved=json.loads((checkpoint/'decision_config.json').read_text()) | |
| need(len(saved['codes'])==255,'Saved Flash code ABI differs') | |
| need((final and 0.05<=saved['temperature']<=20)or saved['temperature']==TEMPERATURE,'Exact parent or fitted final temperature') | |
| need(cfg['parent_revision']==REVISION,'Wrong parent revision') | |
| engine=Decider.__new__(Decider);engine.checkpoint=checkpoint;engine.device=device | |
| engine.temperature=saved['temperature'];engine.codes=list(saved['codes']);engine.max_length=262144 | |
| engine.batch_tokens=32768;engine.max_pixels=1638400;engine.fast=None | |
| engine.processor=AutoProcessor.from_pretrained(checkpoint,local_files_only=True,trust_remote_code=False) | |
| engine.processor.tokenizer.padding_side='left';engine.processor.image_processor.size={'shortest_edge':MIN_PIXELS,'longest_edge':1638400} | |
| for code,token in zip(saved['codes'],saved['token_ids'],strict=True): | |
| need(engine.processor.tokenizer.encode(code,add_special_tokens=False)==[token],'Original code/token row identity changed') | |
| engine.encoder=Encoder(engine.processor,engine.codes) | |
| engine.backbone,info=Qwen3_5Model.from_pretrained(checkpoint,dtype=torch.bfloat16,attn_implementation='sdpa',local_files_only=True,output_loading_info=True) | |
| from loading_diagnostics import strict_loading_report | |
| strict_loading_report(info) | |
| head_state=load_file(str(checkpoint/'readout.safetensors')) | |
| need(set(head_state)=={'weight'}and list(head_state['weight'].shape)==[255,2560] | |
| and head_state['weight'].dtype==(torch.float32 if final else torch.bfloat16),'Complete saved parent BF16 or trained final FP32 master head required') | |
| engine.readout=torch.nn.Linear(2560,255,bias=False,dtype=torch.bfloat16) | |
| status=engine.readout.load_state_dict(head_state,strict=True) | |
| need(not status.missing_keys and not status.unexpected_keys,'Strict saved head load failed') | |
| enable_noncausal_full_attention(engine.backbone.language_model) | |
| if final and (checkpoint/'adaptation_config.json').is_file(): | |
| from language_lora import load_export_adapter | |
| load_export_adapter(engine.backbone,checkpoint) | |
| engine.backbone.requires_grad_(False).eval().to(device);engine.readout.requires_grad_(False).eval().to(device) | |
| return engine,info | |
| def reference_rows(engine,chunks): | |
| # Exact unchanged author body/head gather/BF16-to-FP32 readout reference. | |
| rows=[] | |
| for chunk in chunks: | |
| values=engine.reference_logits([chunk])[0] | |
| need(len(values)==1,'One original question per accepted source record') | |
| rows.append(values[0]) | |
| return rows | |
| def decision_class(): | |
| import torch | |
| class Decision(torch.nn.Module): | |
| def __init__(self,backbone,readout): | |
| super().__init__();self.backbone=backbone;self.readout=readout | |
| def forward(self,inputs,counts,positions,any_image=False,raw=False): | |
| hook=None | |
| if any_image and 'pixel_values'not in inputs: | |
| device=inputs['input_ids'].device | |
| dummy=self.backbone.visual(torch.zeros(256,1536,device=device,dtype=torch.bfloat16),grid_thw=torch.tensor([[1,16,16]],device=device)) | |
| zero=dummy.pooler_output.sum()*0 | |
| hook=self.backbone.get_input_embeddings().register_forward_hook(lambda m,a,o:o+zero.to(o.dtype)) | |
| try:hidden=self.backbone(**inputs,use_cache=False).last_hidden_state | |
| finally: | |
| if hook is not None:hook.remove() | |
| rows=torch.arange(hidden.shape[0],device=hidden.device) | |
| values=self.readout(hidden[rows,positions]).float() | |
| # Explicit saved temperature preserves original initial probabilities. The | |
| # FP32 master stores raw head values; no unannounced fold or new random head. | |
| scaled=values if raw else values/TEMPERATURE | |
| return scaled.masked_fill(torch.arange(255,device=scaled.device)[None]>=counts[:,None],-1e9) | |
| return Decision | |
| def promote_and_wrap(cfg,engine,device): | |
| install_qualified_mlu_conv() | |
| import torch,torch.distributed as dist | |
| from torch.distributed.fsdp import FullyShardedDataParallel as FSDP,MixedPrecision,ShardingStrategy,BackwardPrefetch | |
| from torch.distributed.fsdp.wrap import ModuleWrapPolicy | |
| from accelerate import init_empty_weights | |
| from transformers import AutoConfig | |
| from transformers.models.qwen3_5.modeling_qwen3_5 import Qwen3_5Model,Qwen3_5DecoderLayer,Qwen3_5VisionModel | |
| from flash_author.model import enable_noncausal_full_attention | |
| Decision=decision_class();rank=dist.get_rank();world=dist.get_world_size();shard_size=cfg['shard_size'] | |
| need(world==cfg['world']==8 and shard_size==8,'Exact approved single-host8 sharding contract') | |
| if rank==0: | |
| need(engine is not None,'Original rank0 reference body is the warmstart; no second body') | |
| engine.backbone.float();engine.readout.float();model=Decision(engine.backbone,engine.readout) | |
| else: | |
| config=AutoConfig.from_pretrained(cfg['staged_parent'],local_files_only=True);config._attn_implementation='sdpa' | |
| with init_empty_weights(include_buffers=False):model=Decision(Qwen3_5Model(config),torch.nn.Linear(2560,255,bias=False,dtype=torch.float32)) | |
| enable_noncausal_full_attention(model.backbone.language_model) | |
| from language_lora import install,configure_trainable | |
| if cfg['training_mode']=='language_lora': | |
| install(model.backbone,cfg['lora'],cfg['seed']) | |
| configure_trainable(model,cfg['training_mode']) | |
| model.backbone.enable_input_require_grads() | |
| declared=[] | |
| for name,param in model.named_parameters(): | |
| role='head' if name=='readout.weight' else 'vision' if '.visual.' in name else 'embedding' if 'embed_tokens' in name else 'language' | |
| declared.append({'name':name,'shape':list(param.shape),'dtype':str(param.dtype),'requires_grad':param.requires_grad,'role':role,'elements':param.numel()}) | |
| model.backbone.gradient_checkpointing_enable(gradient_checkpointing_kwargs={'use_reentrant':False}) | |
| def materialize(module): | |
| for name,param in list(module.named_parameters(recurse=False)): | |
| if param.is_meta:setattr(module,name,torch.nn.Parameter(torch.empty(param.shape,device=device,dtype=torch.float32),requires_grad=param.requires_grad)) | |
| for name,buf in list(module.named_buffers(recurse=False)): | |
| need(not buf.is_meta,'Nonpersistent semantic buffer cannot be invented');module._buffers[name]=buf.to(device) | |
| group=None;strategy=ShardingStrategy.FULL_SHARD | |
| if shard_size<world: | |
| shard=replica=None | |
| for start in range(0,world,shard_size): | |
| ranks=list(range(start,start+shard_size));g=dist.new_group(ranks,backend='cncl') | |
| if rank in ranks:shard=g | |
| for local in range(shard_size): | |
| ranks=list(range(local,world,shard_size));g=dist.new_group(ranks,backend='cncl') | |
| if rank in ranks:replica=g | |
| group=(shard,replica);strategy=ShardingStrategy.HYBRID_SHARD | |
| wrapped=FSDP(model,auto_wrap_policy=ModuleWrapPolicy({Qwen3_5DecoderLayer,Qwen3_5VisionModel}), | |
| device_id=device,param_init_fn=materialize,sync_module_states=True,use_orig_params=True, | |
| process_group=group,sharding_strategy=strategy,limit_all_gathers=True,backward_prefetch=None, | |
| mixed_precision=MixedPrecision(param_dtype=torch.bfloat16,reduce_dtype=torch.float32,buffer_dtype=torch.float32)) | |
| wrapped._training_layout_report={'training_mode':cfg['training_mode'],'parameters':declared,'no_actual_gradient_claim_before_backward':True} | |
| return wrapped | |