VisionJEV-4B / native_runtime /warm_model.py
tinnel123's picture
Release VisionJEV 1.0.0 (4B): exact complete native checkpoint and inference
1120183 verified
Raw History Blame Contribute Delete
8.43 kB
"""Full official Flash warmstart. No raw-base or fresh classifier initialization.
The actual frozen author reference runs before FP32 promotion/FSDP on the same
single loaded body. The training wrapper gathers the exact native marker and
retains saved temperature; last-token pooling is not silently substituted.
"""
from pathlib import Path
import json,hashlib
from contract import need,TEMPERATURE,REVISION
def install_qualified_mlu_conv():
from fla.modules.convolution import causal_conv1d
from transformers.models.qwen3_5 import modeling_qwen3_5
def accelerated_conv(x,weight,bias=None,activation=None,**kwargs):
output,_=causal_conv1d(x.transpose(1,2).contiguous(),weight,bias=bias,activation=activation)
return output.transpose(1,2)
modeling_qwen3_5.causal_conv1d_fn=accelerated_conv
def load_native_reference(cfg,device,final=False):
install_qualified_mlu_conv()
import torch
from transformers import AutoProcessor
from transformers.models.qwen3_5.modeling_qwen3_5 import Qwen3_5Model
from safetensors.torch import load_file
from flash_author.model import Decider,enable_noncausal_full_attention
from flash_author.protocol import Encoder,MIN_PIXELS
checkpoint=Path(cfg['staged_parent']);saved=json.loads((checkpoint/'decision_config.json').read_text())
need(len(saved['codes'])==255,'Saved Flash code ABI differs')
need((final and 0.05<=saved['temperature']<=20)or saved['temperature']==TEMPERATURE,'Exact parent or fitted final temperature')
need(cfg['parent_revision']==REVISION,'Wrong parent revision')
engine=Decider.__new__(Decider);engine.checkpoint=checkpoint;engine.device=device
engine.temperature=saved['temperature'];engine.codes=list(saved['codes']);engine.max_length=262144
engine.batch_tokens=32768;engine.max_pixels=1638400;engine.fast=None
engine.processor=AutoProcessor.from_pretrained(checkpoint,local_files_only=True,trust_remote_code=False)
engine.processor.tokenizer.padding_side='left';engine.processor.image_processor.size={'shortest_edge':MIN_PIXELS,'longest_edge':1638400}
for code,token in zip(saved['codes'],saved['token_ids'],strict=True):
need(engine.processor.tokenizer.encode(code,add_special_tokens=False)==[token],'Original code/token row identity changed')
engine.encoder=Encoder(engine.processor,engine.codes)
engine.backbone,info=Qwen3_5Model.from_pretrained(checkpoint,dtype=torch.bfloat16,attn_implementation='sdpa',local_files_only=True,output_loading_info=True)
from loading_diagnostics import strict_loading_report
strict_loading_report(info)
head_state=load_file(str(checkpoint/'readout.safetensors'))
need(set(head_state)=={'weight'}and list(head_state['weight'].shape)==[255,2560]
and head_state['weight'].dtype==(torch.float32 if final else torch.bfloat16),'Complete saved parent BF16 or trained final FP32 master head required')
engine.readout=torch.nn.Linear(2560,255,bias=False,dtype=torch.bfloat16)
status=engine.readout.load_state_dict(head_state,strict=True)
need(not status.missing_keys and not status.unexpected_keys,'Strict saved head load failed')
enable_noncausal_full_attention(engine.backbone.language_model)
if final and (checkpoint/'adaptation_config.json').is_file():
from language_lora import load_export_adapter
load_export_adapter(engine.backbone,checkpoint)
engine.backbone.requires_grad_(False).eval().to(device);engine.readout.requires_grad_(False).eval().to(device)
return engine,info
def reference_rows(engine,chunks):
# Exact unchanged author body/head gather/BF16-to-FP32 readout reference.
rows=[]
for chunk in chunks:
values=engine.reference_logits([chunk])[0]
need(len(values)==1,'One original question per accepted source record')
rows.append(values[0])
return rows
def decision_class():
import torch
class Decision(torch.nn.Module):
def __init__(self,backbone,readout):
super().__init__();self.backbone=backbone;self.readout=readout
def forward(self,inputs,counts,positions,any_image=False,raw=False):
hook=None
if any_image and 'pixel_values'not in inputs:
device=inputs['input_ids'].device
dummy=self.backbone.visual(torch.zeros(256,1536,device=device,dtype=torch.bfloat16),grid_thw=torch.tensor([[1,16,16]],device=device))
zero=dummy.pooler_output.sum()*0
hook=self.backbone.get_input_embeddings().register_forward_hook(lambda m,a,o:o+zero.to(o.dtype))
try:hidden=self.backbone(**inputs,use_cache=False).last_hidden_state
finally:
if hook is not None:hook.remove()
rows=torch.arange(hidden.shape[0],device=hidden.device)
values=self.readout(hidden[rows,positions]).float()
# Explicit saved temperature preserves original initial probabilities. The
# FP32 master stores raw head values; no unannounced fold or new random head.
scaled=values if raw else values/TEMPERATURE
return scaled.masked_fill(torch.arange(255,device=scaled.device)[None]>=counts[:,None],-1e9)
return Decision
def promote_and_wrap(cfg,engine,device):
install_qualified_mlu_conv()
import torch,torch.distributed as dist
from torch.distributed.fsdp import FullyShardedDataParallel as FSDP,MixedPrecision,ShardingStrategy,BackwardPrefetch
from torch.distributed.fsdp.wrap import ModuleWrapPolicy
from accelerate import init_empty_weights
from transformers import AutoConfig
from transformers.models.qwen3_5.modeling_qwen3_5 import Qwen3_5Model,Qwen3_5DecoderLayer,Qwen3_5VisionModel
from flash_author.model import enable_noncausal_full_attention
Decision=decision_class();rank=dist.get_rank();world=dist.get_world_size();shard_size=cfg['shard_size']
need(world==cfg['world']==8 and shard_size==8,'Exact approved single-host8 sharding contract')
if rank==0:
need(engine is not None,'Original rank0 reference body is the warmstart; no second body')
engine.backbone.float();engine.readout.float();model=Decision(engine.backbone,engine.readout)
else:
config=AutoConfig.from_pretrained(cfg['staged_parent'],local_files_only=True);config._attn_implementation='sdpa'
with init_empty_weights(include_buffers=False):model=Decision(Qwen3_5Model(config),torch.nn.Linear(2560,255,bias=False,dtype=torch.float32))
enable_noncausal_full_attention(model.backbone.language_model)
from language_lora import install,configure_trainable
if cfg['training_mode']=='language_lora':
install(model.backbone,cfg['lora'],cfg['seed'])
configure_trainable(model,cfg['training_mode'])
model.backbone.enable_input_require_grads()
declared=[]
for name,param in model.named_parameters():
role='head' if name=='readout.weight' else 'vision' if '.visual.' in name else 'embedding' if 'embed_tokens' in name else 'language'
declared.append({'name':name,'shape':list(param.shape),'dtype':str(param.dtype),'requires_grad':param.requires_grad,'role':role,'elements':param.numel()})
model.backbone.gradient_checkpointing_enable(gradient_checkpointing_kwargs={'use_reentrant':False})
def materialize(module):
for name,param in list(module.named_parameters(recurse=False)):
if param.is_meta:setattr(module,name,torch.nn.Parameter(torch.empty(param.shape,device=device,dtype=torch.float32),requires_grad=param.requires_grad))
for name,buf in list(module.named_buffers(recurse=False)):
need(not buf.is_meta,'Nonpersistent semantic buffer cannot be invented');module._buffers[name]=buf.to(device)
group=None;strategy=ShardingStrategy.FULL_SHARD
if shard_size<world:
shard=replica=None
for start in range(0,world,shard_size):
ranks=list(range(start,start+shard_size));g=dist.new_group(ranks,backend='cncl')
if rank in ranks:shard=g
for local in range(shard_size):
ranks=list(range(local,world,shard_size));g=dist.new_group(ranks,backend='cncl')
if rank in ranks:replica=g
group=(shard,replica);strategy=ShardingStrategy.HYBRID_SHARD
wrapped=FSDP(model,auto_wrap_policy=ModuleWrapPolicy({Qwen3_5DecoderLayer,Qwen3_5VisionModel}),
device_id=device,param_init_fn=materialize,sync_module_states=True,use_orig_params=True,
process_group=group,sharding_strategy=strategy,limit_all_gathers=True,backward_prefetch=None,
mixed_precision=MixedPrecision(param_dtype=torch.bfloat16,reduce_dtype=torch.float32,buffer_dtype=torch.float32))
wrapped._training_layout_report={'training_mode':cfg['training_mode'],'parameters':declared,'no_actual_gradient_claim_before_backward':True}
return wrapped