"""Full official Flash warmstart. No raw-base or fresh classifier initialization. The actual frozen author reference runs before FP32 promotion/FSDP on the same single loaded body. The training wrapper gathers the exact native marker and retains saved temperature; last-token pooling is not silently substituted. """ from pathlib import Path import json,hashlib from contract import need,TEMPERATURE,REVISION def install_qualified_mlu_conv(): from fla.modules.convolution import causal_conv1d from transformers.models.qwen3_5 import modeling_qwen3_5 def accelerated_conv(x,weight,bias=None,activation=None,**kwargs): output,_=causal_conv1d(x.transpose(1,2).contiguous(),weight,bias=bias,activation=activation) return output.transpose(1,2) modeling_qwen3_5.causal_conv1d_fn=accelerated_conv def load_native_reference(cfg,device,final=False): install_qualified_mlu_conv() import torch from transformers import AutoProcessor from transformers.models.qwen3_5.modeling_qwen3_5 import Qwen3_5Model from safetensors.torch import load_file from flash_author.model import Decider,enable_noncausal_full_attention from flash_author.protocol import Encoder,MIN_PIXELS checkpoint=Path(cfg['staged_parent']);saved=json.loads((checkpoint/'decision_config.json').read_text()) need(len(saved['codes'])==255,'Saved Flash code ABI differs') need((final and 0.05<=saved['temperature']<=20)or saved['temperature']==TEMPERATURE,'Exact parent or fitted final temperature') need(cfg['parent_revision']==REVISION,'Wrong parent revision') engine=Decider.__new__(Decider);engine.checkpoint=checkpoint;engine.device=device engine.temperature=saved['temperature'];engine.codes=list(saved['codes']);engine.max_length=262144 engine.batch_tokens=32768;engine.max_pixels=1638400;engine.fast=None engine.processor=AutoProcessor.from_pretrained(checkpoint,local_files_only=True,trust_remote_code=False) engine.processor.tokenizer.padding_side='left';engine.processor.image_processor.size={'shortest_edge':MIN_PIXELS,'longest_edge':1638400} for code,token in zip(saved['codes'],saved['token_ids'],strict=True): need(engine.processor.tokenizer.encode(code,add_special_tokens=False)==[token],'Original code/token row identity changed') engine.encoder=Encoder(engine.processor,engine.codes) engine.backbone,info=Qwen3_5Model.from_pretrained(checkpoint,dtype=torch.bfloat16,attn_implementation='sdpa',local_files_only=True,output_loading_info=True) from loading_diagnostics import strict_loading_report strict_loading_report(info) head_state=load_file(str(checkpoint/'readout.safetensors')) need(set(head_state)=={'weight'}and list(head_state['weight'].shape)==[255,2560] and head_state['weight'].dtype==(torch.float32 if final else torch.bfloat16),'Complete saved parent BF16 or trained final FP32 master head required') engine.readout=torch.nn.Linear(2560,255,bias=False,dtype=torch.bfloat16) status=engine.readout.load_state_dict(head_state,strict=True) need(not status.missing_keys and not status.unexpected_keys,'Strict saved head load failed') enable_noncausal_full_attention(engine.backbone.language_model) if final and (checkpoint/'adaptation_config.json').is_file(): from language_lora import load_export_adapter load_export_adapter(engine.backbone,checkpoint) engine.backbone.requires_grad_(False).eval().to(device);engine.readout.requires_grad_(False).eval().to(device) return engine,info def reference_rows(engine,chunks): # Exact unchanged author body/head gather/BF16-to-FP32 readout reference. rows=[] for chunk in chunks: values=engine.reference_logits([chunk])[0] need(len(values)==1,'One original question per accepted source record') rows.append(values[0]) return rows def decision_class(): import torch class Decision(torch.nn.Module): def __init__(self,backbone,readout): super().__init__();self.backbone=backbone;self.readout=readout def forward(self,inputs,counts,positions,any_image=False,raw=False): hook=None if any_image and 'pixel_values'not in inputs: device=inputs['input_ids'].device dummy=self.backbone.visual(torch.zeros(256,1536,device=device,dtype=torch.bfloat16),grid_thw=torch.tensor([[1,16,16]],device=device)) zero=dummy.pooler_output.sum()*0 hook=self.backbone.get_input_embeddings().register_forward_hook(lambda m,a,o:o+zero.to(o.dtype)) try:hidden=self.backbone(**inputs,use_cache=False).last_hidden_state finally: if hook is not None:hook.remove() rows=torch.arange(hidden.shape[0],device=hidden.device) values=self.readout(hidden[rows,positions]).float() # Explicit saved temperature preserves original initial probabilities. The # FP32 master stores raw head values; no unannounced fold or new random head. scaled=values if raw else values/TEMPERATURE return scaled.masked_fill(torch.arange(255,device=scaled.device)[None]>=counts[:,None],-1e9) return Decision def promote_and_wrap(cfg,engine,device): install_qualified_mlu_conv() import torch,torch.distributed as dist from torch.distributed.fsdp import FullyShardedDataParallel as FSDP,MixedPrecision,ShardingStrategy,BackwardPrefetch from torch.distributed.fsdp.wrap import ModuleWrapPolicy from accelerate import init_empty_weights from transformers import AutoConfig from transformers.models.qwen3_5.modeling_qwen3_5 import Qwen3_5Model,Qwen3_5DecoderLayer,Qwen3_5VisionModel from flash_author.model import enable_noncausal_full_attention Decision=decision_class();rank=dist.get_rank();world=dist.get_world_size();shard_size=cfg['shard_size'] need(world==cfg['world']==8 and shard_size==8,'Exact approved single-host8 sharding contract') if rank==0: need(engine is not None,'Original rank0 reference body is the warmstart; no second body') engine.backbone.float();engine.readout.float();model=Decision(engine.backbone,engine.readout) else: config=AutoConfig.from_pretrained(cfg['staged_parent'],local_files_only=True);config._attn_implementation='sdpa' with init_empty_weights(include_buffers=False):model=Decision(Qwen3_5Model(config),torch.nn.Linear(2560,255,bias=False,dtype=torch.float32)) enable_noncausal_full_attention(model.backbone.language_model) from language_lora import install,configure_trainable if cfg['training_mode']=='language_lora': install(model.backbone,cfg['lora'],cfg['seed']) configure_trainable(model,cfg['training_mode']) model.backbone.enable_input_require_grads() declared=[] for name,param in model.named_parameters(): role='head' if name=='readout.weight' else 'vision' if '.visual.' in name else 'embedding' if 'embed_tokens' in name else 'language' declared.append({'name':name,'shape':list(param.shape),'dtype':str(param.dtype),'requires_grad':param.requires_grad,'role':role,'elements':param.numel()}) model.backbone.gradient_checkpointing_enable(gradient_checkpointing_kwargs={'use_reentrant':False}) def materialize(module): for name,param in list(module.named_parameters(recurse=False)): if param.is_meta:setattr(module,name,torch.nn.Parameter(torch.empty(param.shape,device=device,dtype=torch.float32),requires_grad=param.requires_grad)) for name,buf in list(module.named_buffers(recurse=False)): need(not buf.is_meta,'Nonpersistent semantic buffer cannot be invented');module._buffers[name]=buf.to(device) group=None;strategy=ShardingStrategy.FULL_SHARD if shard_size