"""SafeTensors-only loader for the step-93213 Sparse-AST old/new checkpoint.""" from pathlib import Path import json import torch from safetensors.torch import load_file from test_models import SparseAST from family_oldnew_architecture import SparseASTFamilyOldNew def checkpoint_directory(path_or_repo, revision=None, training=False): p=Path(path_or_repo) if p.is_dir():return p from huggingface_hub import snapshot_download patterns=['training_state.json','training_state.safetensors'] if training else ['config.json','model.safetensors'] return Path(snapshot_download(repo_id=str(path_or_repo),revision=revision,allow_patterns=['latest/'+n for n in patterns]))/'latest' def load_model(path_or_repo,device='cpu',revision=None): root=checkpoint_directory(path_or_repo,revision);cfg=json.loads((root/'config.json').read_text()) with torch.device('meta'): base=SparseAST(d=cfg['d_model'],h=cfg['d_hidden'],layers=cfg['num_layers'],al_hidden=cfg['al_hidden'],seq_len=cfg['position_embedding_capacity'],vocab=cfg['vocab_size']) model=SparseASTFamilyOldNew(base) state=load_file(str(root/'model.safetensors'),device='cpu') for alias,canonical in cfg['tensor_aliases'].items():state[alias]=state[canonical] model.load_state_dict(state,strict=True,assign=True) model.to(device);model.eval();model.training_context_bytes=cfg['training_context_bytes'] return model def decode_state(obj,tensors): if isinstance(obj,dict) and obj.get('__kind__')=='tensor':return tensors[obj['key']] if isinstance(obj,dict) and obj.get('__kind__')=='dict':return {decode_state(k,tensors):decode_state(v,tensors) for k,v in obj['items']} if isinstance(obj,dict) and obj.get('__kind__')=='tuple':return tuple(decode_state(v,tensors) for v in obj['items']) if isinstance(obj,list):return [decode_state(v,tensors) for v in obj] return obj def load_training_state(path_or_repo,revision=None): root=checkpoint_directory(path_or_repo,revision,training=True) return decode_state(json.loads((root/'training_state.json').read_text()),load_file(str(root/'training_state.safetensors'),device='cpu')) @torch.inference_mode() def generate(model,prompt,max_new_bytes=128,context_bytes=128): if not prompt:raise ValueError('Provide a nonempty UTF-8 prompt') if not 1<=context_bytes<=model.seq_len:raise ValueError('Context exceeds position embedding capacity') encoded=list(prompt.encode('utf-8'));device=next(model.parameters()).device for _ in range(max_new_bytes): logits=model(torch.tensor([encoded[-context_bytes:]],dtype=torch.long,device=device))[0,-1,:256] encoded.append(int(logits.argmax())) return bytes(encoded).decode('utf-8',errors='replace')