"""Publish the explicit validated inventory, then verify anonymous public hashes.""" import argparse import hashlib import json import os from pathlib import Path os.environ.setdefault('HF_XET_CACHE',str(Path(__file__).resolve().parents[1]/'.cache'/'hf-xet')) from huggingface_hub import HfApi,CommitOperationAdd from huggingface_hub.utils import RepositoryNotFoundError from package_release import assemble,sha256,ROOT def git_blob_sha(path): data=path.read_bytes() return hashlib.sha1(f'blob {len(data)}\0'.encode()+data).hexdigest() def publish(repo_id,upload=False): api=HfApi() identity=api.whoami()['name'] if repo_id.split('/',1)[0]!=identity: raise ValueError('destination owner does not match authenticated account') files=assemble() print(json.dumps(dict(account=identity,repo=repo_id,files=len(files),bytes=sum(p.stat().st_size for _,p in files))),flush=True) if not upload: return try: previous=api.model_info(repo_id,files_metadata=True) remote={p.rfilename:p for p in previous.siblings} for name,path in files: if name in remote: item=remote[name] expected=item.lfs.sha256 if item.lfs else item.blob_id actual=sha256(path) if item.lfs else git_blob_sha(path) if expected!=actual: raise ValueError(f'refusing to overwrite differing existing file: {name}') parent=previous.sha except RepositoryNotFoundError: api.create_repo(repo_id,repo_type='model',private=False,exist_ok=False) parent=api.model_info(repo_id).sha result=api.create_commit(repo_id=repo_id,repo_type='model',parent_commit=parent, operations=[CommitOperationAdd(path_in_repo=name,path_or_fileobj=str(path)) for name,path in files], commit_message='Publish all four Timestral MLX FP32 models, pipeline, original licences and validation') anonymous=HfApi(token=False) info=anonymous.model_info(repo_id,revision=result.oid,files_metadata=True) if info.private: raise ValueError('repository is not public') remote={p.rfilename:p for p in info.siblings} for name,path in files: item=remote.get(name) if item is None or item.size!=path.stat().st_size: raise ValueError(f'public inventory/size mismatch: {name}') if item.lfs: if item.lfs.sha256!=sha256(path): raise ValueError(f'public LFS hash mismatch: {name}') elif item.blob_id!=git_blob_sha(path): raise ValueError(f'public Git blob mismatch: {name}') if any(any(part.startswith('._') or part in {'.DS_Store','__MACOSX'} for part in name.split('/')) for name in remote): raise ValueError('unexpected Apple metadata in remote inventory') report=dict(repo_id=repo_id,revision=result.oid,commit_url=result.commit_url,public=True, files=len(files),bytes=sum(p.stat().st_size for _,p in files),remote_inventory_verified=True, anonymous_file_hashes_verified=True,apple_metadata_absent=True) (ROOT/'publication.json').write_text(json.dumps(report,indent=2)+'\n') print(json.dumps(report,indent=2),flush=True) if __name__=='__main__': p=argparse.ArgumentParser(description=__doc__) p.add_argument('--repo',default='jbilcke/timestral-mlx') p.add_argument('--upload',action='store_true') a=p.parse_args();publish(a.repo,a.upload)