| """Publish the explicit validated inventory, then verify anonymous public hashes.""" |
| import argparse |
| import hashlib |
| import json |
| import os |
| from pathlib import Path |
| os.environ.setdefault('HF_XET_CACHE',str(Path(__file__).resolve().parents[1]/'.cache'/'hf-xet')) |
| from huggingface_hub import HfApi,CommitOperationAdd |
| from huggingface_hub.utils import RepositoryNotFoundError |
| from package_release import assemble,sha256,ROOT |
|
|
|
|
| def git_blob_sha(path): |
| data=path.read_bytes() |
| return hashlib.sha1(f'blob {len(data)}\0'.encode()+data).hexdigest() |
|
|
|
|
| def publish(repo_id,upload=False): |
| api=HfApi() |
| identity=api.whoami()['name'] |
| if repo_id.split('/',1)[0]!=identity: |
| raise ValueError('destination owner does not match authenticated account') |
| files=assemble() |
| print(json.dumps(dict(account=identity,repo=repo_id,files=len(files),bytes=sum(p.stat().st_size for _,p in files))),flush=True) |
| if not upload: return |
| try: |
| previous=api.model_info(repo_id,files_metadata=True) |
| remote={p.rfilename:p for p in previous.siblings} |
| for name,path in files: |
| if name in remote: |
| item=remote[name] |
| expected=item.lfs.sha256 if item.lfs else item.blob_id |
| actual=sha256(path) if item.lfs else git_blob_sha(path) |
| if expected!=actual: |
| raise ValueError(f'refusing to overwrite differing existing file: {name}') |
| parent=previous.sha |
| except RepositoryNotFoundError: |
| api.create_repo(repo_id,repo_type='model',private=False,exist_ok=False) |
| parent=api.model_info(repo_id).sha |
| result=api.create_commit(repo_id=repo_id,repo_type='model',parent_commit=parent, |
| operations=[CommitOperationAdd(path_in_repo=name,path_or_fileobj=str(path)) for name,path in files], |
| commit_message='Publish all four Timestral MLX FP32 models, pipeline, original licences and validation') |
| anonymous=HfApi(token=False) |
| info=anonymous.model_info(repo_id,revision=result.oid,files_metadata=True) |
| if info.private: raise ValueError('repository is not public') |
| remote={p.rfilename:p for p in info.siblings} |
| for name,path in files: |
| item=remote.get(name) |
| if item is None or item.size!=path.stat().st_size: |
| raise ValueError(f'public inventory/size mismatch: {name}') |
| if item.lfs: |
| if item.lfs.sha256!=sha256(path): raise ValueError(f'public LFS hash mismatch: {name}') |
| elif item.blob_id!=git_blob_sha(path): raise ValueError(f'public Git blob mismatch: {name}') |
| if any(any(part.startswith('._') or part in {'.DS_Store','__MACOSX'} for part in name.split('/')) for name in remote): |
| raise ValueError('unexpected Apple metadata in remote inventory') |
| report=dict(repo_id=repo_id,revision=result.oid,commit_url=result.commit_url,public=True, |
| files=len(files),bytes=sum(p.stat().st_size for _,p in files),remote_inventory_verified=True, |
| anonymous_file_hashes_verified=True,apple_metadata_absent=True) |
| (ROOT/'publication.json').write_text(json.dumps(report,indent=2)+'\n') |
| print(json.dumps(report,indent=2),flush=True) |
|
|
|
|
| if __name__=='__main__': |
| p=argparse.ArgumentParser(description=__doc__) |
| p.add_argument('--repo',default='jbilcke/timestral-mlx') |
| p.add_argument('--upload',action='store_true') |
| a=p.parse_args();publish(a.repo,a.upload) |
|
|
