File size: 3,412 Bytes
d35e174 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 | """Publish the explicit validated inventory, then verify anonymous public hashes."""
import argparse
import hashlib
import json
import os
from pathlib import Path
os.environ.setdefault('HF_XET_CACHE',str(Path(__file__).resolve().parents[1]/'.cache'/'hf-xet'))
from huggingface_hub import HfApi,CommitOperationAdd
from huggingface_hub.utils import RepositoryNotFoundError
from package_release import assemble,sha256,ROOT
def git_blob_sha(path):
data=path.read_bytes()
return hashlib.sha1(f'blob {len(data)}\0'.encode()+data).hexdigest()
def publish(repo_id,upload=False):
api=HfApi()
identity=api.whoami()['name']
if repo_id.split('/',1)[0]!=identity:
raise ValueError('destination owner does not match authenticated account')
files=assemble()
print(json.dumps(dict(account=identity,repo=repo_id,files=len(files),bytes=sum(p.stat().st_size for _,p in files))),flush=True)
if not upload: return
try:
previous=api.model_info(repo_id,files_metadata=True)
remote={p.rfilename:p for p in previous.siblings}
for name,path in files:
if name in remote:
item=remote[name]
expected=item.lfs.sha256 if item.lfs else item.blob_id
actual=sha256(path) if item.lfs else git_blob_sha(path)
if expected!=actual:
raise ValueError(f'refusing to overwrite differing existing file: {name}')
parent=previous.sha
except RepositoryNotFoundError:
api.create_repo(repo_id,repo_type='model',private=False,exist_ok=False)
parent=api.model_info(repo_id).sha
result=api.create_commit(repo_id=repo_id,repo_type='model',parent_commit=parent,
operations=[CommitOperationAdd(path_in_repo=name,path_or_fileobj=str(path)) for name,path in files],
commit_message='Publish all four Timestral MLX FP32 models, pipeline, original licences and validation')
anonymous=HfApi(token=False)
info=anonymous.model_info(repo_id,revision=result.oid,files_metadata=True)
if info.private: raise ValueError('repository is not public')
remote={p.rfilename:p for p in info.siblings}
for name,path in files:
item=remote.get(name)
if item is None or item.size!=path.stat().st_size:
raise ValueError(f'public inventory/size mismatch: {name}')
if item.lfs:
if item.lfs.sha256!=sha256(path): raise ValueError(f'public LFS hash mismatch: {name}')
elif item.blob_id!=git_blob_sha(path): raise ValueError(f'public Git blob mismatch: {name}')
if any(any(part.startswith('._') or part in {'.DS_Store','__MACOSX'} for part in name.split('/')) for name in remote):
raise ValueError('unexpected Apple metadata in remote inventory')
report=dict(repo_id=repo_id,revision=result.oid,commit_url=result.commit_url,public=True,
files=len(files),bytes=sum(p.stat().st_size for _,p in files),remote_inventory_verified=True,
anonymous_file_hashes_verified=True,apple_metadata_absent=True)
(ROOT/'publication.json').write_text(json.dumps(report,indent=2)+'\n')
print(json.dumps(report,indent=2),flush=True)
if __name__=='__main__':
p=argparse.ArgumentParser(description=__doc__)
p.add_argument('--repo',default='jbilcke/timestral-mlx')
p.add_argument('--upload',action='store_true')
a=p.parse_args();publish(a.repo,a.upload)
|