Download code/scripts/publish.py from luyu1021/turbot2va-ltx2: direct link, hf CLI and curl.
- Browser
- Download file 21.3 kB
-
https://huggingface.co/luyu1021/turbot2va-ltx2/resolve/main/code/scripts/publish.py
- Command line
-
hf download hf://luyu1021/turbot2va-ltx2/code/scripts/publish.py
-
curl -L -o publish.py https://huggingface.co/luyu1021/turbot2va-ltx2/resolve/main/code/scripts/publish.py
21.3 kB
| #!/usr/bin/env python3 | |
| """Create the requested GitHub/HF repos, upload the allowlisted release, verify. | |
| Credentials come from GITHUB_TOKEN and HF_TOKEN, or an explicitly supplied | |
| --token-file. Token values never appear in command arguments or Git remotes. | |
| This script publishes only files committed to this new code repository and | |
| checkpoint files explicitly listed in release/upload_manifest.json. | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import hashlib | |
| import json | |
| import os | |
| from pathlib import Path | |
| import re | |
| import subprocess | |
| import sys | |
| from datetime import datetime, timezone | |
| import requests | |
| ROOT = Path(__file__).resolve().parents[1] | |
| SECRET = re.compile(rb'\b(?:hf_[A-Za-z0-9]{20,}|(?:ghp|gho|ghs|ghu|github_pat)_[A-Za-z0-9_]{20,})\b') | |
| def sha(path): | |
| h = hashlib.sha256() | |
| with path.open('rb') as stream: | |
| for chunk in iter(lambda: stream.read(16 * 1024**2), b''): | |
| h.update(chunk) | |
| return h.hexdigest() | |
| def get_tokens(args): | |
| gh, hf = os.environ.get('GITHUB_TOKEN'), os.environ.get('HF_TOKEN') | |
| if args.token_file: | |
| text = args.token_file.read_text() | |
| g = re.search(r'(?:ghp|gho|ghs|ghu|github_pat)_[A-Za-z0-9_]+', text) | |
| candidates = list(dict.fromkeys(re.findall(r'hf_[A-Za-z0-9]+', text))) | |
| gh = gh or (g.group() if g else None) | |
| if not hf and args.target in ('hf', 'both'): | |
| from huggingface_hub import HfApi | |
| for index, candidate in enumerate(candidates): | |
| identity = HfApi(token=candidate).whoami() | |
| role = identity.get('auth', {}).get('accessToken', {}).get('role') | |
| if role == 'read': | |
| print(f'HF credential {index + 1} is read-only; trying next credential.', flush=True) | |
| continue | |
| hf = candidate | |
| break | |
| if not hf and candidates: | |
| raise RuntimeError('The supplied HF credentials are read-only.') | |
| return gh, hf | |
| def git(*args, env=None): | |
| return subprocess.check_output(['git', '-C', str(ROOT), *args], env=env).decode().strip() | |
| def committed_files(): | |
| paths = git('ls-files', '-z').split('\0') | |
| files = [ROOT/p for p in paths if p] | |
| if not files or ROOT/'README.md' not in files: | |
| raise RuntimeError('Commit the curated code repository first.') | |
| if git('status', '--porcelain', '--untracked-files=no'): | |
| raise RuntimeError('Tracked code changes have not been committed.') | |
| for p in files: | |
| if p.is_symlink() or p.stat().st_size > 90*1024**2: | |
| raise RuntimeError(f'Unexpected GitHub file: {p.relative_to(ROOT)}') | |
| if p.suffix in ['.pth','.pt','.safetensors','.distcp','.mp4','.wav']: | |
| raise RuntimeError(f'Weight/media in GitHub list: {p.relative_to(ROOT)}') | |
| if SECRET.search(p.read_bytes()): | |
| raise RuntimeError(f'Credential-like content: {p.relative_to(ROOT)}') | |
| return files | |
| def api_call(session, method, path, token, **kwargs): | |
| response = session.request(method, 'https://api.github.com'+path, | |
| headers={'Authorization':'Bearer '+token, | |
| 'Accept':'application/vnd.github+json'}, timeout=30, **kwargs) | |
| return response | |
| def publish_github(args, token, status): | |
| if not token: | |
| raise RuntimeError('GITHUB_TOKEN is missing.') | |
| s = requests.Session() | |
| if args.direct: | |
| s.trust_env = False | |
| user = api_call(s,'GET','/user',token) | |
| user.raise_for_status() | |
| owner = args.github_owner or user.json()['login'] | |
| if owner != user.json()['login']: | |
| raise RuntimeError('This script creates in the authenticated personal GitHub namespace.') | |
| repo = f'{owner}/turbot2va-ltx2' | |
| previous = api_call(s,'GET',f'/repos/{repo}',token) | |
| if args.preflight: | |
| print(json.dumps({'github_user':owner,'existing_repo_status':previous.status_code})) | |
| return | |
| if previous.status_code == 404: | |
| created = api_call(s,'POST','/user/repos',token,json={'name':'turbot2va-ltx2', | |
| 'private':not args.public,'description':'LTX-2 audio-video DCM/SCM/RCM training, configs and checkpoint provenance'}) | |
| created.raise_for_status() | |
| data = created.json() | |
| else: | |
| previous.raise_for_status() | |
| data = previous.json() | |
| if data['private'] != (not args.public): | |
| raise RuntimeError('Existing repository visibility differs from requested visibility.') | |
| local_receipt = ROOT/'release/GITHUB_RECEIPT.json' | |
| if data.get('size',0) and not local_receipt.exists(): | |
| raise RuntimeError('Requested name already contains unrelated content; no overwrite performed.') | |
| status['github'] = {'repo':repo,'phase':'created','private':data['private']} | |
| save_status(status) | |
| env = os.environ.copy() | |
| env['TURBOT2VA_GITHUB_TOKEN'] = token | |
| if args.direct: | |
| for key in list(env): | |
| if key.lower().endswith('_proxy'): | |
| env.pop(key) | |
| # Askpass prints the credential only into Git's private auth pipe, never | |
| # stdout of this publisher and never a URL persisted in .git/config. | |
| askpass = ROOT/'release/.git_askpass.py' | |
| askpass.write_text('#!/usr/bin/env python3\nimport os,sys\nprint("x-access-token" if "username" in sys.argv[1].lower() else os.environ["TURBOT2VA_GITHUB_TOKEN"])\n') | |
| askpass.chmod(0o700) | |
| env.update(GIT_ASKPASS=str(askpass), GIT_TERMINAL_PROMPT='0') | |
| remote = f'https://github.com/{repo}.git' | |
| try: | |
| git('-c','credential.helper=','push',remote,'HEAD:refs/heads/main',env=env) | |
| finally: | |
| askpass.unlink(missing_ok=True) | |
| remote_ref = api_call(s,'GET',f'/repos/{repo}/git/ref/heads/main',token) | |
| remote_ref.raise_for_status() | |
| commit = git('rev-parse','HEAD') | |
| if remote_ref.json()['object']['sha'] != commit: | |
| raise RuntimeError('GitHub commit verification failed.') | |
| tree = api_call(s,'GET',f'/repos/{repo}/git/trees/{commit}?recursive=1',token) | |
| tree.raise_for_status() | |
| remote_files = {x['path']:x['sha'] for x in tree.json()['tree'] if x['type']=='blob'} | |
| expected = {str(p.relative_to(ROOT)):git('hash-object',str(p)) for p in committed_files()} | |
| if tree.json().get('truncated') or any(remote_files.get(k)!=v for k,v in expected.items()): | |
| raise RuntimeError('GitHub tree content verification failed.') | |
| receipt={'repo':repo,'url':data['html_url'],'commit':commit,'files_verified':len(expected), | |
| 'private':data['private'],'verified_at':datetime.now(timezone.utc).isoformat()} | |
| (ROOT/'release/GITHUB_RECEIPT.json').write_text(json.dumps(receipt,indent=2)+'\n') | |
| status['github']={**receipt,'phase':'verified'} | |
| save_status(status) | |
| print('GitHub verified:',repo,commit,len(expected),'files',flush=True) | |
| def save_status(status): | |
| (ROOT/'release/publish_status.json').write_text(json.dumps(status,ensure_ascii=False,indent=2)+'\n') | |
| def large_upload_files(artifacts, stage): | |
| """Allow only manifest source files and their generated release metadata. | |
| Exact paths include DCP's hidden .metadata. Following local symlinks reads | |
| original bytes without duplicating the checkpoint in staging. | |
| """ | |
| selected={} | |
| for artifact in artifacts: | |
| for entry in artifact['files']: | |
| name=entry['path_in_repo'] | |
| p=stage/name | |
| stat=p.stat() | |
| if stat.st_size!=entry['bytes'] or stat.st_mtime_ns!=entry['mtime_ns']: | |
| raise RuntimeError('Source checkpoint changed: '+name) | |
| selected[name]=p | |
| first=artifact['files'][0]['path_in_repo'] | |
| directory=('checkpoints/'+artifact['id'] if first.startswith('checkpoints/') | |
| else str(Path(first).parent)) | |
| name=directory+'/RELEASE_METADATA.json' | |
| p=stage/name | |
| metadata=json.loads(p.read_text()) | |
| if metadata.get('id')!=artifact['id']: | |
| raise RuntimeError('Unexpected release metadata: '+name) | |
| selected[name]=p | |
| return selected | |
| def large_upload_checksums(selected, stage): | |
| """Reuse the SDK's content hashes only for unchanged, committed files.""" | |
| from huggingface_hub._local_folder import read_upload_metadata | |
| expected={} | |
| for name,p in selected.items(): | |
| metadata=read_upload_metadata(stage,name) | |
| if not metadata.is_committed or not metadata.sha256 or metadata.size!=p.stat().st_size: | |
| raise RuntimeError('Missing completed large-upload metadata: '+name) | |
| expected[name]={'bytes':metadata.size,'sha256':metadata.sha256} | |
| return expected | |
| def publish_hf(args, token, status): | |
| if not token: | |
| raise RuntimeError('HF_TOKEN is missing.') | |
| from huggingface_hub import HfApi, CommitOperationAdd, hf_hub_url | |
| from huggingface_hub.errors import RepositoryNotFoundError | |
| if args.direct: | |
| from huggingface_hub import configure_http_backend | |
| def backend(): | |
| session=requests.Session();session.trust_env=False;return session | |
| configure_http_backend(backend_factory=backend) | |
| api = HfApi(token=token) | |
| user=api.whoami()['name'] | |
| owner=args.hf_owner or user | |
| repo=f'{owner}/turbot2va-ltx2' | |
| try: | |
| info=api.repo_info(repo,repo_type='model') | |
| except RepositoryNotFoundError: | |
| info=None | |
| if args.preflight: | |
| print(json.dumps({'hf_user':user,'repo':repo,'exists':info is not None})) | |
| return | |
| if info is None: | |
| api.create_repo(repo_id=repo,repo_type='model',private=not args.public,exist_ok=False) | |
| elif info.private != (not args.public): | |
| raise RuntimeError('Existing HF repo visibility differs; no visibility mutation performed.') | |
| elif len(info.siblings or [])>1 and not (ROOT/'release/HF_RECEIPT.json').exists(): | |
| status_file=ROOT/'release/publish_status.json' | |
| if status.get('hf',{}).get('repo')!=repo: | |
| raise RuntimeError('Requested HF name already contains unrelated content.') | |
| status['hf']={'repo':repo,'private':not args.public,'phase':'created'} | |
| save_status(status) | |
| # Upload documentation before large checkpoints, so interrupted uploads | |
| # remain readable and resumable without claiming completion. | |
| files=committed_files() | |
| operations=[CommitOperationAdd(path_in_repo='code/'+str(p.relative_to(ROOT)),path_or_fileobj=str(p)) for p in files] | |
| card=ROOT/'release/MODEL_CARD.md' | |
| operations.append(CommitOperationAdd(path_in_repo='README.md',path_or_fileobj=str(card))) | |
| operations.append(CommitOperationAdd(path_in_repo='LICENSE',path_or_fileobj=str(ROOT/'LICENSE'))) | |
| api.create_commit(repo_id=repo,repo_type='model',operations=operations, | |
| commit_message='Add documented LTX-2 training source, recipes and checkpoint release plan') | |
| if args.code_only: | |
| info=api.repo_info(repo,repo_type='model',files_metadata=True) | |
| remote={f.rfilename:f for f in info.siblings} | |
| for p in files: | |
| entry=remote.get('code/'+str(p.relative_to(ROOT))) | |
| if entry is None or entry.size!=p.stat().st_size: | |
| raise RuntimeError('HF documentation verification failed: '+str(p.relative_to(ROOT))) | |
| for name,path in [('README.md',card),('LICENSE',ROOT/'LICENSE')]: | |
| entry=remote.get(name) | |
| if entry is None or entry.size!=path.stat().st_size: | |
| raise RuntimeError('HF root documentation verification failed: '+name) | |
| # A documentation refresh must not erase a completed weight upload. | |
| # Keep its verification revision separate from this new code revision. | |
| receipt_path=ROOT/'release/HF_RECEIPT.json' | |
| previous=json.loads(receipt_path.read_text()) if receipt_path.exists() else {} | |
| if previous.get('repo')==repo: | |
| status['hf'].update(checkpoint_receipt=previous) | |
| status['hf'].update(phase='documentation_verified',revision=info.sha, | |
| source_files_verified=len(files), | |
| checkpoints_present=any(name.startswith(('inference/','checkpoints/')) for name in remote), | |
| weights_uploaded_this_call=False) | |
| documentation_receipt={'repo':repo,'private':info.private,'revision':info.sha, | |
| 'source_files_verified':len(files),'root_files_verified':2, | |
| 'weights_uploaded_this_call':False, | |
| 'verified_at':datetime.now(timezone.utc).isoformat()} | |
| (ROOT/'release/HF_DOCUMENTATION_RECEIPT.json').write_text(json.dumps(documentation_receipt,indent=2)+'\n') | |
| save_status(status) | |
| print('HF documentation verified:',repo,info.sha,len(files),'source files',flush=True) | |
| return | |
| plan=json.loads((ROOT/'release/upload_manifest.json').read_text()) | |
| artifacts=plan.get('inference_artifacts', []) + plan['artifacts'] | |
| if args.artifact: | |
| artifacts=[a for a in artifacts if a['id'] in args.artifact] | |
| missing=set(args.artifact)-{a['id'] for a in artifacts} | |
| if missing: | |
| raise RuntimeError('Unknown artifact IDs: '+', '.join(sorted(missing))) | |
| expected={} | |
| cache_path=ROOT/'release/checkpoint_checksums.json' | |
| cached=json.loads(cache_path.read_text()) if cache_path.exists() else {} | |
| if getattr(args,'large_folder',False): | |
| selected=large_upload_files(artifacts,args.hf_stage) | |
| status['hf'].update(phase='uploading',mode='resumable_large_folder', | |
| selected_artifacts=[a['id'] for a in artifacts], | |
| selected_files=len(selected), | |
| selected_bytes=sum(p.stat().st_size for p in selected.values())) | |
| save_status(status) | |
| print('Resumable upload:',len(selected),'files,',status['hf']['selected_bytes'],'bytes',flush=True) | |
| # The SDK caches hashing, LFS transfer and commit state independently. | |
| # A restart resumes completed tasks. The allowlist excludes staging | |
| # caches and anything outside the selected checkpoint directories. | |
| api.upload_large_folder(repo_id=repo,repo_type='model',folder_path=args.hf_stage, | |
| private=not args.public,allow_patterns=list(selected), | |
| num_workers=args.upload_workers,print_report_every=60) | |
| large_upload_files(artifacts,args.hf_stage) # Refuse changes during transfer. | |
| expected=large_upload_checksums(selected,args.hf_stage) | |
| for artifact in ([] if getattr(args,'large_folder',False) else artifacts): | |
| print('Preparing artifact:',artifact['id'],flush=True) | |
| remote_directory = str(Path(artifact['files'][0]['path_in_repo']).parent) | |
| # Model DCP files live below sharded/, while the artifact upload root | |
| # includes its metadata, optimizer and RNG siblings as well. | |
| if remote_directory.startswith('checkpoints/'): | |
| remote_directory='checkpoints/'+artifact['id'] | |
| stage=args.hf_stage/remote_directory | |
| for entry in artifact['files']: | |
| p=args.hf_stage/entry['path_in_repo'] | |
| stat=p.stat() | |
| if stat.st_size!=entry['bytes'] or stat.st_mtime_ns!=entry['mtime_ns']: | |
| raise RuntimeError(f'Source checkpoint changed: {entry["path_in_repo"]}') | |
| old=cached.get(entry['path_in_repo'],{}) | |
| if old.get('bytes')!=stat.st_size or old.get('mtime_ns')!=stat.st_mtime_ns: | |
| old={'bytes':stat.st_size,'mtime_ns':stat.st_mtime_ns,'sha256':sha(p)} | |
| cached[entry['path_in_repo']]=old | |
| cache_path.write_text(json.dumps(cached,indent=2)+'\n') | |
| expected[entry['path_in_repo']]={'bytes':stat.st_size,'sha256':old['sha256']} | |
| status['hf'].update(phase='uploading',artifact=artifact['id']);save_status(status) | |
| # upload_folder uploads this bounded artifact (<= ~227 GB), and the | |
| # server deduplicates already uploaded LFS data on a repeated attempt. | |
| api.upload_folder(repo_id=repo,repo_type='model',folder_path=str(stage), | |
| path_in_repo=remote_directory, | |
| commit_message='Add '+artifact['id'],ignore_patterns=['.cache/**']) | |
| for p in stage.rglob('*'): | |
| if p.is_file() and not '.cache' in p.parts: | |
| name=str(p.relative_to(args.hf_stage)) | |
| if name not in expected: | |
| expected[name]={'bytes':p.stat().st_size,'sha256':sha(p)} | |
| checksum_file=ROOT/'release/HF_CHECKSUMS.json' | |
| checksum_file.write_text(json.dumps(expected,indent=2)+'\n') | |
| api.upload_file(repo_id=repo,repo_type='model',path_or_fileobj=str(checksum_file), | |
| path_in_repo='CHECKSUMS.json',commit_message='Record checkpoint SHA-256 checksums') | |
| info=api.repo_info(repo,repo_type='model',files_metadata=True) | |
| remote={f.rfilename:f for f in info.siblings} | |
| verified=0 | |
| for name,local in expected.items(): | |
| f=remote.get(name) | |
| if f is None or f.size!=local['bytes']: | |
| raise RuntimeError('HF size/file verification failed: '+name) | |
| lfs=f.lfs | |
| remote_sha=(lfs.get('sha256') if isinstance(lfs,dict) else getattr(lfs,'sha256',None)) if lfs else None | |
| if remote_sha: | |
| if remote_sha!=local['sha256']: | |
| raise RuntimeError('HF SHA256 mismatch: '+name) | |
| elif local['bytes']<20*1024**2: | |
| from huggingface_hub.utils import get_session | |
| r=get_session().get(hf_hub_url(repo,name,revision=info.sha),headers={'Authorization':'Bearer '+token},timeout=60) | |
| r.raise_for_status() | |
| if hashlib.sha256(r.content).hexdigest()!=local['sha256']: | |
| raise RuntimeError('HF downloaded SHA256 mismatch: '+name) | |
| else: | |
| raise RuntimeError('No verifiable remote SHA256 for large file: '+name) | |
| verified+=1 | |
| # Also verify uploaded source paths and sizes, which share the GitHub tree. | |
| for p in files: | |
| f=remote.get('code/'+str(p.relative_to(ROOT))) | |
| if f is None or f.size!=p.stat().st_size: | |
| raise RuntimeError('HF code verification failed: '+str(p.relative_to(ROOT))) | |
| receipt={'repo':repo,'repo_type':'model','private':info.private,'revision':info.sha, | |
| 'verified_artifacts':[a['id'] for a in artifacts], | |
| 'all_planned_artifacts_verified':len(artifacts)==len(plan['artifacts'])+len(plan.get('inference_artifacts',[])), | |
| 'checkpoint_files_verified':verified,'source_files_verified':len(files), | |
| 'checkpoint_bytes':sum(x['bytes'] for x in expected.values()), | |
| 'verified_at':datetime.now(timezone.utc).isoformat()} | |
| (ROOT/'release/HF_RECEIPT.json').write_text(json.dumps(receipt,indent=2)+'\n') | |
| status['hf']={**receipt,'phase':'verified'};save_status(status) | |
| print('HF verified:',repo,info.sha,verified,'checkpoint files',flush=True) | |
| def main(): | |
| parser=argparse.ArgumentParser(description=__doc__) | |
| parser.add_argument('--token-file',type=Path) | |
| parser.add_argument('--github-owner') | |
| parser.add_argument('--hf-owner') | |
| parser.add_argument('--hf-stage',type=Path,default=ROOT.parents[1]/'output/releases/turbot2va-ltx2-hf') | |
| parser.add_argument('--public',action='store_true',help='Default is private.') | |
| parser.add_argument('--target',choices=['github','hf','both'],default='both') | |
| parser.add_argument('--code-only',action='store_true',help='Upload and verify HF documentation without checkpoints.') | |
| parser.add_argument('--artifact',action='append',help='Upload only this manifest artifact ID; may be repeated.') | |
| parser.add_argument('--large-folder',action='store_true',help='Use resumable SDK hashing, LFS transfers and commits for the selected allowlist.') | |
| parser.add_argument('--upload-workers',type=int,default=8,help='Concurrent workers for --large-folder (default: 8).') | |
| parser.add_argument('--preflight',action='store_true',help='Check identity/access without creating or uploading.') | |
| parser.add_argument('--direct',action='store_true',help='Ignore process proxy environment for this attempt.') | |
| args=parser.parse_args() | |
| if args.upload_workers<2: | |
| parser.error('--upload-workers must be at least 2') | |
| gh,hf=get_tokens(args) | |
| committed_files() | |
| path=ROOT/'release/publish_status.json' | |
| status=json.loads(path.read_text()) if path.exists() else {} | |
| status['attempted_at']=datetime.now(timezone.utc).isoformat() | |
| failures=[] | |
| for target,func,token in [('github',publish_github,gh),('hf',publish_hf,hf)]: | |
| if args.target not in ['both',target]:continue | |
| try: | |
| func(args,token,status) | |
| except Exception as e: | |
| # Requests exceptions may contain long proxy traces, but no | |
| # authorization headers are printed. Scrub recognized token forms. | |
| detail=re.sub(SECRET.pattern.decode(), '[REDACTED]', str(e))[:1000] | |
| status.setdefault(target,{}).update(error_type=type(e).__name__,error=detail) | |
| save_status(status) | |
| print(f'{target}: {type(e).__name__}: {detail}',file=sys.stderr,flush=True) | |
| failures.append(target) | |
| if failures:raise SystemExit(1) | |
| if __name__=='__main__':main() | |