#!/usr/bin/env python3 """Create the requested GitHub/HF repos, upload the allowlisted release, verify. Credentials come from GITHUB_TOKEN and HF_TOKEN, or an explicitly supplied --token-file. Token values never appear in command arguments or Git remotes. This script publishes only files committed to this new code repository and checkpoint files explicitly listed in release/upload_manifest.json. """ from __future__ import annotations import argparse import hashlib import json import os from pathlib import Path import re import subprocess import sys from datetime import datetime, timezone import requests ROOT = Path(__file__).resolve().parents[1] SECRET = re.compile(rb'\b(?:hf_[A-Za-z0-9]{20,}|(?:ghp|gho|ghs|ghu|github_pat)_[A-Za-z0-9_]{20,})\b') def sha(path): h = hashlib.sha256() with path.open('rb') as stream: for chunk in iter(lambda: stream.read(16 * 1024**2), b''): h.update(chunk) return h.hexdigest() def get_tokens(args): gh, hf = os.environ.get('GITHUB_TOKEN'), os.environ.get('HF_TOKEN') if args.token_file: text = args.token_file.read_text() g = re.search(r'(?:ghp|gho|ghs|ghu|github_pat)_[A-Za-z0-9_]+', text) candidates = list(dict.fromkeys(re.findall(r'hf_[A-Za-z0-9]+', text))) gh = gh or (g.group() if g else None) if not hf and args.target in ('hf', 'both'): from huggingface_hub import HfApi for index, candidate in enumerate(candidates): identity = HfApi(token=candidate).whoami() role = identity.get('auth', {}).get('accessToken', {}).get('role') if role == 'read': print(f'HF credential {index + 1} is read-only; trying next credential.', flush=True) continue hf = candidate break if not hf and candidates: raise RuntimeError('The supplied HF credentials are read-only.') return gh, hf def git(*args, env=None): return subprocess.check_output(['git', '-C', str(ROOT), *args], env=env).decode().strip() def committed_files(): paths = git('ls-files', '-z').split('\0') files = [ROOT/p for p in paths if p] if not files or ROOT/'README.md' not in files: raise RuntimeError('Commit the curated code repository first.') if git('status', '--porcelain', '--untracked-files=no'): raise RuntimeError('Tracked code changes have not been committed.') for p in files: if p.is_symlink() or p.stat().st_size > 90*1024**2: raise RuntimeError(f'Unexpected GitHub file: {p.relative_to(ROOT)}') if p.suffix in ['.pth','.pt','.safetensors','.distcp','.mp4','.wav']: raise RuntimeError(f'Weight/media in GitHub list: {p.relative_to(ROOT)}') if SECRET.search(p.read_bytes()): raise RuntimeError(f'Credential-like content: {p.relative_to(ROOT)}') return files def api_call(session, method, path, token, **kwargs): response = session.request(method, 'https://api.github.com'+path, headers={'Authorization':'Bearer '+token, 'Accept':'application/vnd.github+json'}, timeout=30, **kwargs) return response def publish_github(args, token, status): if not token: raise RuntimeError('GITHUB_TOKEN is missing.') s = requests.Session() if args.direct: s.trust_env = False user = api_call(s,'GET','/user',token) user.raise_for_status() owner = args.github_owner or user.json()['login'] if owner != user.json()['login']: raise RuntimeError('This script creates in the authenticated personal GitHub namespace.') repo = f'{owner}/turbot2va-ltx2' previous = api_call(s,'GET',f'/repos/{repo}',token) if args.preflight: print(json.dumps({'github_user':owner,'existing_repo_status':previous.status_code})) return if previous.status_code == 404: created = api_call(s,'POST','/user/repos',token,json={'name':'turbot2va-ltx2', 'private':not args.public,'description':'LTX-2 audio-video DCM/SCM/RCM training, configs and checkpoint provenance'}) created.raise_for_status() data = created.json() else: previous.raise_for_status() data = previous.json() if data['private'] != (not args.public): raise RuntimeError('Existing repository visibility differs from requested visibility.') local_receipt = ROOT/'release/GITHUB_RECEIPT.json' if data.get('size',0) and not local_receipt.exists(): raise RuntimeError('Requested name already contains unrelated content; no overwrite performed.') status['github'] = {'repo':repo,'phase':'created','private':data['private']} save_status(status) env = os.environ.copy() env['TURBOT2VA_GITHUB_TOKEN'] = token if args.direct: for key in list(env): if key.lower().endswith('_proxy'): env.pop(key) # Askpass prints the credential only into Git's private auth pipe, never # stdout of this publisher and never a URL persisted in .git/config. askpass = ROOT/'release/.git_askpass.py' askpass.write_text('#!/usr/bin/env python3\nimport os,sys\nprint("x-access-token" if "username" in sys.argv[1].lower() else os.environ["TURBOT2VA_GITHUB_TOKEN"])\n') askpass.chmod(0o700) env.update(GIT_ASKPASS=str(askpass), GIT_TERMINAL_PROMPT='0') remote = f'https://github.com/{repo}.git' try: git('-c','credential.helper=','push',remote,'HEAD:refs/heads/main',env=env) finally: askpass.unlink(missing_ok=True) remote_ref = api_call(s,'GET',f'/repos/{repo}/git/ref/heads/main',token) remote_ref.raise_for_status() commit = git('rev-parse','HEAD') if remote_ref.json()['object']['sha'] != commit: raise RuntimeError('GitHub commit verification failed.') tree = api_call(s,'GET',f'/repos/{repo}/git/trees/{commit}?recursive=1',token) tree.raise_for_status() remote_files = {x['path']:x['sha'] for x in tree.json()['tree'] if x['type']=='blob'} expected = {str(p.relative_to(ROOT)):git('hash-object',str(p)) for p in committed_files()} if tree.json().get('truncated') or any(remote_files.get(k)!=v for k,v in expected.items()): raise RuntimeError('GitHub tree content verification failed.') receipt={'repo':repo,'url':data['html_url'],'commit':commit,'files_verified':len(expected), 'private':data['private'],'verified_at':datetime.now(timezone.utc).isoformat()} (ROOT/'release/GITHUB_RECEIPT.json').write_text(json.dumps(receipt,indent=2)+'\n') status['github']={**receipt,'phase':'verified'} save_status(status) print('GitHub verified:',repo,commit,len(expected),'files',flush=True) def save_status(status): (ROOT/'release/publish_status.json').write_text(json.dumps(status,ensure_ascii=False,indent=2)+'\n') def large_upload_files(artifacts, stage): """Allow only manifest source files and their generated release metadata. Exact paths include DCP's hidden .metadata. Following local symlinks reads original bytes without duplicating the checkpoint in staging. """ selected={} for artifact in artifacts: for entry in artifact['files']: name=entry['path_in_repo'] p=stage/name stat=p.stat() if stat.st_size!=entry['bytes'] or stat.st_mtime_ns!=entry['mtime_ns']: raise RuntimeError('Source checkpoint changed: '+name) selected[name]=p first=artifact['files'][0]['path_in_repo'] directory=('checkpoints/'+artifact['id'] if first.startswith('checkpoints/') else str(Path(first).parent)) name=directory+'/RELEASE_METADATA.json' p=stage/name metadata=json.loads(p.read_text()) if metadata.get('id')!=artifact['id']: raise RuntimeError('Unexpected release metadata: '+name) selected[name]=p return selected def large_upload_checksums(selected, stage): """Reuse the SDK's content hashes only for unchanged, committed files.""" from huggingface_hub._local_folder import read_upload_metadata expected={} for name,p in selected.items(): metadata=read_upload_metadata(stage,name) if not metadata.is_committed or not metadata.sha256 or metadata.size!=p.stat().st_size: raise RuntimeError('Missing completed large-upload metadata: '+name) expected[name]={'bytes':metadata.size,'sha256':metadata.sha256} return expected def publish_hf(args, token, status): if not token: raise RuntimeError('HF_TOKEN is missing.') from huggingface_hub import HfApi, CommitOperationAdd, hf_hub_url from huggingface_hub.errors import RepositoryNotFoundError if args.direct: from huggingface_hub import configure_http_backend def backend(): session=requests.Session();session.trust_env=False;return session configure_http_backend(backend_factory=backend) api = HfApi(token=token) user=api.whoami()['name'] owner=args.hf_owner or user repo=f'{owner}/turbot2va-ltx2' try: info=api.repo_info(repo,repo_type='model') except RepositoryNotFoundError: info=None if args.preflight: print(json.dumps({'hf_user':user,'repo':repo,'exists':info is not None})) return if info is None: api.create_repo(repo_id=repo,repo_type='model',private=not args.public,exist_ok=False) elif info.private != (not args.public): raise RuntimeError('Existing HF repo visibility differs; no visibility mutation performed.') elif len(info.siblings or [])>1 and not (ROOT/'release/HF_RECEIPT.json').exists(): status_file=ROOT/'release/publish_status.json' if status.get('hf',{}).get('repo')!=repo: raise RuntimeError('Requested HF name already contains unrelated content.') status['hf']={'repo':repo,'private':not args.public,'phase':'created'} save_status(status) # Upload documentation before large checkpoints, so interrupted uploads # remain readable and resumable without claiming completion. files=committed_files() operations=[CommitOperationAdd(path_in_repo='code/'+str(p.relative_to(ROOT)),path_or_fileobj=str(p)) for p in files] card=ROOT/'release/MODEL_CARD.md' operations.append(CommitOperationAdd(path_in_repo='README.md',path_or_fileobj=str(card))) operations.append(CommitOperationAdd(path_in_repo='LICENSE',path_or_fileobj=str(ROOT/'LICENSE'))) api.create_commit(repo_id=repo,repo_type='model',operations=operations, commit_message='Add documented LTX-2 training source, recipes and checkpoint release plan') if args.code_only: info=api.repo_info(repo,repo_type='model',files_metadata=True) remote={f.rfilename:f for f in info.siblings} for p in files: entry=remote.get('code/'+str(p.relative_to(ROOT))) if entry is None or entry.size!=p.stat().st_size: raise RuntimeError('HF documentation verification failed: '+str(p.relative_to(ROOT))) for name,path in [('README.md',card),('LICENSE',ROOT/'LICENSE')]: entry=remote.get(name) if entry is None or entry.size!=path.stat().st_size: raise RuntimeError('HF root documentation verification failed: '+name) # A documentation refresh must not erase a completed weight upload. # Keep its verification revision separate from this new code revision. receipt_path=ROOT/'release/HF_RECEIPT.json' previous=json.loads(receipt_path.read_text()) if receipt_path.exists() else {} if previous.get('repo')==repo: status['hf'].update(checkpoint_receipt=previous) status['hf'].update(phase='documentation_verified',revision=info.sha, source_files_verified=len(files), checkpoints_present=any(name.startswith(('inference/','checkpoints/')) for name in remote), weights_uploaded_this_call=False) documentation_receipt={'repo':repo,'private':info.private,'revision':info.sha, 'source_files_verified':len(files),'root_files_verified':2, 'weights_uploaded_this_call':False, 'verified_at':datetime.now(timezone.utc).isoformat()} (ROOT/'release/HF_DOCUMENTATION_RECEIPT.json').write_text(json.dumps(documentation_receipt,indent=2)+'\n') save_status(status) print('HF documentation verified:',repo,info.sha,len(files),'source files',flush=True) return plan=json.loads((ROOT/'release/upload_manifest.json').read_text()) artifacts=plan.get('inference_artifacts', []) + plan['artifacts'] if args.artifact: artifacts=[a for a in artifacts if a['id'] in args.artifact] missing=set(args.artifact)-{a['id'] for a in artifacts} if missing: raise RuntimeError('Unknown artifact IDs: '+', '.join(sorted(missing))) expected={} cache_path=ROOT/'release/checkpoint_checksums.json' cached=json.loads(cache_path.read_text()) if cache_path.exists() else {} if getattr(args,'large_folder',False): selected=large_upload_files(artifacts,args.hf_stage) status['hf'].update(phase='uploading',mode='resumable_large_folder', selected_artifacts=[a['id'] for a in artifacts], selected_files=len(selected), selected_bytes=sum(p.stat().st_size for p in selected.values())) save_status(status) print('Resumable upload:',len(selected),'files,',status['hf']['selected_bytes'],'bytes',flush=True) # The SDK caches hashing, LFS transfer and commit state independently. # A restart resumes completed tasks. The allowlist excludes staging # caches and anything outside the selected checkpoint directories. api.upload_large_folder(repo_id=repo,repo_type='model',folder_path=args.hf_stage, private=not args.public,allow_patterns=list(selected), num_workers=args.upload_workers,print_report_every=60) large_upload_files(artifacts,args.hf_stage) # Refuse changes during transfer. expected=large_upload_checksums(selected,args.hf_stage) for artifact in ([] if getattr(args,'large_folder',False) else artifacts): print('Preparing artifact:',artifact['id'],flush=True) remote_directory = str(Path(artifact['files'][0]['path_in_repo']).parent) # Model DCP files live below sharded/, while the artifact upload root # includes its metadata, optimizer and RNG siblings as well. if remote_directory.startswith('checkpoints/'): remote_directory='checkpoints/'+artifact['id'] stage=args.hf_stage/remote_directory for entry in artifact['files']: p=args.hf_stage/entry['path_in_repo'] stat=p.stat() if stat.st_size!=entry['bytes'] or stat.st_mtime_ns!=entry['mtime_ns']: raise RuntimeError(f'Source checkpoint changed: {entry["path_in_repo"]}') old=cached.get(entry['path_in_repo'],{}) if old.get('bytes')!=stat.st_size or old.get('mtime_ns')!=stat.st_mtime_ns: old={'bytes':stat.st_size,'mtime_ns':stat.st_mtime_ns,'sha256':sha(p)} cached[entry['path_in_repo']]=old cache_path.write_text(json.dumps(cached,indent=2)+'\n') expected[entry['path_in_repo']]={'bytes':stat.st_size,'sha256':old['sha256']} status['hf'].update(phase='uploading',artifact=artifact['id']);save_status(status) # upload_folder uploads this bounded artifact (<= ~227 GB), and the # server deduplicates already uploaded LFS data on a repeated attempt. api.upload_folder(repo_id=repo,repo_type='model',folder_path=str(stage), path_in_repo=remote_directory, commit_message='Add '+artifact['id'],ignore_patterns=['.cache/**']) for p in stage.rglob('*'): if p.is_file() and not '.cache' in p.parts: name=str(p.relative_to(args.hf_stage)) if name not in expected: expected[name]={'bytes':p.stat().st_size,'sha256':sha(p)} checksum_file=ROOT/'release/HF_CHECKSUMS.json' checksum_file.write_text(json.dumps(expected,indent=2)+'\n') api.upload_file(repo_id=repo,repo_type='model',path_or_fileobj=str(checksum_file), path_in_repo='CHECKSUMS.json',commit_message='Record checkpoint SHA-256 checksums') info=api.repo_info(repo,repo_type='model',files_metadata=True) remote={f.rfilename:f for f in info.siblings} verified=0 for name,local in expected.items(): f=remote.get(name) if f is None or f.size!=local['bytes']: raise RuntimeError('HF size/file verification failed: '+name) lfs=f.lfs remote_sha=(lfs.get('sha256') if isinstance(lfs,dict) else getattr(lfs,'sha256',None)) if lfs else None if remote_sha: if remote_sha!=local['sha256']: raise RuntimeError('HF SHA256 mismatch: '+name) elif local['bytes']<20*1024**2: from huggingface_hub.utils import get_session r=get_session().get(hf_hub_url(repo,name,revision=info.sha),headers={'Authorization':'Bearer '+token},timeout=60) r.raise_for_status() if hashlib.sha256(r.content).hexdigest()!=local['sha256']: raise RuntimeError('HF downloaded SHA256 mismatch: '+name) else: raise RuntimeError('No verifiable remote SHA256 for large file: '+name) verified+=1 # Also verify uploaded source paths and sizes, which share the GitHub tree. for p in files: f=remote.get('code/'+str(p.relative_to(ROOT))) if f is None or f.size!=p.stat().st_size: raise RuntimeError('HF code verification failed: '+str(p.relative_to(ROOT))) receipt={'repo':repo,'repo_type':'model','private':info.private,'revision':info.sha, 'verified_artifacts':[a['id'] for a in artifacts], 'all_planned_artifacts_verified':len(artifacts)==len(plan['artifacts'])+len(plan.get('inference_artifacts',[])), 'checkpoint_files_verified':verified,'source_files_verified':len(files), 'checkpoint_bytes':sum(x['bytes'] for x in expected.values()), 'verified_at':datetime.now(timezone.utc).isoformat()} (ROOT/'release/HF_RECEIPT.json').write_text(json.dumps(receipt,indent=2)+'\n') status['hf']={**receipt,'phase':'verified'};save_status(status) print('HF verified:',repo,info.sha,verified,'checkpoint files',flush=True) def main(): parser=argparse.ArgumentParser(description=__doc__) parser.add_argument('--token-file',type=Path) parser.add_argument('--github-owner') parser.add_argument('--hf-owner') parser.add_argument('--hf-stage',type=Path,default=ROOT.parents[1]/'output/releases/turbot2va-ltx2-hf') parser.add_argument('--public',action='store_true',help='Default is private.') parser.add_argument('--target',choices=['github','hf','both'],default='both') parser.add_argument('--code-only',action='store_true',help='Upload and verify HF documentation without checkpoints.') parser.add_argument('--artifact',action='append',help='Upload only this manifest artifact ID; may be repeated.') parser.add_argument('--large-folder',action='store_true',help='Use resumable SDK hashing, LFS transfers and commits for the selected allowlist.') parser.add_argument('--upload-workers',type=int,default=8,help='Concurrent workers for --large-folder (default: 8).') parser.add_argument('--preflight',action='store_true',help='Check identity/access without creating or uploading.') parser.add_argument('--direct',action='store_true',help='Ignore process proxy environment for this attempt.') args=parser.parse_args() if args.upload_workers<2: parser.error('--upload-workers must be at least 2') gh,hf=get_tokens(args) committed_files() path=ROOT/'release/publish_status.json' status=json.loads(path.read_text()) if path.exists() else {} status['attempted_at']=datetime.now(timezone.utc).isoformat() failures=[] for target,func,token in [('github',publish_github,gh),('hf',publish_hf,hf)]: if args.target not in ['both',target]:continue try: func(args,token,status) except Exception as e: # Requests exceptions may contain long proxy traces, but no # authorization headers are printed. Scrub recognized token forms. detail=re.sub(SECRET.pattern.decode(), '[REDACTED]', str(e))[:1000] status.setdefault(target,{}).update(error_type=type(e).__name__,error=detail) save_status(status) print(f'{target}: {type(e).__name__}: {detail}',file=sys.stderr,flush=True) failures.append(target) if failures:raise SystemExit(1) if __name__=='__main__':main()