turbot2va-ltx2 / code /scripts /publish.py
luyu1021's picture
Add documented LTX-2 training source, recipes and checkpoint release plan
7802ddc verified
Raw History Blame Contribute Delete
21.3 kB
#!/usr/bin/env python3
"""Create the requested GitHub/HF repos, upload the allowlisted release, verify.
Credentials come from GITHUB_TOKEN and HF_TOKEN, or an explicitly supplied
--token-file. Token values never appear in command arguments or Git remotes.
This script publishes only files committed to this new code repository and
checkpoint files explicitly listed in release/upload_manifest.json.
"""
from __future__ import annotations
import argparse
import hashlib
import json
import os
from pathlib import Path
import re
import subprocess
import sys
from datetime import datetime, timezone
import requests
ROOT = Path(__file__).resolve().parents[1]
SECRET = re.compile(rb'\b(?:hf_[A-Za-z0-9]{20,}|(?:ghp|gho|ghs|ghu|github_pat)_[A-Za-z0-9_]{20,})\b')
def sha(path):
h = hashlib.sha256()
with path.open('rb') as stream:
for chunk in iter(lambda: stream.read(16 * 1024**2), b''):
h.update(chunk)
return h.hexdigest()
def get_tokens(args):
gh, hf = os.environ.get('GITHUB_TOKEN'), os.environ.get('HF_TOKEN')
if args.token_file:
text = args.token_file.read_text()
g = re.search(r'(?:ghp|gho|ghs|ghu|github_pat)_[A-Za-z0-9_]+', text)
candidates = list(dict.fromkeys(re.findall(r'hf_[A-Za-z0-9]+', text)))
gh = gh or (g.group() if g else None)
if not hf and args.target in ('hf', 'both'):
from huggingface_hub import HfApi
for index, candidate in enumerate(candidates):
identity = HfApi(token=candidate).whoami()
role = identity.get('auth', {}).get('accessToken', {}).get('role')
if role == 'read':
print(f'HF credential {index + 1} is read-only; trying next credential.', flush=True)
continue
hf = candidate
break
if not hf and candidates:
raise RuntimeError('The supplied HF credentials are read-only.')
return gh, hf
def git(*args, env=None):
return subprocess.check_output(['git', '-C', str(ROOT), *args], env=env).decode().strip()
def committed_files():
paths = git('ls-files', '-z').split('\0')
files = [ROOT/p for p in paths if p]
if not files or ROOT/'README.md' not in files:
raise RuntimeError('Commit the curated code repository first.')
if git('status', '--porcelain', '--untracked-files=no'):
raise RuntimeError('Tracked code changes have not been committed.')
for p in files:
if p.is_symlink() or p.stat().st_size > 90*1024**2:
raise RuntimeError(f'Unexpected GitHub file: {p.relative_to(ROOT)}')
if p.suffix in ['.pth','.pt','.safetensors','.distcp','.mp4','.wav']:
raise RuntimeError(f'Weight/media in GitHub list: {p.relative_to(ROOT)}')
if SECRET.search(p.read_bytes()):
raise RuntimeError(f'Credential-like content: {p.relative_to(ROOT)}')
return files
def api_call(session, method, path, token, **kwargs):
response = session.request(method, 'https://api.github.com'+path,
headers={'Authorization':'Bearer '+token,
'Accept':'application/vnd.github+json'}, timeout=30, **kwargs)
return response
def publish_github(args, token, status):
if not token:
raise RuntimeError('GITHUB_TOKEN is missing.')
s = requests.Session()
if args.direct:
s.trust_env = False
user = api_call(s,'GET','/user',token)
user.raise_for_status()
owner = args.github_owner or user.json()['login']
if owner != user.json()['login']:
raise RuntimeError('This script creates in the authenticated personal GitHub namespace.')
repo = f'{owner}/turbot2va-ltx2'
previous = api_call(s,'GET',f'/repos/{repo}',token)
if args.preflight:
print(json.dumps({'github_user':owner,'existing_repo_status':previous.status_code}))
return
if previous.status_code == 404:
created = api_call(s,'POST','/user/repos',token,json={'name':'turbot2va-ltx2',
'private':not args.public,'description':'LTX-2 audio-video DCM/SCM/RCM training, configs and checkpoint provenance'})
created.raise_for_status()
data = created.json()
else:
previous.raise_for_status()
data = previous.json()
if data['private'] != (not args.public):
raise RuntimeError('Existing repository visibility differs from requested visibility.')
local_receipt = ROOT/'release/GITHUB_RECEIPT.json'
if data.get('size',0) and not local_receipt.exists():
raise RuntimeError('Requested name already contains unrelated content; no overwrite performed.')
status['github'] = {'repo':repo,'phase':'created','private':data['private']}
save_status(status)
env = os.environ.copy()
env['TURBOT2VA_GITHUB_TOKEN'] = token
if args.direct:
for key in list(env):
if key.lower().endswith('_proxy'):
env.pop(key)
# Askpass prints the credential only into Git's private auth pipe, never
# stdout of this publisher and never a URL persisted in .git/config.
askpass = ROOT/'release/.git_askpass.py'
askpass.write_text('#!/usr/bin/env python3\nimport os,sys\nprint("x-access-token" if "username" in sys.argv[1].lower() else os.environ["TURBOT2VA_GITHUB_TOKEN"])\n')
askpass.chmod(0o700)
env.update(GIT_ASKPASS=str(askpass), GIT_TERMINAL_PROMPT='0')
remote = f'https://github.com/{repo}.git'
try:
git('-c','credential.helper=','push',remote,'HEAD:refs/heads/main',env=env)
finally:
askpass.unlink(missing_ok=True)
remote_ref = api_call(s,'GET',f'/repos/{repo}/git/ref/heads/main',token)
remote_ref.raise_for_status()
commit = git('rev-parse','HEAD')
if remote_ref.json()['object']['sha'] != commit:
raise RuntimeError('GitHub commit verification failed.')
tree = api_call(s,'GET',f'/repos/{repo}/git/trees/{commit}?recursive=1',token)
tree.raise_for_status()
remote_files = {x['path']:x['sha'] for x in tree.json()['tree'] if x['type']=='blob'}
expected = {str(p.relative_to(ROOT)):git('hash-object',str(p)) for p in committed_files()}
if tree.json().get('truncated') or any(remote_files.get(k)!=v for k,v in expected.items()):
raise RuntimeError('GitHub tree content verification failed.')
receipt={'repo':repo,'url':data['html_url'],'commit':commit,'files_verified':len(expected),
'private':data['private'],'verified_at':datetime.now(timezone.utc).isoformat()}
(ROOT/'release/GITHUB_RECEIPT.json').write_text(json.dumps(receipt,indent=2)+'\n')
status['github']={**receipt,'phase':'verified'}
save_status(status)
print('GitHub verified:',repo,commit,len(expected),'files',flush=True)
def save_status(status):
(ROOT/'release/publish_status.json').write_text(json.dumps(status,ensure_ascii=False,indent=2)+'\n')
def large_upload_files(artifacts, stage):
"""Allow only manifest source files and their generated release metadata.
Exact paths include DCP's hidden .metadata. Following local symlinks reads
original bytes without duplicating the checkpoint in staging.
"""
selected={}
for artifact in artifacts:
for entry in artifact['files']:
name=entry['path_in_repo']
p=stage/name
stat=p.stat()
if stat.st_size!=entry['bytes'] or stat.st_mtime_ns!=entry['mtime_ns']:
raise RuntimeError('Source checkpoint changed: '+name)
selected[name]=p
first=artifact['files'][0]['path_in_repo']
directory=('checkpoints/'+artifact['id'] if first.startswith('checkpoints/')
else str(Path(first).parent))
name=directory+'/RELEASE_METADATA.json'
p=stage/name
metadata=json.loads(p.read_text())
if metadata.get('id')!=artifact['id']:
raise RuntimeError('Unexpected release metadata: '+name)
selected[name]=p
return selected
def large_upload_checksums(selected, stage):
"""Reuse the SDK's content hashes only for unchanged, committed files."""
from huggingface_hub._local_folder import read_upload_metadata
expected={}
for name,p in selected.items():
metadata=read_upload_metadata(stage,name)
if not metadata.is_committed or not metadata.sha256 or metadata.size!=p.stat().st_size:
raise RuntimeError('Missing completed large-upload metadata: '+name)
expected[name]={'bytes':metadata.size,'sha256':metadata.sha256}
return expected
def publish_hf(args, token, status):
if not token:
raise RuntimeError('HF_TOKEN is missing.')
from huggingface_hub import HfApi, CommitOperationAdd, hf_hub_url
from huggingface_hub.errors import RepositoryNotFoundError
if args.direct:
from huggingface_hub import configure_http_backend
def backend():
session=requests.Session();session.trust_env=False;return session
configure_http_backend(backend_factory=backend)
api = HfApi(token=token)
user=api.whoami()['name']
owner=args.hf_owner or user
repo=f'{owner}/turbot2va-ltx2'
try:
info=api.repo_info(repo,repo_type='model')
except RepositoryNotFoundError:
info=None
if args.preflight:
print(json.dumps({'hf_user':user,'repo':repo,'exists':info is not None}))
return
if info is None:
api.create_repo(repo_id=repo,repo_type='model',private=not args.public,exist_ok=False)
elif info.private != (not args.public):
raise RuntimeError('Existing HF repo visibility differs; no visibility mutation performed.')
elif len(info.siblings or [])>1 and not (ROOT/'release/HF_RECEIPT.json').exists():
status_file=ROOT/'release/publish_status.json'
if status.get('hf',{}).get('repo')!=repo:
raise RuntimeError('Requested HF name already contains unrelated content.')
status['hf']={'repo':repo,'private':not args.public,'phase':'created'}
save_status(status)
# Upload documentation before large checkpoints, so interrupted uploads
# remain readable and resumable without claiming completion.
files=committed_files()
operations=[CommitOperationAdd(path_in_repo='code/'+str(p.relative_to(ROOT)),path_or_fileobj=str(p)) for p in files]
card=ROOT/'release/MODEL_CARD.md'
operations.append(CommitOperationAdd(path_in_repo='README.md',path_or_fileobj=str(card)))
operations.append(CommitOperationAdd(path_in_repo='LICENSE',path_or_fileobj=str(ROOT/'LICENSE')))
api.create_commit(repo_id=repo,repo_type='model',operations=operations,
commit_message='Add documented LTX-2 training source, recipes and checkpoint release plan')
if args.code_only:
info=api.repo_info(repo,repo_type='model',files_metadata=True)
remote={f.rfilename:f for f in info.siblings}
for p in files:
entry=remote.get('code/'+str(p.relative_to(ROOT)))
if entry is None or entry.size!=p.stat().st_size:
raise RuntimeError('HF documentation verification failed: '+str(p.relative_to(ROOT)))
for name,path in [('README.md',card),('LICENSE',ROOT/'LICENSE')]:
entry=remote.get(name)
if entry is None or entry.size!=path.stat().st_size:
raise RuntimeError('HF root documentation verification failed: '+name)
# A documentation refresh must not erase a completed weight upload.
# Keep its verification revision separate from this new code revision.
receipt_path=ROOT/'release/HF_RECEIPT.json'
previous=json.loads(receipt_path.read_text()) if receipt_path.exists() else {}
if previous.get('repo')==repo:
status['hf'].update(checkpoint_receipt=previous)
status['hf'].update(phase='documentation_verified',revision=info.sha,
source_files_verified=len(files),
checkpoints_present=any(name.startswith(('inference/','checkpoints/')) for name in remote),
weights_uploaded_this_call=False)
documentation_receipt={'repo':repo,'private':info.private,'revision':info.sha,
'source_files_verified':len(files),'root_files_verified':2,
'weights_uploaded_this_call':False,
'verified_at':datetime.now(timezone.utc).isoformat()}
(ROOT/'release/HF_DOCUMENTATION_RECEIPT.json').write_text(json.dumps(documentation_receipt,indent=2)+'\n')
save_status(status)
print('HF documentation verified:',repo,info.sha,len(files),'source files',flush=True)
return
plan=json.loads((ROOT/'release/upload_manifest.json').read_text())
artifacts=plan.get('inference_artifacts', []) + plan['artifacts']
if args.artifact:
artifacts=[a for a in artifacts if a['id'] in args.artifact]
missing=set(args.artifact)-{a['id'] for a in artifacts}
if missing:
raise RuntimeError('Unknown artifact IDs: '+', '.join(sorted(missing)))
expected={}
cache_path=ROOT/'release/checkpoint_checksums.json'
cached=json.loads(cache_path.read_text()) if cache_path.exists() else {}
if getattr(args,'large_folder',False):
selected=large_upload_files(artifacts,args.hf_stage)
status['hf'].update(phase='uploading',mode='resumable_large_folder',
selected_artifacts=[a['id'] for a in artifacts],
selected_files=len(selected),
selected_bytes=sum(p.stat().st_size for p in selected.values()))
save_status(status)
print('Resumable upload:',len(selected),'files,',status['hf']['selected_bytes'],'bytes',flush=True)
# The SDK caches hashing, LFS transfer and commit state independently.
# A restart resumes completed tasks. The allowlist excludes staging
# caches and anything outside the selected checkpoint directories.
api.upload_large_folder(repo_id=repo,repo_type='model',folder_path=args.hf_stage,
private=not args.public,allow_patterns=list(selected),
num_workers=args.upload_workers,print_report_every=60)
large_upload_files(artifacts,args.hf_stage) # Refuse changes during transfer.
expected=large_upload_checksums(selected,args.hf_stage)
for artifact in ([] if getattr(args,'large_folder',False) else artifacts):
print('Preparing artifact:',artifact['id'],flush=True)
remote_directory = str(Path(artifact['files'][0]['path_in_repo']).parent)
# Model DCP files live below sharded/, while the artifact upload root
# includes its metadata, optimizer and RNG siblings as well.
if remote_directory.startswith('checkpoints/'):
remote_directory='checkpoints/'+artifact['id']
stage=args.hf_stage/remote_directory
for entry in artifact['files']:
p=args.hf_stage/entry['path_in_repo']
stat=p.stat()
if stat.st_size!=entry['bytes'] or stat.st_mtime_ns!=entry['mtime_ns']:
raise RuntimeError(f'Source checkpoint changed: {entry["path_in_repo"]}')
old=cached.get(entry['path_in_repo'],{})
if old.get('bytes')!=stat.st_size or old.get('mtime_ns')!=stat.st_mtime_ns:
old={'bytes':stat.st_size,'mtime_ns':stat.st_mtime_ns,'sha256':sha(p)}
cached[entry['path_in_repo']]=old
cache_path.write_text(json.dumps(cached,indent=2)+'\n')
expected[entry['path_in_repo']]={'bytes':stat.st_size,'sha256':old['sha256']}
status['hf'].update(phase='uploading',artifact=artifact['id']);save_status(status)
# upload_folder uploads this bounded artifact (<= ~227 GB), and the
# server deduplicates already uploaded LFS data on a repeated attempt.
api.upload_folder(repo_id=repo,repo_type='model',folder_path=str(stage),
path_in_repo=remote_directory,
commit_message='Add '+artifact['id'],ignore_patterns=['.cache/**'])
for p in stage.rglob('*'):
if p.is_file() and not '.cache' in p.parts:
name=str(p.relative_to(args.hf_stage))
if name not in expected:
expected[name]={'bytes':p.stat().st_size,'sha256':sha(p)}
checksum_file=ROOT/'release/HF_CHECKSUMS.json'
checksum_file.write_text(json.dumps(expected,indent=2)+'\n')
api.upload_file(repo_id=repo,repo_type='model',path_or_fileobj=str(checksum_file),
path_in_repo='CHECKSUMS.json',commit_message='Record checkpoint SHA-256 checksums')
info=api.repo_info(repo,repo_type='model',files_metadata=True)
remote={f.rfilename:f for f in info.siblings}
verified=0
for name,local in expected.items():
f=remote.get(name)
if f is None or f.size!=local['bytes']:
raise RuntimeError('HF size/file verification failed: '+name)
lfs=f.lfs
remote_sha=(lfs.get('sha256') if isinstance(lfs,dict) else getattr(lfs,'sha256',None)) if lfs else None
if remote_sha:
if remote_sha!=local['sha256']:
raise RuntimeError('HF SHA256 mismatch: '+name)
elif local['bytes']<20*1024**2:
from huggingface_hub.utils import get_session
r=get_session().get(hf_hub_url(repo,name,revision=info.sha),headers={'Authorization':'Bearer '+token},timeout=60)
r.raise_for_status()
if hashlib.sha256(r.content).hexdigest()!=local['sha256']:
raise RuntimeError('HF downloaded SHA256 mismatch: '+name)
else:
raise RuntimeError('No verifiable remote SHA256 for large file: '+name)
verified+=1
# Also verify uploaded source paths and sizes, which share the GitHub tree.
for p in files:
f=remote.get('code/'+str(p.relative_to(ROOT)))
if f is None or f.size!=p.stat().st_size:
raise RuntimeError('HF code verification failed: '+str(p.relative_to(ROOT)))
receipt={'repo':repo,'repo_type':'model','private':info.private,'revision':info.sha,
'verified_artifacts':[a['id'] for a in artifacts],
'all_planned_artifacts_verified':len(artifacts)==len(plan['artifacts'])+len(plan.get('inference_artifacts',[])),
'checkpoint_files_verified':verified,'source_files_verified':len(files),
'checkpoint_bytes':sum(x['bytes'] for x in expected.values()),
'verified_at':datetime.now(timezone.utc).isoformat()}
(ROOT/'release/HF_RECEIPT.json').write_text(json.dumps(receipt,indent=2)+'\n')
status['hf']={**receipt,'phase':'verified'};save_status(status)
print('HF verified:',repo,info.sha,verified,'checkpoint files',flush=True)
def main():
parser=argparse.ArgumentParser(description=__doc__)
parser.add_argument('--token-file',type=Path)
parser.add_argument('--github-owner')
parser.add_argument('--hf-owner')
parser.add_argument('--hf-stage',type=Path,default=ROOT.parents[1]/'output/releases/turbot2va-ltx2-hf')
parser.add_argument('--public',action='store_true',help='Default is private.')
parser.add_argument('--target',choices=['github','hf','both'],default='both')
parser.add_argument('--code-only',action='store_true',help='Upload and verify HF documentation without checkpoints.')
parser.add_argument('--artifact',action='append',help='Upload only this manifest artifact ID; may be repeated.')
parser.add_argument('--large-folder',action='store_true',help='Use resumable SDK hashing, LFS transfers and commits for the selected allowlist.')
parser.add_argument('--upload-workers',type=int,default=8,help='Concurrent workers for --large-folder (default: 8).')
parser.add_argument('--preflight',action='store_true',help='Check identity/access without creating or uploading.')
parser.add_argument('--direct',action='store_true',help='Ignore process proxy environment for this attempt.')
args=parser.parse_args()
if args.upload_workers<2:
parser.error('--upload-workers must be at least 2')
gh,hf=get_tokens(args)
committed_files()
path=ROOT/'release/publish_status.json'
status=json.loads(path.read_text()) if path.exists() else {}
status['attempted_at']=datetime.now(timezone.utc).isoformat()
failures=[]
for target,func,token in [('github',publish_github,gh),('hf',publish_hf,hf)]:
if args.target not in ['both',target]:continue
try:
func(args,token,status)
except Exception as e:
# Requests exceptions may contain long proxy traces, but no
# authorization headers are printed. Scrub recognized token forms.
detail=re.sub(SECRET.pattern.decode(), '[REDACTED]', str(e))[:1000]
status.setdefault(target,{}).update(error_type=type(e).__name__,error=detail)
save_status(status)
print(f'{target}: {type(e).__name__}: {detail}',file=sys.stderr,flush=True)
failures.append(target)
if failures:raise SystemExit(1)
if __name__=='__main__':main()