Blame for tools/batch_examples.py

"""Bounded operator batch using IDOL Commons' pinned pipeline, schema and renderer.

Authenticated as the dataset owner via Hub; does not modify the web app's token
scope checks or impersonate an OAuth browser session. Each asset is committed
atomically with its validation, licence documents and catalogue entries.
"""
import os,sys,json,time,hashlib,uuid,shutil,subprocess,signal,copy,base64
from pathlib import Path
from datetime import datetime,timezone
from huggingface_hub import HfApi,snapshot_download,hf_hub_download,CommitOperationAdd
from huggingface_hub.utils import disable_progress_bars,HfHubHTTPError

SPACE='fffiloni/idol-commons-dev';DATASET='fffiloni/idol-community-assets'
REVISION='f9bad8e9a95a13029293a8d6e9b1af23bc7a6297'
SOURCE='2fd68f6aaa69df7c178503a788ad8deb4c7d3117'
SUBJECT='61868ce808aae0b5499a2a95';REPORT='batches/idol-examples-2026-11-01.json'
LIMIT=int(sys.argv[1]) if len(sys.argv)>1 else 94
signal.alarm(1800 if LIMIT==1 else 6600)
started=time.monotonic();token=os.environ.pop('HF_TOKEN');disable_progress_bars();api=HfApi(token=token)
def stamp():return datetime.now(timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ')
def sha(p):return hashlib.sha256(Path(p).read_bytes()).hexdigest()
def out(event,**data):print(event+' '+json.dumps(data,ensure_ascii=False),flush=True)
who=api.whoami();assert who['id']==SUBJECT and who['name']=='fffiloni'
variables=api.get_space_variables(SPACE);assert SUBJECT in variables['MODERATOR_HF_SUBJECTS'].value.split(',')
author={'hf_subject':who['id'],'hf_username':who['name'],'verified_at':stamp()}
root=Path('/tmp/idol-examples-app');snapshot_download(SPACE,repo_type='space',revision=REVISION,local_dir=root,token=token)
os.chdir(root);sys.path.insert(0,str(root))
from backend.contract import Record,Parameters,RightsDeclaration
from backend.catalogue import validate_manifest
from backend.repository import HubRepository
from backend.duplicates import example_identity,CATALOGUE
from backend.generation import load_engine,MODEL_REVISION,PIPELINE_REVISION
from backend.validation import validate_ply,normalize_input
from backend.rendering import render_preview
from backend.licensing import documents
from backend.settings import Settings
from importlib.metadata import distributions

assert SOURCE==PIPELINE_REVISION==CATALOGUE['revision']
presets=json.loads((root/'frontend/src/idol-presets.json').read_text());assert len(presets)==94
fingerprints={e['image']:e for e in CATALOGUE['examples']};assert set(fingerprints)=={p['image'] for p in presets}
assert 'license: apache-2.0' in Path(hf_hub_download('speridlabs/idol','README.md',repo_type='space',revision=SOURCE,token=False)).read_text()
cfg=Settings(state=Path('/tmp/idol-batch-state'),model_token=token,engine=False,publishing=False,app_repo=SPACE,app_revision=REVISION,
             dinov3_revision=variables['DINOV3_REVISION'].value,flux_revision=variables['FLUX_VAE_REVISION'].value)
cfg.state.mkdir(parents=True,exist_ok=True)
browser=shutil.which('chromium') or shutil.which('chromium-browser');assert browser,'Chromium must be installed in the Space image'
os.environ.update(CHROMIUM_EXECUTABLE=browser,CHROMIUM_NO_SANDBOX='1',RENDER_SOFTWARE='1')
safe={k:v for k,v in os.environ.items() if k in ('PATH','HOME','LANG','SSL_CERT_FILE')}
subprocess.run(['npm','ci','--prefix','frontend','--ignore-scripts','--no-audit','--no-fund'],env=safe,check=True,timeout=180,stdout=subprocess.DEVNULL)
runtime=''.join(sorted({f'{d.metadata["Name"]}=={d.version}\n' for d in distributions() if d.metadata['Name']}))
runtime+='# IDOL Commons operator GPU batch; source '+SPACE+'@'+REVISION+'\n'
runtime_sha=hashlib.sha256(runtime.encode()).hexdigest()
repo=HubRepository(DATASET,token)
head,manifest=repo.snapshot();records=validate_manifest(manifest)
expected={(p['image'],MODEL_REVISION) for p in presets}
present={example_identity(r.provenance.model_dump()):r for r in records}
out('BATCH_PLAN',total=len(presets),already_present=sum(k in present for k in expected),limit=LIMIT,author=author,moderator_verified=True,space_revision=REVISION)

def rights(p):
 url=f'https://huggingface.co/spaces/speridlabs/idol/blob/{SOURCE}/{p["image"]}'
 return RightsDeclaration(basis='licensed_source',source_license='Apache-2.0',
   attribution=f'Input: {p["title"]} — SperidLabs IDOL demo. {url} Generated 3D adaptation contributed by @fffiloni.',
   copyright_notice='Source-image rights remain with their respective holders. 3D adaptation and contribution: fffiloni, 2026, to the extent applicable.',
   notices=f'The source Space declares Apache-2.0 at revision {SOURCE}. Original image and preset credits: SperidLabs / IDOL. Modified: image-to-3D generation with the original IDOL preset; preview rendered from the resulting Gaussian splat. No trademark or personality-rights permission is implied.').model_dump()

def commit_files(parent,files,next_manifest,message,progress):
 validate_manifest(next_manifest)
 files={**files,'collection.json':(json.dumps(next_manifest,ensure_ascii=False,separators=(',',':'))+'\n').encode(),
        'data/records.jsonl':(''.join(json.dumps(r,ensure_ascii=False,separators=(',',':'))+'\n' for r in next_manifest['records'])).encode(),
        REPORT:(json.dumps(progress,ensure_ascii=False,indent=2)+'\n').encode()}
 assert all(n.startswith(('assets/','runtimes/','batches/')) or n in ('collection.json','data/records.jsonl') for n in files)
 return api.create_commit(DATASET,repo_type='dataset',parent_commit=parent,operations=[CommitOperationAdd(path_in_repo=n,path_or_fileobj=str(v) if isinstance(v,Path) else v) for n,v in files.items()],commit_message=message).oid

# Freeze the reviewed table once for this run; generation engines remain pinned above.
editorial_revision=api.space_info(SPACE).sha
editorial_raw=Path(hf_hub_download(SPACE,'metadata/idol-examples.json',repo_type='space',revision=editorial_revision,token=token)).read_bytes()
from backend.contract import Editorial
editorial_table=json.loads(editorial_raw)
assert editorial_table['source_revision']==SOURCE
editorials={k:Editorial.model_validate(v).model_dump() for k,v in editorial_table['examples'].items()}
assert set(editorials)==set(fingerprints)
progress={'batch':'idol-examples-2026-11-01','source_revision':SOURCE,'model_revision':MODEL_REVISION,'app_revision':REVISION,'operator':author,'editorial_revision':editorial_revision,'editorial_sha256':hashlib.sha256(editorial_raw).hexdigest(),'requested':94,'started_at':stamp(),'completed':[],'failures':[],'status':'running'}
pipeline=None;created=0
for p in presets:
 if created>=LIMIT or time.monotonic()-started>6300:break
 key=(p['image'],MODEL_REVISION);head,manifest=repo.snapshot();existing=next((r for r in validate_manifest(manifest) if example_identity(r.provenance.model_dump())==key),None)
 if existing:
  assert existing.author.hf_subject==SUBJECT,'Do not relicense another contributor'
  if existing.license!='Apache-2.0' and LIMIT!=1:
   for attempt in range(5):
    head,manifest=repo.snapshot();old=next(r for r in manifest['records'] if r['id']==existing.id);new=copy.deepcopy(old);r=rights(p)
    old_grants=old.get('license_grants') or [{'license':old['license'],'granted_at':old['created_at'],'terms_version':old['publication_terms_version'],'rights':{k:v for k,v in old['rights'].items() if k!='review_status'}}]
    new.update(schema_version='1.2.0',publication_terms_version='1.2.0',license='Apache-2.0',rights={**r,'review_status':'contributor_attested_not_legally_reviewed'},license_grants=old_grants+[{'license':'Apache-2.0','granted_at':stamp(),'terms_version':'1.2.0','rights':r}])
    files=documents(new);Record.model_validate(new);files[new['files']['splat']['path'].rsplit('/',1)[0]+'/metadata.json']=(json.dumps(new,ensure_ascii=False,indent=2)+'\n').encode()
    next_manifest={**manifest,'records':[new if x['id']==existing.id else x for x in manifest['records']]}
    try:commit=commit_files(head,files,next_manifest,'Record Apache licence and source credits for '+p['title'],progress);break
    except HfHubHTTPError as e:
     if e.response is None or e.response.status_code not in (409,412) or attempt==4:raise
   out('EXISTING_LICENCE_UPDATED',image=p['image'],id=existing.id,commit=commit,previous_grants_retained=True)
  out('EXAMPLE_EXISTS',image=p['image'],id=existing.id);progress['completed'].append({'image':p['image'],'id':existing.id,'reused':True});continue
 try:
  folder=cfg.state/str(uuid.uuid4());folder.mkdir()
  raw=Path(hf_hub_download('speridlabs/idol',p['image'],repo_type='space',revision=SOURCE,token=False));assert sha(raw)==fingerprints[p['image']]['raw_sha256']
  info=normalize_input(raw.read_bytes(),folder/'input.png');assert info['sha256']==fingerprints[p['image']]['normalized_sha256']
  if pipeline is None:
   import torch
   assert torch.cuda.is_available();out('GPU_READY',name=torch.cuda.get_device_name(0));pipeline=load_engine(cfg);out('MODEL_LOADED')
  params={'mode':p['mode'],'seed':p['seed'],'steps':p['steps'],'cfg_strength':float(p['cfg']),'rescale_t':float(p['rescale_t']),'num_gaussians':p['num_gaussians'],'randomize_requested':False,'crop_margin':1.2 if p['mode']=='synthetic' else 1.7,'dinov3_dtype':'bf16'}
  Parameters.model_validate(params);t=time.monotonic()
  result=pipeline(str(folder/'input.png'),**{k:params[k] for k in ('mode','seed','steps','cfg_strength','rescale_t','num_gaussians','crop_margin')})
  result.gaussian.save_ply(str(folder/'model.ply'));del result
  validation=validate_ply(folder/'model.ply',params['num_gaussians']);out('GENERATED',image=p['image'],seconds=round(time.monotonic()-t,2),bytes=validation['bytes'])
  preview,framing=render_preview(folder,validation)
  identifier=str(uuid.uuid4());prefix=f'assets/{identifier[:2]}/{identifier}/';r=rights(p);created_at=stamp()
  provenance={'kind':'generated_in_app','generation_id':str(uuid.uuid4()),'app':{'repo':SPACE,'revision':REVISION},'pipeline_source':{'repo':'speridlabs/idol','revision':SOURCE},'model':{'repo':'speridlabs/idol','revision':MODEL_REVISION},'encoders':{'dinov3':{'repo':'facebook/dinov3-vith16plus-pretrain-lvd1689m','revision':cfg.dinov3_revision},'flux2_vae':{'repo':'black-forest-labs/FLUX.2-dev','revision':cfg.flux_revision}},'runtime_lock_sha256':runtime_sha,'source_image':{'kind':'uploaded','sha256':info['sha256']}}
  base={'schema_version':'1.2.0','id':identifier,**editorials[p['image']],'author':author,'created_at':created_at,'license':'Apache-2.0','rights':{**r,'review_status':'contributor_attested_not_legally_reviewed'},'license_grants':[{'license':'Apache-2.0','granted_at':created_at,'terms_version':'1.2.0','rights':r}],'provenance':provenance,'generation':params,'format':'ply-3dgs','representation':{k:validation[k] for k in ('gaussian_count','header_bytes')},'files':{'splat':{'path':prefix+'model.ply','bytes':validation['bytes'],'sha256':validation['sha256'],'media_type':'application/octet-stream'}},'framing':framing,'source_image_public':False,'publication_terms_version':'1.2.0'}
  base['representation'].update(encoding='binary_little_endian',property_layout='idol-sh0-v1',sh_degree=0,up_axis='y',quaternion_order='wxyz',units='unitless')
  for field,name,side in [('preview','preview.png',1024),('preview_small','preview-small.png',512)]:base['files'][field]={'path':prefix+name,**preview['files'][name],'media_type':'image/png','width':side,'height':side,'alpha':True}
  for attempt in range(5):
   head,manifest=repo.snapshot()
   duplicate=next((x for x in manifest['records'] if example_identity(x['provenance'])==key),None)
   if duplicate:out('CONCURRENT_DUPLICATE_SKIPPED',image=p['image'],id=duplicate['id']);break
   record={**base,'collection_sequence':manifest['next_sequence']};files={prefix+n:folder/n for n in ('model.ply','preview.png','preview-small.png')};files.update(documents(record));Record.model_validate(record)
   files[prefix+'metadata.json']=(json.dumps(record,ensure_ascii=False,indent=2)+'\n').encode();files['runtimes/'+runtime_sha+'.txt']=runtime.encode()
   next_manifest={**manifest,'next_sequence':manifest['next_sequence']+1,'records':[*manifest['records'],record]}
   snapshot={**progress,'completed':[*progress['completed'],{'image':p['image'],'id':identifier,'reused':False}]}
   try:commit=commit_files(head,files,next_manifest,'Add IDOL example: '+p['title'],snapshot);progress=snapshot;created+=1;out('EXAMPLE_PUBLISHED',image=p['image'],id=identifier,commit=commit,created=created);break
   except HfHubHTTPError as e:
    if e.response is None or e.response.status_code not in (409,412) or attempt==4:raise
  if LIMIT==1:
   image=base64.b64encode((folder/'preview-small.png').read_bytes()).decode()
   for i in range(0,len(image),8000):print('PROBE_IMAGE '+str(i//8000)+' '+image[i:i+8000],flush=True)
  shutil.rmtree(folder)
 except Exception as e:
  progress['failures'].append({'image':p['image'],'error_type':type(e).__name__,'message':str(e)[:350]});out('EXAMPLE_FAILED',image=p['image'],error_type=type(e).__name__,message=str(e)[:350])
  if LIMIT==1 or len(progress['failures'])>=3:raise

head,manifest=repo.snapshot();complete=[r for r in validate_manifest(manifest) if example_identity(r.provenance.model_dump()) in expected]
progress.update(status='complete' if len(complete)==94 and all(r.license=='Apache-2.0' for r in complete) else 'probe-complete' if LIMIT==1 and created==1 else 'partial',finished_at=stamp(),public_example_count=len(complete),apache_count=sum(r.license=='Apache-2.0' for r in complete),created_this_job=created,seconds=round(time.monotonic()-started,1))
api.create_commit(DATASET,repo_type='dataset',operations=[CommitOperationAdd(path_in_repo=REPORT,path_or_fileobj=(json.dumps(progress,ensure_ascii=False,indent=2)+'\n').encode())],commit_message='Checkpoint IDOL example batch: '+progress['status'])
out('BATCH_RESULT',**{k:v for k,v in progress.items() if k not in ('completed','failures')},failures=progress['failures'])