Spaces:

fffiloni
/

idol-commons

Running on Zero

Add bilingual descriptions, French search and optional Z-Image prompts

8e0f286 verified 1 day ago

13.9 kB

"""Bounded operator batch using IDOL Commons' pinned pipeline, schema and renderer.
Authenticated as the dataset owner via Hub; does not modify the web app's token
scope checks or impersonate an OAuth browser session. Each asset is committed
atomically with its validation, licence documents and catalogue entries.
"""
import os,sys,json,time,hashlib,uuid,shutil,subprocess,signal,copy,base64
from pathlib import Path
from datetime import datetime,timezone
from huggingface_hub import HfApi,snapshot_download,hf_hub_download,CommitOperationAdd
from huggingface_hub.utils import disable_progress_bars,HfHubHTTPError

SPACE='fffiloni/idol-commons-dev';DATASET='fffiloni/idol-community-assets'
REVISION='f9bad8e9a95a13029293a8d6e9b1af23bc7a6297'
SOURCE='2fd68f6aaa69df7c178503a788ad8deb4c7d3117'
SUBJECT='61868ce808aae0b5499a2a95';REPORT='batches/idol-examples-2026-11-01.json'
LIMIT=int(sys.argv[1]) if len(sys.argv)>1 else 94
signal.alarm(1800 if LIMIT==1 else 6600)
started=time.monotonic();token=os.environ.pop('HF_TOKEN');disable_progress_bars();api=HfApi(token=token)
def stamp():return datetime.now(timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ')
def sha(p):return hashlib.sha256(Path(p).read_bytes()).hexdigest()
def out(event,**data):print(event+' '+json.dumps(data,ensure_ascii=False),flush=True)
who=api.whoami();assert who['id']==SUBJECT and who['name']=='fffiloni'
variables=api.get_space_variables(SPACE);assert SUBJECT in variables['MODERATOR_HF_SUBJECTS'].value.split(',')
author={'hf_subject':who['id'],'hf_username':who['name'],'verified_at':stamp()}
root=Path('/tmp/idol-examples-app');snapshot_download(SPACE,repo_type='space',revision=REVISION,local_dir=root,token=token)
os.chdir(root);sys.path.insert(0,str(root))
from backend.contract import Record,Parameters,RightsDeclaration
from backend.catalogue import validate_manifest
from backend.repository import HubRepository
from backend.duplicates import example_identity,CATALOGUE
from backend.generation import load_engine,MODEL_REVISION,PIPELINE_REVISION
from backend.validation import validate_ply,normalize_input
from backend.rendering import render_preview
from backend.licensing import documents
from backend.settings import Settings
from importlib.metadata import distributions

assert SOURCE==PIPELINE_REVISION==CATALOGUE['revision']
presets=json.loads((root/'frontend/src/idol-presets.json').read_text());assert len(presets)==94
fingerprints={e['image']:e for e in CATALOGUE['examples']};assert set(fingerprints)=={p['image'] for p in presets}
assert 'license: apache-2.0' in Path(hf_hub_download('speridlabs/idol','README.md',repo_type='space',revision=SOURCE,token=False)).read_text()
cfg=Settings(state=Path('/tmp/idol-batch-state'),model_token=token,engine=False,publishing=False,app_repo=SPACE,app_revision=REVISION,
             dinov3_revision=variables['DINOV3_REVISION'].value,flux_revision=variables['FLUX_VAE_REVISION'].value)
cfg.state.mkdir(parents=True,exist_ok=True)
browser=shutil.which('chromium') or shutil.which('chromium-browser');assert browser,'Chromium must be installed in the Space image'
os.environ.update(CHROMIUM_EXECUTABLE=browser,CHROMIUM_NO_SANDBOX='1',RENDER_SOFTWARE='1')
safe={k:v for k,v in os.environ.items() if k in ('PATH','HOME','LANG','SSL_CERT_FILE')}
subprocess.run(['npm','ci','--prefix','frontend','--ignore-scripts','--no-audit','--no-fund'],env=safe,check=True,timeout=180,stdout=subprocess.DEVNULL)
runtime=''.join(sorted({f'{d.metadata["Name"]}=={d.version}\n' for d in distributions() if d.metadata['Name']}))
runtime+='# IDOL Commons operator GPU batch; source '+SPACE+'@'+REVISION+'\n'
runtime_sha=hashlib.sha256(runtime.encode()).hexdigest()
repo=HubRepository(DATASET,token)
head,manifest=repo.snapshot();records=validate_manifest(manifest)
expected={(p['image'],MODEL_REVISION) for p in presets}
present={example_identity(r.provenance.model_dump()):r for r in records}
out('BATCH_PLAN',total=len(presets),already_present=sum(k in present for k in expected),limit=LIMIT,author=author,moderator_verified=True,space_revision=REVISION)

def rights(p):
 url=f'https://huggingface.co/spaces/speridlabs/idol/blob/{SOURCE}/{p["image"]}'
 return RightsDeclaration(basis='licensed_source',source_license='Apache-2.0',
   attribution=f'Input: {p["title"]} — SperidLabs IDOL demo. {url} Generated 3D adaptation contributed by @fffiloni.',
   copyright_notice='Source-image rights remain with their respective holders. 3D adaptation and contribution: fffiloni, 2026, to the extent applicable.',
   notices=f'The source Space declares Apache-2.0 at revision {SOURCE}. Original image and preset credits: SperidLabs / IDOL. Modified: image-to-3D generation with the original IDOL preset; preview rendered from the resulting Gaussian splat. No trademark or personality-rights permission is implied.').model_dump()

def commit_files(parent,files,next_manifest,message,progress):
 validate_manifest(next_manifest)
 files={**files,'collection.json':(json.dumps(next_manifest,ensure_ascii=False,separators=(',',':'))+'\n').encode(),
        'data/records.jsonl':(''.join(json.dumps(r,ensure_ascii=False,separators=(',',':'))+'\n' for r in next_manifest['records'])).encode(),
        REPORT:(json.dumps(progress,ensure_ascii=False,indent=2)+'\n').encode()}
 assert all(n.startswith(('assets/','runtimes/','batches/')) or n in ('collection.json','data/records.jsonl') for n in files)
 return api.create_commit(DATASET,repo_type='dataset',parent_commit=parent,operations=[CommitOperationAdd(path_in_repo=n,path_or_fileobj=str(v) if isinstance(v,Path) else v) for n,v in files.items()],commit_message=message).oid

# Freeze the reviewed table once for this run; generation engines remain pinned above.
editorial_revision=api.space_info(SPACE).sha
editorial_raw=Path(hf_hub_download(SPACE,'metadata/idol-examples.json',repo_type='space',revision=editorial_revision,token=token)).read_bytes()
from backend.contract import Editorial
editorial_table=json.loads(editorial_raw)
assert editorial_table['source_revision']==SOURCE
editorials={k:Editorial.model_validate(v).model_dump() for k,v in editorial_table['examples'].items()}
assert set(editorials)==set(fingerprints)
progress={'batch':'idol-examples-2026-11-01','source_revision':SOURCE,'model_revision':MODEL_REVISION,'app_revision':REVISION,'operator':author,'editorial_revision':editorial_revision,'editorial_sha256':hashlib.sha256(editorial_raw).hexdigest(),'requested':94,'started_at':stamp(),'completed':[],'failures':[],'status':'running'}
pipeline=None;created=0
for p in presets:
 if created>=LIMIT or time.monotonic()-started>6300:break
 key=(p['image'],MODEL_REVISION);head,manifest=repo.snapshot();existing=next((r for r in validate_manifest(manifest) if example_identity(r.provenance.model_dump())==key),None)
 if existing:
  assert existing.author.hf_subject==SUBJECT,'Do not relicense another contributor'
  if existing.license!='Apache-2.0' and LIMIT!=1:
   for attempt in range(5):
    head,manifest=repo.snapshot();old=next(r for r in manifest['records'] if r['id']==existing.id);new=copy.deepcopy(old);r=rights(p)
    old_grants=old.get('license_grants') or [{'license':old['license'],'granted_at':old['created_at'],'terms_version':old['publication_terms_version'],'rights':{k:v for k,v in old['rights'].items() if k!='review_status'}}]
    new.update(schema_version='1.2.0',publication_terms_version='1.2.0',license='Apache-2.0',rights={**r,'review_status':'contributor_attested_not_legally_reviewed'},license_grants=old_grants+[{'license':'Apache-2.0','granted_at':stamp(),'terms_version':'1.2.0','rights':r}])
    files=documents(new);Record.model_validate(new);files[new['files']['splat']['path'].rsplit('/',1)[0]+'/metadata.json']=(json.dumps(new,ensure_ascii=False,indent=2)+'\n').encode()
    next_manifest={**manifest,'records':[new if x['id']==existing.id else x for x in manifest['records']]}
    try:commit=commit_files(head,files,next_manifest,'Record Apache licence and source credits for '+p['title'],progress);break
    except HfHubHTTPError as e:
     if e.response is None or e.response.status_code not in (409,412) or attempt==4:raise
   out('EXISTING_LICENCE_UPDATED',image=p['image'],id=existing.id,commit=commit,previous_grants_retained=True)
  out('EXAMPLE_EXISTS',image=p['image'],id=existing.id);progress['completed'].append({'image':p['image'],'id':existing.id,'reused':True});continue
 try:
  folder=cfg.state/str(uuid.uuid4());folder.mkdir()
  raw=Path(hf_hub_download('speridlabs/idol',p['image'],repo_type='space',revision=SOURCE,token=False));assert sha(raw)==fingerprints[p['image']]['raw_sha256']
  info=normalize_input(raw.read_bytes(),folder/'input.png');assert info['sha256']==fingerprints[p['image']]['normalized_sha256']
  if pipeline is None:
   import torch
   assert torch.cuda.is_available();out('GPU_READY',name=torch.cuda.get_device_name(0));pipeline=load_engine(cfg);out('MODEL_LOADED')
  params={'mode':p['mode'],'seed':p['seed'],'steps':p['steps'],'cfg_strength':float(p['cfg']),'rescale_t':float(p['rescale_t']),'num_gaussians':p['num_gaussians'],'randomize_requested':False,'crop_margin':1.2 if p['mode']=='synthetic' else 1.7,'dinov3_dtype':'bf16'}
  Parameters.model_validate(params);t=time.monotonic()
  result=pipeline(str(folder/'input.png'),**{k:params[k] for k in ('mode','seed','steps','cfg_strength','rescale_t','num_gaussians','crop_margin')})
  result.gaussian.save_ply(str(folder/'model.ply'));del result
  validation=validate_ply(folder/'model.ply',params['num_gaussians']);out('GENERATED',image=p['image'],seconds=round(time.monotonic()-t,2),bytes=validation['bytes'])
  preview,framing=render_preview(folder,validation)
  identifier=str(uuid.uuid4());prefix=f'assets/{identifier[:2]}/{identifier}/';r=rights(p);created_at=stamp()
  provenance={'kind':'generated_in_app','generation_id':str(uuid.uuid4()),'app':{'repo':SPACE,'revision':REVISION},'pipeline_source':{'repo':'speridlabs/idol','revision':SOURCE},'model':{'repo':'speridlabs/idol','revision':MODEL_REVISION},'encoders':{'dinov3':{'repo':'facebook/dinov3-vith16plus-pretrain-lvd1689m','revision':cfg.dinov3_revision},'flux2_vae':{'repo':'black-forest-labs/FLUX.2-dev','revision':cfg.flux_revision}},'runtime_lock_sha256':runtime_sha,'source_image':{'kind':'uploaded','sha256':info['sha256']}}
  base={'schema_version':'1.2.0','id':identifier,**editorials[p['image']],'author':author,'created_at':created_at,'license':'Apache-2.0','rights':{**r,'review_status':'contributor_attested_not_legally_reviewed'},'license_grants':[{'license':'Apache-2.0','granted_at':created_at,'terms_version':'1.2.0','rights':r}],'provenance':provenance,'generation':params,'format':'ply-3dgs','representation':{k:validation[k] for k in ('gaussian_count','header_bytes')},'files':{'splat':{'path':prefix+'model.ply','bytes':validation['bytes'],'sha256':validation['sha256'],'media_type':'application/octet-stream'}},'framing':framing,'source_image_public':False,'publication_terms_version':'1.2.0'}
  base['representation'].update(encoding='binary_little_endian',property_layout='idol-sh0-v1',sh_degree=0,up_axis='y',quaternion_order='wxyz',units='unitless')
  for field,name,side in [('preview','preview.png',1024),('preview_small','preview-small.png',512)]:base['files'][field]={'path':prefix+name,**preview['files'][name],'media_type':'image/png','width':side,'height':side,'alpha':True}
  for attempt in range(5):
   head,manifest=repo.snapshot()
   duplicate=next((x for x in manifest['records'] if example_identity(x['provenance'])==key),None)
   if duplicate:out('CONCURRENT_DUPLICATE_SKIPPED',image=p['image'],id=duplicate['id']);break
   record={**base,'collection_sequence':manifest['next_sequence']};files={prefix+n:folder/n for n in ('model.ply','preview.png','preview-small.png')};files.update(documents(record));Record.model_validate(record)
   files[prefix+'metadata.json']=(json.dumps(record,ensure_ascii=False,indent=2)+'\n').encode();files['runtimes/'+runtime_sha+'.txt']=runtime.encode()
   next_manifest={**manifest,'next_sequence':manifest['next_sequence']+1,'records':[*manifest['records'],record]}
   snapshot={**progress,'completed':[*progress['completed'],{'image':p['image'],'id':identifier,'reused':False}]}
   try:commit=commit_files(head,files,next_manifest,'Add IDOL example: '+p['title'],snapshot);progress=snapshot;created+=1;out('EXAMPLE_PUBLISHED',image=p['image'],id=identifier,commit=commit,created=created);break
   except HfHubHTTPError as e:
    if e.response is None or e.response.status_code not in (409,412) or attempt==4:raise
  if LIMIT==1:
   image=base64.b64encode((folder/'preview-small.png').read_bytes()).decode()
   for i in range(0,len(image),8000):print('PROBE_IMAGE '+str(i//8000)+' '+image[i:i+8000],flush=True)
  shutil.rmtree(folder)
 except Exception as e:
  progress['failures'].append({'image':p['image'],'error_type':type(e).__name__,'message':str(e)[:350]});out('EXAMPLE_FAILED',image=p['image'],error_type=type(e).__name__,message=str(e)[:350])
  if LIMIT==1 or len(progress['failures'])>=3:raise

head,manifest=repo.snapshot();complete=[r for r in validate_manifest(manifest) if example_identity(r.provenance.model_dump()) in expected]
progress.update(status='complete' if len(complete)==94 and all(r.license=='Apache-2.0' for r in complete) else 'probe-complete' if LIMIT==1 and created==1 else 'partial',finished_at=stamp(),public_example_count=len(complete),apache_count=sum(r.license=='Apache-2.0' for r in complete),created_this_job=created,seconds=round(time.monotonic()-started,1))
api.create_commit(DATASET,repo_type='dataset',operations=[CommitOperationAdd(path_in_repo=REPORT,path_or_fileobj=(json.dumps(progress,ensure_ascii=False,indent=2)+'\n').encode())],commit_message='Checkpoint IDOL example batch: '+progress['status'])
out('BATCH_RESULT',**{k:v for k,v in progress.items() if k not in ('completed','failures')},failures=progress['failures'])

	"""Bounded operator batch using IDOL Commons' pinned pipeline, schema and renderer.

	Authenticated as the dataset owner via Hub; does not modify the web app's token
	scope checks or impersonate an OAuth browser session. Each asset is committed
	atomically with its validation, licence documents and catalogue entries.
	"""
	import os,sys,json,time,hashlib,uuid,shutil,subprocess,signal,copy,base64
	from pathlib import Path
	from datetime import datetime,timezone
	from huggingface_hub import HfApi,snapshot_download,hf_hub_download,CommitOperationAdd
	from huggingface_hub.utils import disable_progress_bars,HfHubHTTPError

	SPACE='fffiloni/idol-commons-dev';DATASET='fffiloni/idol-community-assets'
	REVISION='f9bad8e9a95a13029293a8d6e9b1af23bc7a6297'
	SOURCE='2fd68f6aaa69df7c178503a788ad8deb4c7d3117'
	SUBJECT='61868ce808aae0b5499a2a95';REPORT='batches/idol-examples-2026-11-01.json'
	LIMIT=int(sys.argv[1]) if len(sys.argv)>1 else 94
	signal.alarm(1800 if LIMIT==1 else 6600)
	started=time.monotonic();token=os.environ.pop('HF_TOKEN');disable_progress_bars();api=HfApi(token=token)
	def stamp():return datetime.now(timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ')
	def sha(p):return hashlib.sha256(Path(p).read_bytes()).hexdigest()
	def out(event,**data):print(event+' '+json.dumps(data,ensure_ascii=False),flush=True)
	who=api.whoami();assert who['id']==SUBJECT and who['name']=='fffiloni'
	variables=api.get_space_variables(SPACE);assert SUBJECT in variables['MODERATOR_HF_SUBJECTS'].value.split(',')
	author={'hf_subject':who['id'],'hf_username':who['name'],'verified_at':stamp()}
	root=Path('/tmp/idol-examples-app');snapshot_download(SPACE,repo_type='space',revision=REVISION,local_dir=root,token=token)
	os.chdir(root);sys.path.insert(0,str(root))
	from backend.contract import Record,Parameters,RightsDeclaration
	from backend.catalogue import validate_manifest
	from backend.repository import HubRepository
	from backend.duplicates import example_identity,CATALOGUE
	from backend.generation import load_engine,MODEL_REVISION,PIPELINE_REVISION
	from backend.validation import validate_ply,normalize_input
	from backend.rendering import render_preview
	from backend.licensing import documents
	from backend.settings import Settings
	from importlib.metadata import distributions

	assert SOURCE==PIPELINE_REVISION==CATALOGUE['revision']
	presets=json.loads((root/'frontend/src/idol-presets.json').read_text());assert len(presets)==94
	fingerprints={e['image']:e for e in CATALOGUE['examples']};assert set(fingerprints)=={p['image'] for p in presets}
	assert 'license: apache-2.0' in Path(hf_hub_download('speridlabs/idol','README.md',repo_type='space',revision=SOURCE,token=False)).read_text()
	cfg=Settings(state=Path('/tmp/idol-batch-state'),model_token=token,engine=False,publishing=False,app_repo=SPACE,app_revision=REVISION,
	dinov3_revision=variables['DINOV3_REVISION'].value,flux_revision=variables['FLUX_VAE_REVISION'].value)
	cfg.state.mkdir(parents=True,exist_ok=True)
	browser=shutil.which('chromium') or shutil.which('chromium-browser');assert browser,'Chromium must be installed in the Space image'
	os.environ.update(CHROMIUM_EXECUTABLE=browser,CHROMIUM_NO_SANDBOX='1',RENDER_SOFTWARE='1')
	safe={k:v for k,v in os.environ.items() if k in ('PATH','HOME','LANG','SSL_CERT_FILE')}
	subprocess.run(['npm','ci','--prefix','frontend','--ignore-scripts','--no-audit','--no-fund'],env=safe,check=True,timeout=180,stdout=subprocess.DEVNULL)
	runtime=''.join(sorted({f'{d.metadata["Name"]}=={d.version}\n' for d in distributions() if d.metadata['Name']}))
	runtime+='# IDOL Commons operator GPU batch; source '+SPACE+'@'+REVISION+'\n'
	runtime_sha=hashlib.sha256(runtime.encode()).hexdigest()
	repo=HubRepository(DATASET,token)
	head,manifest=repo.snapshot();records=validate_manifest(manifest)
	expected={(p['image'],MODEL_REVISION) for p in presets}
	present={example_identity(r.provenance.model_dump()):r for r in records}
	out('BATCH_PLAN',total=len(presets),already_present=sum(k in present for k in expected),limit=LIMIT,author=author,moderator_verified=True,space_revision=REVISION)

	def rights(p):
	url=f'https://huggingface.co/spaces/speridlabs/idol/blob/{SOURCE}/{p["image"]}'
	return RightsDeclaration(basis='licensed_source',source_license='Apache-2.0',
	attribution=f'Input: {p["title"]} — SperidLabs IDOL demo. {url} Generated 3D adaptation contributed by @fffiloni.',
	copyright_notice='Source-image rights remain with their respective holders. 3D adaptation and contribution: fffiloni, 2026, to the extent applicable.',
	notices=f'The source Space declares Apache-2.0 at revision {SOURCE}. Original image and preset credits: SperidLabs / IDOL. Modified: image-to-3D generation with the original IDOL preset; preview rendered from the resulting Gaussian splat. No trademark or personality-rights permission is implied.').model_dump()

	def commit_files(parent,files,next_manifest,message,progress):
	validate_manifest(next_manifest)
	files={**files,'collection.json':(json.dumps(next_manifest,ensure_ascii=False,separators=(',',':'))+'\n').encode(),
	'data/records.jsonl':(''.join(json.dumps(r,ensure_ascii=False,separators=(',',':'))+'\n' for r in next_manifest['records'])).encode(),
	REPORT:(json.dumps(progress,ensure_ascii=False,indent=2)+'\n').encode()}
	assert all(n.startswith(('assets/','runtimes/','batches/')) or n in ('collection.json','data/records.jsonl') for n in files)
	return api.create_commit(DATASET,repo_type='dataset',parent_commit=parent,operations=[CommitOperationAdd(path_in_repo=n,path_or_fileobj=str(v) if isinstance(v,Path) else v) for n,v in files.items()],commit_message=message).oid

	# Freeze the reviewed table once for this run; generation engines remain pinned above.
	editorial_revision=api.space_info(SPACE).sha
	editorial_raw=Path(hf_hub_download(SPACE,'metadata/idol-examples.json',repo_type='space',revision=editorial_revision,token=token)).read_bytes()
	from backend.contract import Editorial
	editorial_table=json.loads(editorial_raw)
	assert editorial_table['source_revision']==SOURCE
	editorials={k:Editorial.model_validate(v).model_dump() for k,v in editorial_table['examples'].items()}
	assert set(editorials)==set(fingerprints)
	progress={'batch':'idol-examples-2026-11-01','source_revision':SOURCE,'model_revision':MODEL_REVISION,'app_revision':REVISION,'operator':author,'editorial_revision':editorial_revision,'editorial_sha256':hashlib.sha256(editorial_raw).hexdigest(),'requested':94,'started_at':stamp(),'completed':[],'failures':[],'status':'running'}
	pipeline=None;created=0
	for p in presets:
	if created>=LIMIT or time.monotonic()-started>6300:break
	key=(p['image'],MODEL_REVISION);head,manifest=repo.snapshot();existing=next((r for r in validate_manifest(manifest) if example_identity(r.provenance.model_dump())==key),None)
	if existing:
	assert existing.author.hf_subject==SUBJECT,'Do not relicense another contributor'
	if existing.license!='Apache-2.0' and LIMIT!=1:
	for attempt in range(5):
	head,manifest=repo.snapshot();old=next(r for r in manifest['records'] if r['id']==existing.id);new=copy.deepcopy(old);r=rights(p)
	old_grants=old.get('license_grants') or [{'license':old['license'],'granted_at':old['created_at'],'terms_version':old['publication_terms_version'],'rights':{k:v for k,v in old['rights'].items() if k!='review_status'}}]
	new.update(schema_version='1.2.0',publication_terms_version='1.2.0',license='Apache-2.0',rights={**r,'review_status':'contributor_attested_not_legally_reviewed'},license_grants=old_grants+[{'license':'Apache-2.0','granted_at':stamp(),'terms_version':'1.2.0','rights':r}])
	files=documents(new);Record.model_validate(new);files[new['files']['splat']['path'].rsplit('/',1)[0]+'/metadata.json']=(json.dumps(new,ensure_ascii=False,indent=2)+'\n').encode()
	next_manifest={**manifest,'records':[new if x['id']==existing.id else x for x in manifest['records']]}
	try:commit=commit_files(head,files,next_manifest,'Record Apache licence and source credits for '+p['title'],progress);break
	except HfHubHTTPError as e:
	if e.response is None or e.response.status_code not in (409,412) or attempt==4:raise
	out('EXISTING_LICENCE_UPDATED',image=p['image'],id=existing.id,commit=commit,previous_grants_retained=True)
	out('EXAMPLE_EXISTS',image=p['image'],id=existing.id);progress['completed'].append({'image':p['image'],'id':existing.id,'reused':True});continue
	try:
	folder=cfg.state/str(uuid.uuid4());folder.mkdir()
	raw=Path(hf_hub_download('speridlabs/idol',p['image'],repo_type='space',revision=SOURCE,token=False));assert sha(raw)==fingerprints[p['image']]['raw_sha256']
	info=normalize_input(raw.read_bytes(),folder/'input.png');assert info['sha256']==fingerprints[p['image']]['normalized_sha256']
	if pipeline is None:
	import torch
	assert torch.cuda.is_available();out('GPU_READY',name=torch.cuda.get_device_name(0));pipeline=load_engine(cfg);out('MODEL_LOADED')
	params={'mode':p['mode'],'seed':p['seed'],'steps':p['steps'],'cfg_strength':float(p['cfg']),'rescale_t':float(p['rescale_t']),'num_gaussians':p['num_gaussians'],'randomize_requested':False,'crop_margin':1.2 if p['mode']=='synthetic' else 1.7,'dinov3_dtype':'bf16'}
	Parameters.model_validate(params);t=time.monotonic()
	result=pipeline(str(folder/'input.png'),**{k:params[k] for k in ('mode','seed','steps','cfg_strength','rescale_t','num_gaussians','crop_margin')})
	result.gaussian.save_ply(str(folder/'model.ply'));del result
	validation=validate_ply(folder/'model.ply',params['num_gaussians']);out('GENERATED',image=p['image'],seconds=round(time.monotonic()-t,2),bytes=validation['bytes'])
	preview,framing=render_preview(folder,validation)
	identifier=str(uuid.uuid4());prefix=f'assets/{identifier[:2]}/{identifier}/';r=rights(p);created_at=stamp()
	provenance={'kind':'generated_in_app','generation_id':str(uuid.uuid4()),'app':{'repo':SPACE,'revision':REVISION},'pipeline_source':{'repo':'speridlabs/idol','revision':SOURCE},'model':{'repo':'speridlabs/idol','revision':MODEL_REVISION},'encoders':{'dinov3':{'repo':'facebook/dinov3-vith16plus-pretrain-lvd1689m','revision':cfg.dinov3_revision},'flux2_vae':{'repo':'black-forest-labs/FLUX.2-dev','revision':cfg.flux_revision}},'runtime_lock_sha256':runtime_sha,'source_image':{'kind':'uploaded','sha256':info['sha256']}}
	base={'schema_version':'1.2.0','id':identifier,editorials[p['image']],'author':author,'created_at':created_at,'license':'Apache-2.0','rights':{r,'review_status':'contributor_attested_not_legally_reviewed'},'license_grants':[{'license':'Apache-2.0','granted_at':created_at,'terms_version':'1.2.0','rights':r}],'provenance':provenance,'generation':params,'format':'ply-3dgs','representation':{k:validation[k] for k in ('gaussian_count','header_bytes')},'files':{'splat':{'path':prefix+'model.ply','bytes':validation['bytes'],'sha256':validation['sha256'],'media_type':'application/octet-stream'}},'framing':framing,'source_image_public':False,'publication_terms_version':'1.2.0'}
	base['representation'].update(encoding='binary_little_endian',property_layout='idol-sh0-v1',sh_degree=0,up_axis='y',quaternion_order='wxyz',units='unitless')
	for field,name,side in [('preview','preview.png',1024),('preview_small','preview-small.png',512)]:base['files'][field]={'path':prefix+name,**preview['files'][name],'media_type':'image/png','width':side,'height':side,'alpha':True}
	for attempt in range(5):
	head,manifest=repo.snapshot()
	duplicate=next((x for x in manifest['records'] if example_identity(x['provenance'])==key),None)
	if duplicate:out('CONCURRENT_DUPLICATE_SKIPPED',image=p['image'],id=duplicate['id']);break
	record={**base,'collection_sequence':manifest['next_sequence']};files={prefix+n:folder/n for n in ('model.ply','preview.png','preview-small.png')};files.update(documents(record));Record.model_validate(record)
	files[prefix+'metadata.json']=(json.dumps(record,ensure_ascii=False,indent=2)+'\n').encode();files['runtimes/'+runtime_sha+'.txt']=runtime.encode()
	next_manifest={*manifest,'next_sequence':manifest['next_sequence']+1,'records':[manifest['records'],record]}
	snapshot={*progress,'completed':[progress['completed'],{'image':p['image'],'id':identifier,'reused':False}]}
	try:commit=commit_files(head,files,next_manifest,'Add IDOL example: '+p['title'],snapshot);progress=snapshot;created+=1;out('EXAMPLE_PUBLISHED',image=p['image'],id=identifier,commit=commit,created=created);break
	except HfHubHTTPError as e:
	if e.response is None or e.response.status_code not in (409,412) or attempt==4:raise
	if LIMIT==1:
	image=base64.b64encode((folder/'preview-small.png').read_bytes()).decode()
	for i in range(0,len(image),8000):print('PROBE_IMAGE '+str(i//8000)+' '+image[i:i+8000],flush=True)
	shutil.rmtree(folder)
	except Exception as e:
	progress['failures'].append({'image':p['image'],'error_type':type(e).__name__,'message':str(e)[:350]});out('EXAMPLE_FAILED',image=p['image'],error_type=type(e).__name__,message=str(e)[:350])
	if LIMIT==1 or len(progress['failures'])>=3:raise

	head,manifest=repo.snapshot();complete=[r for r in validate_manifest(manifest) if example_identity(r.provenance.model_dump()) in expected]
	progress.update(status='complete' if len(complete)==94 and all(r.license=='Apache-2.0' for r in complete) else 'probe-complete' if LIMIT==1 and created==1 else 'partial',finished_at=stamp(),public_example_count=len(complete),apache_count=sum(r.license=='Apache-2.0' for r in complete),created_this_job=created,seconds=round(time.monotonic()-started,1))
	api.create_commit(DATASET,repo_type='dataset',operations=[CommitOperationAdd(path_in_repo=REPORT,path_or_fileobj=(json.dumps(progress,ensure_ascii=False,indent=2)+'\n').encode())],commit_message='Checkpoint IDOL example batch: '+progress['status'])
	out('BATCH_RESULT',**{k:v for k,v in progress.items() if k not in ('completed','failures')},failures=progress['failures'])