"""3D subset builder: browse 3D collections, build a subset, download it or push it to the Hub. Each collection is a Hub repo in the collection format (gso_browser/collection3d/format.py): `collection.json` + `catalog.parquet` + one folder per object. `COLLECTIONS` lists them (the first is the default). Thumbnails and 3D previews load straight from the Hub; this server only serves catalogs, streams zips and runs Hub exports with the visitor's OAuth token. Dev mode: `LOCAL_OBJECTS_DIR=/path/to/build/out uvicorn app:app --reload` serves a local build of the first collection. """ from __future__ import annotations import json import os import re import sys import threading import time import uuid import pandas as pd from fastapi import FastAPI, Form, HTTPException, Request from fastapi.responses import FileResponse, JSONResponse, StreamingResponse from fastapi.staticfiles import StaticFiles from huggingface_hub import HfApi, attach_huggingface_oauth, hf_hub_download, parse_huggingface_oauth from huggingface_hub.errors import EntryNotFoundError from pydantic import BaseModel import exporter try: # trainer_space/jobs.py, copied in by publish_space.sh so both Spaces launch runs the same way import trellis2_jobs as T except ImportError: # running from a checkout sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "..", "trainer_space")) import jobs as T DEFAULT_COLLECTIONS = "hugging-3d/gso-objects,hugging-3d/smithsonian-3d-objects,hugging-3d/abo-3d-objects,hugging-3d/nasa-3d-objects" COLLECTION_REPOS = [r for r in os.environ.get("COLLECTIONS", DEFAULT_COLLECTIONS).split(",") if r] EXTRA_LATENTS = [r for r in os.environ.get("LATENTS_REPOS", "").split(",") if r] # appended to every manifest's list LOCAL_DIR = os.environ.get("LOCAL_OBJECTS_DIR") SPACE_ID = os.environ.get("SPACE_ID", "hugging-3d/data-finetuning-subset-builder") SPACE_URL = f"https://huggingface.co/spaces/{SPACE_ID}" APP_URL = f"https://{os.environ['SPACE_HOST']}/" if os.environ.get("SPACE_HOST") else None # None: use the page URL TRAINER_SPACE = os.environ.get("TRAINER_SPACE", "hugging-3d/trellis2-lora-trainer") TRAINER_URL = f"https://huggingface.co/spaces/{TRAINER_SPACE}" ZIP_MAX_BYTES = int(float(os.environ.get("ZIP_MAX_GB", "2")) * 2**30) EXPORT_MAX_BYTES = int(float(os.environ.get("EXPORT_MAX_GB", "30")) * 2**30) STATIC = os.path.join(os.path.dirname(os.path.abspath(__file__)), "static") REPO_NAME = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,95}$") CLIENT_COLUMNS = ["object_id", "name", "description", "category", "brand", "group", "faces", "license", "is_watertight", "size_x_cm", "size_y_cm", "size_z_cm", "max_dim_cm", "flatness", "all_bytes", "glb_bytes", "preview_glb_bytes", "prep_sha256", "latents_repo", "source_url"] app = FastAPI(title="3D subset builder") attach_huggingface_oauth(app) # ---------------------------------------------------------------------------- data class Collection: """One collection repo: its manifest, catalog (with a latents_repo column) and file access.""" def __init__(self, repo: str, local_dir: str | None = None): self.repo, self.local_dir = repo, local_dir self.man = json.load(open(self._file("collection.json"))) self.man["objects_repo"] = repo self.id = self.man["id"] cat = pd.read_parquet(self._file("catalog.parquet")) if "prep_sha256" not in cat: cat["prep_sha256"] = cat["obj_sha256"] cat["latents_repo"] = "" self.latents_repos = list(dict.fromkeys(self.man.get("latents_repos", []) + EXTRA_LATENTS)) for latents in reversed(self.latents_repos): # first listed repo wins try: shas = set(pd.read_csv(hf_hub_download(latents, "metadata.csv", repo_type="dataset")).sha256) except Exception as e: # a missing/private latents repo shouldn't take the Space down print(f"[{self.id}] skipping latents {latents}: {e}") continue hit = cat.prep_sha256.isin(shas) cat.loc[hit, "latents_repo"] = latents print(f"[{self.id}] latents {latents}: {hit.sum()} objects") cat.index = pd.Index(cat.object_id.values) self.catalog = cat self.columns = [c for c in dict.fromkeys(CLIENT_COLUMNS + list(self.man.get("facets", {}))) if c in cat] print(f"[{self.id}] {repo}: {len(cat)} objects") def _file(self, name: str) -> str: if self.local_dir and os.path.exists(os.path.join(self.local_dir, name)): return os.path.join(self.local_dir, name) return hf_hub_download(self.repo, name, repo_type="dataset") def fetch(self, path: str, dest: str) -> str | None: """Local path of one collection file; None if the object doesn't have it.""" if self.local_dir: p = os.path.join(self.local_dir, path) return p if os.path.exists(p) else None try: return hf_hub_download(self.repo, path, repo_type="dataset", local_dir=dest) except EntryNotFoundError: return None @property def asset_base(self) -> str: return f"/local/" if self.local_dir else f"https://huggingface.co/datasets/{self.repo}/resolve/main/" def summary(self) -> dict: m = self.man return {"id": self.id, "title": m["title"], "tagline": m.get("tagline", ""), "repo": self.repo, "license": m["license"], "count": len(self.catalog), "tile": m["paths"]["tile"], "sample": self.catalog.object_id.iloc[len(self.catalog) // 3], "asset_base": self.asset_base} def load_collections() -> dict[str, Collection]: out: dict[str, Collection] = {} for i, repo in enumerate(COLLECTION_REPOS): try: c = Collection(repo, LOCAL_DIR if i == 0 else None) out[c.id] = c except Exception as e: # one broken collection shouldn't take the others down print(f"[collections] skipping {repo}: {type(e).__name__}: {e}") assert out, "no collection could be loaded" return out COLLECTIONS = load_collections() DEFAULT = next(iter(COLLECTIONS)) def get_collection(cid: str | None) -> Collection: if cid and cid not in COLLECTIONS: raise HTTPException(404, f"unknown collection {cid!r}") return COLLECTIONS[cid or DEFAULT] def rows_for(col: Collection, ids: list[str]) -> pd.DataFrame: ids = list(dict.fromkeys(ids)) unknown = [i for i in ids if i not in col.catalog.index] if unknown: raise HTTPException(400, f"unknown object ids: {unknown[:5]}") if not ids: raise HTTPException(400, "the subset is empty") return col.catalog.loc[ids].sort_values("object_id") def check_formats(col: Collection, formats: list[str]) -> list[str]: files = col.man["paths"]["files"] bad = set(formats) - files.keys() if bad: raise HTTPException(400, f"unknown formats: {sorted(bad)}") return [f for f in files if f in formats] # canonical order def extras_for(col: Collection, rows: pd.DataFrame, formats: list[str], share_url: str, repo_id: str, latents_repo: str | None, on_hub: bool = True) -> dict[str, str | bytes]: with_latents = rows[rows.latents_repo == latents_repo] if latents_repo else rows.iloc[:0] holdout = exporter.holdout_split(with_latents) extras: dict[str, str | bytes] = { "subset.txt": exporter.subset_txt(rows, col.man), "catalog.parquet": exporter.catalog_bytes(rows, formats, col.man, repo_id if on_hub else None), "README.md": exporter.card(repo_id, rows, formats, col.man, SPACE_URL, share_url, latents_repo, len(with_latents), len(holdout)), } if holdout: extras["holdout.txt"] = "\n".join(holdout) + "\n" return extras def main_latents_repo(rows: pd.DataFrame) -> str | None: repos = rows.latents_repo[rows.latents_repo != ""] return repos.mode().iloc[0] if len(repos) else None # ---------------------------------------------------------------------------- api def writable_orgs(info) -> list[str]: return [o.preferred_username for o in (info.user_info.orgs or []) if (o.role_in_org or "write") in ("admin", "write", "contributor")] @app.get("/api/collections") def collections() -> dict: return {"default": DEFAULT, "collections": [c.summary() for c in COLLECTIONS.values()]} @app.get("/api/catalog") def catalog(c: str | None = None) -> JSONResponse: col = get_collection(c) m = col.man rows = col.catalog[col.columns].astype(object).where(col.catalog[col.columns].notna(), None).to_dict(orient="records") return JSONResponse({ "collection": col.id, "title": m["title"], "tagline": m.get("tagline", ""), "description": m.get("description", ""), "objects_repo": col.repo, "latents_repos": col.latents_repos, "asset_base": col.asset_base, "app_url": APP_URL, "license": m["license"], "license_url": m["license_url"], "credit": m["credit"], "facets": m.get("facets", {"category": "Category"}), "real_size": m.get("real_size", False), "search_fields": m.get("search_fields", ["name", "description", "category"]), "paths": m["paths"], "formats": m["formats"], "default_formats": m.get("default_formats", ["glb"]), "training_note": m.get("training_note", ""), "zip_max_bytes": ZIP_MAX_BYTES, "export_max_bytes": EXPORT_MAX_BYTES, "trainer_url": TRAINER_URL, "subset_enabled": T.SUBSET_ENABLED, "prep_include": m["prep_include"], "rows": rows, }, headers={"Cache-Control": "public, max-age=300"}) @app.get("/api/me") def me(request: Request) -> dict: info = parse_huggingface_oauth(request) if info is None: return {"user": None} u = info.user_info return {"user": u.preferred_username, "avatar": u.picture, "orgs": writable_orgs(info), "can_train": "jobs" in (info.scope or "").split()} @app.post("/api/zip") def make_zip(payload: str = Form(...)) -> StreamingResponse: """Streamed zip. Posted from a plain