Spaces:
Running on CPU Upgrade
Running on CPU Upgrade
Download app.py from hugging-3d/data-finetuning-subset-builder: direct link, hf CLI and curl.
- Browser
- Download file 19.6 kB
-
https://huggingface.co/spaces/hugging-3d/data-finetuning-subset-builder/resolve/main/app.py
- Command line
-
hf download hf://spaces/hugging-3d/data-finetuning-subset-builder/app.py
-
curl -L -o app.py https://huggingface.co/spaces/hugging-3d/data-finetuning-subset-builder/resolve/main/app.py
19.6 kB
| """3D subset builder: browse 3D collections, build a subset, download it or push it to the Hub. | |
| Each collection is a Hub repo in the collection format (gso_browser/collection3d/format.py): | |
| `collection.json` + `catalog.parquet` + one folder per object. `COLLECTIONS` lists them (the first | |
| is the default). Thumbnails and 3D previews load straight from the Hub; this server only serves | |
| catalogs, streams zips and runs Hub exports with the visitor's OAuth token. | |
| Dev mode: `LOCAL_OBJECTS_DIR=/path/to/build/out uvicorn app:app --reload` serves a local build of | |
| the first collection. | |
| """ | |
| from __future__ import annotations | |
| import json | |
| import os | |
| import re | |
| import sys | |
| import threading | |
| import time | |
| import uuid | |
| import pandas as pd | |
| from fastapi import FastAPI, Form, HTTPException, Request | |
| from fastapi.responses import FileResponse, JSONResponse, StreamingResponse | |
| from fastapi.staticfiles import StaticFiles | |
| from huggingface_hub import HfApi, attach_huggingface_oauth, hf_hub_download, parse_huggingface_oauth | |
| from huggingface_hub.errors import EntryNotFoundError | |
| from pydantic import BaseModel | |
| import exporter | |
| try: # trainer_space/jobs.py, copied in by publish_space.sh so both Spaces launch runs the same way | |
| import trellis2_jobs as T | |
| except ImportError: # running from a checkout | |
| sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "..", "trainer_space")) | |
| import jobs as T | |
| DEFAULT_COLLECTIONS = "hugging-3d/gso-objects,hugging-3d/smithsonian-3d-objects,hugging-3d/abo-3d-objects,hugging-3d/nasa-3d-objects" | |
| COLLECTION_REPOS = [r for r in os.environ.get("COLLECTIONS", DEFAULT_COLLECTIONS).split(",") if r] | |
| EXTRA_LATENTS = [r for r in os.environ.get("LATENTS_REPOS", "").split(",") if r] # appended to every manifest's list | |
| LOCAL_DIR = os.environ.get("LOCAL_OBJECTS_DIR") | |
| SPACE_ID = os.environ.get("SPACE_ID", "hugging-3d/data-finetuning-subset-builder") | |
| SPACE_URL = f"https://huggingface.co/spaces/{SPACE_ID}" | |
| APP_URL = f"https://{os.environ['SPACE_HOST']}/" if os.environ.get("SPACE_HOST") else None # None: use the page URL | |
| TRAINER_SPACE = os.environ.get("TRAINER_SPACE", "hugging-3d/trellis2-lora-trainer") | |
| TRAINER_URL = f"https://huggingface.co/spaces/{TRAINER_SPACE}" | |
| ZIP_MAX_BYTES = int(float(os.environ.get("ZIP_MAX_GB", "2")) * 2**30) | |
| EXPORT_MAX_BYTES = int(float(os.environ.get("EXPORT_MAX_GB", "30")) * 2**30) | |
| STATIC = os.path.join(os.path.dirname(os.path.abspath(__file__)), "static") | |
| REPO_NAME = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,95}$") | |
| CLIENT_COLUMNS = ["object_id", "name", "description", "category", "brand", "group", "faces", "license", | |
| "is_watertight", "size_x_cm", "size_y_cm", "size_z_cm", "max_dim_cm", "flatness", | |
| "all_bytes", "glb_bytes", "preview_glb_bytes", "prep_sha256", "latents_repo", "source_url"] | |
| app = FastAPI(title="3D subset builder") | |
| attach_huggingface_oauth(app) | |
| # ---------------------------------------------------------------------------- data | |
| class Collection: | |
| """One collection repo: its manifest, catalog (with a latents_repo column) and file access.""" | |
| def __init__(self, repo: str, local_dir: str | None = None): | |
| self.repo, self.local_dir = repo, local_dir | |
| self.man = json.load(open(self._file("collection.json"))) | |
| self.man["objects_repo"] = repo | |
| self.id = self.man["id"] | |
| cat = pd.read_parquet(self._file("catalog.parquet")) | |
| if "prep_sha256" not in cat: | |
| cat["prep_sha256"] = cat["obj_sha256"] | |
| cat["latents_repo"] = "" | |
| self.latents_repos = list(dict.fromkeys(self.man.get("latents_repos", []) + EXTRA_LATENTS)) | |
| for latents in reversed(self.latents_repos): # first listed repo wins | |
| try: | |
| shas = set(pd.read_csv(hf_hub_download(latents, "metadata.csv", repo_type="dataset")).sha256) | |
| except Exception as e: # a missing/private latents repo shouldn't take the Space down | |
| print(f"[{self.id}] skipping latents {latents}: {e}") | |
| continue | |
| hit = cat.prep_sha256.isin(shas) | |
| cat.loc[hit, "latents_repo"] = latents | |
| print(f"[{self.id}] latents {latents}: {hit.sum()} objects") | |
| cat.index = pd.Index(cat.object_id.values) | |
| self.catalog = cat | |
| self.columns = [c for c in dict.fromkeys(CLIENT_COLUMNS + list(self.man.get("facets", {}))) if c in cat] | |
| print(f"[{self.id}] {repo}: {len(cat)} objects") | |
| def _file(self, name: str) -> str: | |
| if self.local_dir and os.path.exists(os.path.join(self.local_dir, name)): | |
| return os.path.join(self.local_dir, name) | |
| return hf_hub_download(self.repo, name, repo_type="dataset") | |
| def fetch(self, path: str, dest: str) -> str | None: | |
| """Local path of one collection file; None if the object doesn't have it.""" | |
| if self.local_dir: | |
| p = os.path.join(self.local_dir, path) | |
| return p if os.path.exists(p) else None | |
| try: | |
| return hf_hub_download(self.repo, path, repo_type="dataset", local_dir=dest) | |
| except EntryNotFoundError: | |
| return None | |
| def asset_base(self) -> str: | |
| return f"/local/" if self.local_dir else f"https://huggingface.co/datasets/{self.repo}/resolve/main/" | |
| def summary(self) -> dict: | |
| m = self.man | |
| return {"id": self.id, "title": m["title"], "tagline": m.get("tagline", ""), "repo": self.repo, | |
| "license": m["license"], "count": len(self.catalog), "tile": m["paths"]["tile"], | |
| "sample": self.catalog.object_id.iloc[len(self.catalog) // 3], "asset_base": self.asset_base} | |
| def load_collections() -> dict[str, Collection]: | |
| out: dict[str, Collection] = {} | |
| for i, repo in enumerate(COLLECTION_REPOS): | |
| try: | |
| c = Collection(repo, LOCAL_DIR if i == 0 else None) | |
| out[c.id] = c | |
| except Exception as e: # one broken collection shouldn't take the others down | |
| print(f"[collections] skipping {repo}: {type(e).__name__}: {e}") | |
| assert out, "no collection could be loaded" | |
| return out | |
| COLLECTIONS = load_collections() | |
| DEFAULT = next(iter(COLLECTIONS)) | |
| def get_collection(cid: str | None) -> Collection: | |
| if cid and cid not in COLLECTIONS: | |
| raise HTTPException(404, f"unknown collection {cid!r}") | |
| return COLLECTIONS[cid or DEFAULT] | |
| def rows_for(col: Collection, ids: list[str]) -> pd.DataFrame: | |
| ids = list(dict.fromkeys(ids)) | |
| unknown = [i for i in ids if i not in col.catalog.index] | |
| if unknown: | |
| raise HTTPException(400, f"unknown object ids: {unknown[:5]}") | |
| if not ids: | |
| raise HTTPException(400, "the subset is empty") | |
| return col.catalog.loc[ids].sort_values("object_id") | |
| def check_formats(col: Collection, formats: list[str]) -> list[str]: | |
| files = col.man["paths"]["files"] | |
| bad = set(formats) - files.keys() | |
| if bad: | |
| raise HTTPException(400, f"unknown formats: {sorted(bad)}") | |
| return [f for f in files if f in formats] # canonical order | |
| def extras_for(col: Collection, rows: pd.DataFrame, formats: list[str], share_url: str, repo_id: str, | |
| latents_repo: str | None, on_hub: bool = True) -> dict[str, str | bytes]: | |
| with_latents = rows[rows.latents_repo == latents_repo] if latents_repo else rows.iloc[:0] | |
| holdout = exporter.holdout_split(with_latents) | |
| extras: dict[str, str | bytes] = { | |
| "subset.txt": exporter.subset_txt(rows, col.man), | |
| "catalog.parquet": exporter.catalog_bytes(rows, formats, col.man, repo_id if on_hub else None), | |
| "README.md": exporter.card(repo_id, rows, formats, col.man, SPACE_URL, share_url, | |
| latents_repo, len(with_latents), len(holdout)), | |
| } | |
| if holdout: | |
| extras["holdout.txt"] = "\n".join(holdout) + "\n" | |
| return extras | |
| def main_latents_repo(rows: pd.DataFrame) -> str | None: | |
| repos = rows.latents_repo[rows.latents_repo != ""] | |
| return repos.mode().iloc[0] if len(repos) else None | |
| # ---------------------------------------------------------------------------- api | |
| def writable_orgs(info) -> list[str]: | |
| return [o.preferred_username for o in (info.user_info.orgs or []) | |
| if (o.role_in_org or "write") in ("admin", "write", "contributor")] | |
| def collections() -> dict: | |
| return {"default": DEFAULT, "collections": [c.summary() for c in COLLECTIONS.values()]} | |
| def catalog(c: str | None = None) -> JSONResponse: | |
| col = get_collection(c) | |
| m = col.man | |
| rows = col.catalog[col.columns].astype(object).where(col.catalog[col.columns].notna(), None).to_dict(orient="records") | |
| return JSONResponse({ | |
| "collection": col.id, "title": m["title"], "tagline": m.get("tagline", ""), "description": m.get("description", ""), | |
| "objects_repo": col.repo, "latents_repos": col.latents_repos, "asset_base": col.asset_base, "app_url": APP_URL, | |
| "license": m["license"], "license_url": m["license_url"], "credit": m["credit"], | |
| "facets": m.get("facets", {"category": "Category"}), "real_size": m.get("real_size", False), | |
| "search_fields": m.get("search_fields", ["name", "description", "category"]), | |
| "paths": m["paths"], "formats": m["formats"], "default_formats": m.get("default_formats", ["glb"]), | |
| "training_note": m.get("training_note", ""), | |
| "zip_max_bytes": ZIP_MAX_BYTES, "export_max_bytes": EXPORT_MAX_BYTES, | |
| "trainer_url": TRAINER_URL, "subset_enabled": T.SUBSET_ENABLED, "prep_include": m["prep_include"], | |
| "rows": rows, | |
| }, headers={"Cache-Control": "public, max-age=300"}) | |
| def me(request: Request) -> dict: | |
| info = parse_huggingface_oauth(request) | |
| if info is None: | |
| return {"user": None} | |
| u = info.user_info | |
| return {"user": u.preferred_username, "avatar": u.picture, | |
| "orgs": writable_orgs(info), "can_train": "jobs" in (info.scope or "").split()} | |
| def make_zip(payload: str = Form(...)) -> StreamingResponse: | |
| """Streamed zip. Posted from a plain <form> so the browser handles the download itself.""" | |
| req = json.loads(payload) | |
| col = get_collection(req.get("collection")) | |
| rows = rows_for(col, req.get("ids", [])) | |
| formats = check_formats(col, req.get("formats", ["glb"])) | |
| if not formats: | |
| raise HTTPException(400, "pick at least one format") | |
| size = exporter.estimate_bytes(rows, formats, col.man) | |
| if size > ZIP_MAX_BYTES: | |
| raise HTTPException(413, f"~{size / 2**30:.1f} GB is over the {ZIP_MAX_BYTES / 2**30:.0f} GB zip limit; " | |
| "use the download script or export to the Hub") | |
| name = req.get("name") or f"{col.id}_subset" | |
| name = name if REPO_NAME.match(name) else f"{col.id}_subset" | |
| latents = main_latents_repo(rows) | |
| extras = extras_for(col, rows, formats, req.get("share_url", SPACE_URL), f"your-username/{name}", latents, on_hub=False) | |
| extras["download_subset.py"] = exporter.download_script(col.man, list(rows.object_id), formats) | |
| paths = exporter.object_paths(rows.object_id, formats, col.man) | |
| return StreamingResponse(exporter.zip_stream(paths, col.fetch, extras, name), media_type="application/zip", | |
| headers={"Content-Disposition": f'attachment; filename="{name}.zip"'}) | |
| class ExportRequest(BaseModel): | |
| ids: list[str] | |
| formats: list[str] | |
| repo_id: str | |
| private: bool = True | |
| share_url: str = "" | |
| collection: str = "" | |
| JOBS: dict[str, dict] = {} | |
| def check_repo_id(info, repo_id: str) -> None: | |
| owner, _, name = repo_id.partition("/") | |
| allowed = {info.user_info.preferred_username, *writable_orgs(info)} | |
| if owner not in allowed or not REPO_NAME.match(name): | |
| raise HTTPException(400, f"repo id must be <you or your org>/<name>, got {repo_id!r}") | |
| def repo_taken(info, repo_id: str) -> str | None: | |
| """Why repo_id can't be used for a new dataset, or None. Uses the visitor's token, so their private | |
| repos count too; an export already running into that name counts as taken.""" | |
| if any(j["repo_id"] == repo_id and j["status"] == "running" for j in JOBS.values()): | |
| return f"{repo_id} is being created right now" | |
| if HfApi(token=info.access_token).repo_exists(repo_id, repo_type="dataset"): | |
| return f"{repo_id} already exists" | |
| return None | |
| def repo_available(repo_id: str, request: Request) -> dict: | |
| info = parse_huggingface_oauth(request) | |
| if info is None: | |
| raise HTTPException(401, "sign in with Hugging Face first") | |
| check_repo_id(info, repo_id) | |
| reason = repo_taken(info, repo_id) | |
| return {"repo_id": repo_id, "available": reason is None, "reason": reason} | |
| def export(req: ExportRequest, request: Request) -> dict: | |
| info = parse_huggingface_oauth(request) | |
| if info is None: | |
| raise HTTPException(401, "sign in with Hugging Face first") | |
| check_repo_id(info, req.repo_id) | |
| reason = repo_taken(info, req.repo_id) | |
| if reason: # never write into an existing repo: pick a new name | |
| raise HTTPException(409, f"{reason}; choose another name") | |
| col = get_collection(req.collection) | |
| rows = rows_for(col, req.ids) | |
| formats = check_formats(col, req.formats) | |
| size = exporter.estimate_bytes(rows, formats, col.man) | |
| if size > EXPORT_MAX_BYTES: | |
| raise HTTPException(413, f"~{size / 2**30:.1f} GB is over the {EXPORT_MAX_BYTES / 2**30:.0f} GB export limit") | |
| extras = extras_for(col, rows, formats, req.share_url or SPACE_URL, req.repo_id, main_latents_repo(rows)) | |
| job_id = uuid.uuid4().hex[:12] | |
| job = JOBS[job_id] = {"status": "running", "message": "starting", "progress": 0.0, "url": None, | |
| "owner": info.user_info.preferred_username, "repo_id": req.repo_id, "started": time.time()} | |
| def progress(msg: str, frac: float) -> None: | |
| job.update(message=msg, progress=round(frac, 3)) | |
| def run() -> None: | |
| try: | |
| job["url"] = exporter.export_to_hub(info.access_token, req.repo_id, req.private, rows, formats, | |
| extras, col.fetch, progress, col.man) | |
| job["status"] = "done" | |
| except Exception as e: | |
| job.update(status="error", message=f"{type(e).__name__}: {e}") | |
| threading.Thread(target=run, daemon=True).start() | |
| return {"job": job_id} | |
| def export_status(job_id: str) -> dict: | |
| if job_id not in JOBS: | |
| raise HTTPException(404, "unknown export") | |
| return {k: v for k, v in JOBS[job_id].items() if k not in ("owner", "repo_id")} | |
| # ---------------------------------------------------------------------------- training | |
| class TrainRequest(BaseModel): | |
| ids: list[str] | |
| formats: list[str] | |
| repo_id: str # the exported subset | |
| flows: list[str] = ["ss", "shape"] | |
| test: bool = True | |
| private: bool = False | |
| prep_flavor: str = T.RECOMMENDED["prep"] | |
| train_flavor: str = T.RECOMMENDED["train"] | |
| agreed: bool = False | |
| collection: str = "" | |
| def build_train(req: TrainRequest, owner: str) -> tuple: | |
| """Same jobs the trainer Space would launch. Train straight from latents when every object | |
| has them (needs --subset in the image); otherwise preprocess the exported OBJs first.""" | |
| if req.prep_flavor not in T.GPU_FLAVORS or req.train_flavor not in T.GPU_FLAVORS: | |
| raise T.UserError("unknown GPU flavor") | |
| flows = [f for f in ("ss", "shape") if f in req.flows] | |
| if not flows: | |
| raise T.UserError("pick at least one flow") | |
| col = get_collection(req.collection) | |
| rows = rows_for(col, req.ids) | |
| run = T.Run(owner, T.run_name(req.repo_id.partition("/")[2], req.test)) | |
| sets = dict(T.TEST_RUN["sets"]) if req.test else {} | |
| p_timeout, t_timeout = (T.TEST_RUN["timeout"],) * 2 if req.test else ("6h", "12h") | |
| # --subset also brings the export's group-aware holdout.txt, so colorway variants can't leak | |
| subset = req.repo_id if T.SUBSET_ENABLED else None | |
| latents = main_latents_repo(rows) | |
| if subset and latents and (rows.latents_repo == latents).all(): | |
| return run, None, T.train_specs(flows, latents, run, req.private, req.private, sets, subset, | |
| req.train_flavor, t_timeout) | |
| prep_files = {g.rsplit("/", 1)[-1] for g in col.man["prep_include"]} | |
| exported = {p.rsplit("/", 1)[-1] for f in req.formats for p in col.man["paths"]["files"].get(f, [])} | |
| if not prep_files & exported: | |
| raise T.UserError(f"Training needs the files TRELLIS.2 preprocesses ({', '.join(sorted(prep_files))}): " | |
| "include them and export again.") | |
| trains = T.train_specs(flows, run.latents_repo, run, req.private, req.private, sets, subset, | |
| req.train_flavor, t_timeout) | |
| prep = T.prep_args(req.repo_id, col.man["prep_include"], run.latents_repo, | |
| req.private, T.TEST_RUN["max_objects"] if req.test else None, 0.05, 16, 8) | |
| return run, T.prep_spec(prep, req.prep_flavor, p_timeout, run, trains, auto=True), trains | |
| def oauth_or_401(request: Request): | |
| info = parse_huggingface_oauth(request) | |
| if info is None: | |
| raise HTTPException(401, "sign in with Hugging Face first") | |
| return info | |
| def train_options() -> dict: | |
| return {stage: [{"value": v, "label": label} for label, v in T.flavor_choices(stage)] for stage in ("prep", "train")} | \ | |
| {"recommended": T.RECOMMENDED, "trainer_url": TRAINER_URL, "test": T.TEST_RUN} | |
| def train_plan(req: TrainRequest, request: Request) -> dict: | |
| info = oauth_or_401(request) | |
| try: | |
| run, prep, trains = build_train(req, info.user_info.preferred_username) | |
| except T.UserError as e: | |
| raise HTTPException(400, str(e)) | |
| jobs, total = T.costs(([prep] if prep else []) + trains) | |
| return {"run": run.name, "prep": prep is not None, "jobs": jobs, "total_usd": total, | |
| "latents_repo": run.latents_repo if prep else None, "lora_repo": run.lora_repo, | |
| "trackio_space": run.trackio_space} | |
| def train(req: TrainRequest, request: Request) -> dict: | |
| info = oauth_or_401(request) | |
| if "jobs" not in (info.scope or "").split(): | |
| raise HTTPException(403, "Sign out and in again: your login predates training support and can't run jobs.") | |
| if not req.agreed: | |
| raise HTTPException(400, "confirm that the jobs are billed to your account") | |
| token = info.access_token | |
| try: | |
| run, prep, trains = build_train(req, info.user_info.preferred_username) | |
| needed = (T.hours(prep.timeout) if prep else 0) + max(T.hours(s.timeout) for s in trains) | |
| T.preflight(token, info.access_token_expires_at.timestamp(), needed, training=True) | |
| launched = [T.launch(token, spec) for spec in ([prep] if prep else trains)] | |
| except T.UserError as e: | |
| raise HTTPException(400, str(e)) | |
| return {"run": run.name, "jobs": [{"id": j.id, "url": j.url} for j in launched], | |
| "trainer_url": f"{TRAINER_URL}?run={run.name}"} | |
| # ---------------------------------------------------------------------------- static | |
| if LOCAL_DIR: | |
| app.mount("/local", StaticFiles(directory=LOCAL_DIR), name="local") | |
| app.mount("/static", StaticFiles(directory=STATIC), name="static") | |
| def index() -> FileResponse: | |
| return FileResponse(os.path.join(STATIC, "index.html")) | |