suvadityamuk's picture
suvadityamuk HF Staff
Update GSO subset builder
16ea65c verified
Raw History Blame Contribute Delete
19.6 kB
"""3D subset builder: browse 3D collections, build a subset, download it or push it to the Hub.
Each collection is a Hub repo in the collection format (gso_browser/collection3d/format.py):
`collection.json` + `catalog.parquet` + one folder per object. `COLLECTIONS` lists them (the first
is the default). Thumbnails and 3D previews load straight from the Hub; this server only serves
catalogs, streams zips and runs Hub exports with the visitor's OAuth token.
Dev mode: `LOCAL_OBJECTS_DIR=/path/to/build/out uvicorn app:app --reload` serves a local build of
the first collection.
"""
from __future__ import annotations
import json
import os
import re
import sys
import threading
import time
import uuid
import pandas as pd
from fastapi import FastAPI, Form, HTTPException, Request
from fastapi.responses import FileResponse, JSONResponse, StreamingResponse
from fastapi.staticfiles import StaticFiles
from huggingface_hub import HfApi, attach_huggingface_oauth, hf_hub_download, parse_huggingface_oauth
from huggingface_hub.errors import EntryNotFoundError
from pydantic import BaseModel
import exporter
try: # trainer_space/jobs.py, copied in by publish_space.sh so both Spaces launch runs the same way
import trellis2_jobs as T
except ImportError: # running from a checkout
sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "..", "trainer_space"))
import jobs as T
DEFAULT_COLLECTIONS = "hugging-3d/gso-objects,hugging-3d/smithsonian-3d-objects,hugging-3d/abo-3d-objects,hugging-3d/nasa-3d-objects"
COLLECTION_REPOS = [r for r in os.environ.get("COLLECTIONS", DEFAULT_COLLECTIONS).split(",") if r]
EXTRA_LATENTS = [r for r in os.environ.get("LATENTS_REPOS", "").split(",") if r] # appended to every manifest's list
LOCAL_DIR = os.environ.get("LOCAL_OBJECTS_DIR")
SPACE_ID = os.environ.get("SPACE_ID", "hugging-3d/data-finetuning-subset-builder")
SPACE_URL = f"https://huggingface.co/spaces/{SPACE_ID}"
APP_URL = f"https://{os.environ['SPACE_HOST']}/" if os.environ.get("SPACE_HOST") else None # None: use the page URL
TRAINER_SPACE = os.environ.get("TRAINER_SPACE", "hugging-3d/trellis2-lora-trainer")
TRAINER_URL = f"https://huggingface.co/spaces/{TRAINER_SPACE}"
ZIP_MAX_BYTES = int(float(os.environ.get("ZIP_MAX_GB", "2")) * 2**30)
EXPORT_MAX_BYTES = int(float(os.environ.get("EXPORT_MAX_GB", "30")) * 2**30)
STATIC = os.path.join(os.path.dirname(os.path.abspath(__file__)), "static")
REPO_NAME = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,95}$")
CLIENT_COLUMNS = ["object_id", "name", "description", "category", "brand", "group", "faces", "license",
"is_watertight", "size_x_cm", "size_y_cm", "size_z_cm", "max_dim_cm", "flatness",
"all_bytes", "glb_bytes", "preview_glb_bytes", "prep_sha256", "latents_repo", "source_url"]
app = FastAPI(title="3D subset builder")
attach_huggingface_oauth(app)
# ---------------------------------------------------------------------------- data
class Collection:
"""One collection repo: its manifest, catalog (with a latents_repo column) and file access."""
def __init__(self, repo: str, local_dir: str | None = None):
self.repo, self.local_dir = repo, local_dir
self.man = json.load(open(self._file("collection.json")))
self.man["objects_repo"] = repo
self.id = self.man["id"]
cat = pd.read_parquet(self._file("catalog.parquet"))
if "prep_sha256" not in cat:
cat["prep_sha256"] = cat["obj_sha256"]
cat["latents_repo"] = ""
self.latents_repos = list(dict.fromkeys(self.man.get("latents_repos", []) + EXTRA_LATENTS))
for latents in reversed(self.latents_repos): # first listed repo wins
try:
shas = set(pd.read_csv(hf_hub_download(latents, "metadata.csv", repo_type="dataset")).sha256)
except Exception as e: # a missing/private latents repo shouldn't take the Space down
print(f"[{self.id}] skipping latents {latents}: {e}")
continue
hit = cat.prep_sha256.isin(shas)
cat.loc[hit, "latents_repo"] = latents
print(f"[{self.id}] latents {latents}: {hit.sum()} objects")
cat.index = pd.Index(cat.object_id.values)
self.catalog = cat
self.columns = [c for c in dict.fromkeys(CLIENT_COLUMNS + list(self.man.get("facets", {}))) if c in cat]
print(f"[{self.id}] {repo}: {len(cat)} objects")
def _file(self, name: str) -> str:
if self.local_dir and os.path.exists(os.path.join(self.local_dir, name)):
return os.path.join(self.local_dir, name)
return hf_hub_download(self.repo, name, repo_type="dataset")
def fetch(self, path: str, dest: str) -> str | None:
"""Local path of one collection file; None if the object doesn't have it."""
if self.local_dir:
p = os.path.join(self.local_dir, path)
return p if os.path.exists(p) else None
try:
return hf_hub_download(self.repo, path, repo_type="dataset", local_dir=dest)
except EntryNotFoundError:
return None
@property
def asset_base(self) -> str:
return f"/local/" if self.local_dir else f"https://huggingface.co/datasets/{self.repo}/resolve/main/"
def summary(self) -> dict:
m = self.man
return {"id": self.id, "title": m["title"], "tagline": m.get("tagline", ""), "repo": self.repo,
"license": m["license"], "count": len(self.catalog), "tile": m["paths"]["tile"],
"sample": self.catalog.object_id.iloc[len(self.catalog) // 3], "asset_base": self.asset_base}
def load_collections() -> dict[str, Collection]:
out: dict[str, Collection] = {}
for i, repo in enumerate(COLLECTION_REPOS):
try:
c = Collection(repo, LOCAL_DIR if i == 0 else None)
out[c.id] = c
except Exception as e: # one broken collection shouldn't take the others down
print(f"[collections] skipping {repo}: {type(e).__name__}: {e}")
assert out, "no collection could be loaded"
return out
COLLECTIONS = load_collections()
DEFAULT = next(iter(COLLECTIONS))
def get_collection(cid: str | None) -> Collection:
if cid and cid not in COLLECTIONS:
raise HTTPException(404, f"unknown collection {cid!r}")
return COLLECTIONS[cid or DEFAULT]
def rows_for(col: Collection, ids: list[str]) -> pd.DataFrame:
ids = list(dict.fromkeys(ids))
unknown = [i for i in ids if i not in col.catalog.index]
if unknown:
raise HTTPException(400, f"unknown object ids: {unknown[:5]}")
if not ids:
raise HTTPException(400, "the subset is empty")
return col.catalog.loc[ids].sort_values("object_id")
def check_formats(col: Collection, formats: list[str]) -> list[str]:
files = col.man["paths"]["files"]
bad = set(formats) - files.keys()
if bad:
raise HTTPException(400, f"unknown formats: {sorted(bad)}")
return [f for f in files if f in formats] # canonical order
def extras_for(col: Collection, rows: pd.DataFrame, formats: list[str], share_url: str, repo_id: str,
latents_repo: str | None, on_hub: bool = True) -> dict[str, str | bytes]:
with_latents = rows[rows.latents_repo == latents_repo] if latents_repo else rows.iloc[:0]
holdout = exporter.holdout_split(with_latents)
extras: dict[str, str | bytes] = {
"subset.txt": exporter.subset_txt(rows, col.man),
"catalog.parquet": exporter.catalog_bytes(rows, formats, col.man, repo_id if on_hub else None),
"README.md": exporter.card(repo_id, rows, formats, col.man, SPACE_URL, share_url,
latents_repo, len(with_latents), len(holdout)),
}
if holdout:
extras["holdout.txt"] = "\n".join(holdout) + "\n"
return extras
def main_latents_repo(rows: pd.DataFrame) -> str | None:
repos = rows.latents_repo[rows.latents_repo != ""]
return repos.mode().iloc[0] if len(repos) else None
# ---------------------------------------------------------------------------- api
def writable_orgs(info) -> list[str]:
return [o.preferred_username for o in (info.user_info.orgs or [])
if (o.role_in_org or "write") in ("admin", "write", "contributor")]
@app.get("/api/collections")
def collections() -> dict:
return {"default": DEFAULT, "collections": [c.summary() for c in COLLECTIONS.values()]}
@app.get("/api/catalog")
def catalog(c: str | None = None) -> JSONResponse:
col = get_collection(c)
m = col.man
rows = col.catalog[col.columns].astype(object).where(col.catalog[col.columns].notna(), None).to_dict(orient="records")
return JSONResponse({
"collection": col.id, "title": m["title"], "tagline": m.get("tagline", ""), "description": m.get("description", ""),
"objects_repo": col.repo, "latents_repos": col.latents_repos, "asset_base": col.asset_base, "app_url": APP_URL,
"license": m["license"], "license_url": m["license_url"], "credit": m["credit"],
"facets": m.get("facets", {"category": "Category"}), "real_size": m.get("real_size", False),
"search_fields": m.get("search_fields", ["name", "description", "category"]),
"paths": m["paths"], "formats": m["formats"], "default_formats": m.get("default_formats", ["glb"]),
"training_note": m.get("training_note", ""),
"zip_max_bytes": ZIP_MAX_BYTES, "export_max_bytes": EXPORT_MAX_BYTES,
"trainer_url": TRAINER_URL, "subset_enabled": T.SUBSET_ENABLED, "prep_include": m["prep_include"],
"rows": rows,
}, headers={"Cache-Control": "public, max-age=300"})
@app.get("/api/me")
def me(request: Request) -> dict:
info = parse_huggingface_oauth(request)
if info is None:
return {"user": None}
u = info.user_info
return {"user": u.preferred_username, "avatar": u.picture,
"orgs": writable_orgs(info), "can_train": "jobs" in (info.scope or "").split()}
@app.post("/api/zip")
def make_zip(payload: str = Form(...)) -> StreamingResponse:
"""Streamed zip. Posted from a plain <form> so the browser handles the download itself."""
req = json.loads(payload)
col = get_collection(req.get("collection"))
rows = rows_for(col, req.get("ids", []))
formats = check_formats(col, req.get("formats", ["glb"]))
if not formats:
raise HTTPException(400, "pick at least one format")
size = exporter.estimate_bytes(rows, formats, col.man)
if size > ZIP_MAX_BYTES:
raise HTTPException(413, f"~{size / 2**30:.1f} GB is over the {ZIP_MAX_BYTES / 2**30:.0f} GB zip limit; "
"use the download script or export to the Hub")
name = req.get("name") or f"{col.id}_subset"
name = name if REPO_NAME.match(name) else f"{col.id}_subset"
latents = main_latents_repo(rows)
extras = extras_for(col, rows, formats, req.get("share_url", SPACE_URL), f"your-username/{name}", latents, on_hub=False)
extras["download_subset.py"] = exporter.download_script(col.man, list(rows.object_id), formats)
paths = exporter.object_paths(rows.object_id, formats, col.man)
return StreamingResponse(exporter.zip_stream(paths, col.fetch, extras, name), media_type="application/zip",
headers={"Content-Disposition": f'attachment; filename="{name}.zip"'})
class ExportRequest(BaseModel):
ids: list[str]
formats: list[str]
repo_id: str
private: bool = True
share_url: str = ""
collection: str = ""
JOBS: dict[str, dict] = {}
def check_repo_id(info, repo_id: str) -> None:
owner, _, name = repo_id.partition("/")
allowed = {info.user_info.preferred_username, *writable_orgs(info)}
if owner not in allowed or not REPO_NAME.match(name):
raise HTTPException(400, f"repo id must be <you or your org>/<name>, got {repo_id!r}")
def repo_taken(info, repo_id: str) -> str | None:
"""Why repo_id can't be used for a new dataset, or None. Uses the visitor's token, so their private
repos count too; an export already running into that name counts as taken."""
if any(j["repo_id"] == repo_id and j["status"] == "running" for j in JOBS.values()):
return f"{repo_id} is being created right now"
if HfApi(token=info.access_token).repo_exists(repo_id, repo_type="dataset"):
return f"{repo_id} already exists"
return None
@app.get("/api/repo-available")
def repo_available(repo_id: str, request: Request) -> dict:
info = parse_huggingface_oauth(request)
if info is None:
raise HTTPException(401, "sign in with Hugging Face first")
check_repo_id(info, repo_id)
reason = repo_taken(info, repo_id)
return {"repo_id": repo_id, "available": reason is None, "reason": reason}
@app.post("/api/export")
def export(req: ExportRequest, request: Request) -> dict:
info = parse_huggingface_oauth(request)
if info is None:
raise HTTPException(401, "sign in with Hugging Face first")
check_repo_id(info, req.repo_id)
reason = repo_taken(info, req.repo_id)
if reason: # never write into an existing repo: pick a new name
raise HTTPException(409, f"{reason}; choose another name")
col = get_collection(req.collection)
rows = rows_for(col, req.ids)
formats = check_formats(col, req.formats)
size = exporter.estimate_bytes(rows, formats, col.man)
if size > EXPORT_MAX_BYTES:
raise HTTPException(413, f"~{size / 2**30:.1f} GB is over the {EXPORT_MAX_BYTES / 2**30:.0f} GB export limit")
extras = extras_for(col, rows, formats, req.share_url or SPACE_URL, req.repo_id, main_latents_repo(rows))
job_id = uuid.uuid4().hex[:12]
job = JOBS[job_id] = {"status": "running", "message": "starting", "progress": 0.0, "url": None,
"owner": info.user_info.preferred_username, "repo_id": req.repo_id, "started": time.time()}
def progress(msg: str, frac: float) -> None:
job.update(message=msg, progress=round(frac, 3))
def run() -> None:
try:
job["url"] = exporter.export_to_hub(info.access_token, req.repo_id, req.private, rows, formats,
extras, col.fetch, progress, col.man)
job["status"] = "done"
except Exception as e:
job.update(status="error", message=f"{type(e).__name__}: {e}")
threading.Thread(target=run, daemon=True).start()
return {"job": job_id}
@app.get("/api/export/{job_id}")
def export_status(job_id: str) -> dict:
if job_id not in JOBS:
raise HTTPException(404, "unknown export")
return {k: v for k, v in JOBS[job_id].items() if k not in ("owner", "repo_id")}
# ---------------------------------------------------------------------------- training
class TrainRequest(BaseModel):
ids: list[str]
formats: list[str]
repo_id: str # the exported subset
flows: list[str] = ["ss", "shape"]
test: bool = True
private: bool = False
prep_flavor: str = T.RECOMMENDED["prep"]
train_flavor: str = T.RECOMMENDED["train"]
agreed: bool = False
collection: str = ""
def build_train(req: TrainRequest, owner: str) -> tuple:
"""Same jobs the trainer Space would launch. Train straight from latents when every object
has them (needs --subset in the image); otherwise preprocess the exported OBJs first."""
if req.prep_flavor not in T.GPU_FLAVORS or req.train_flavor not in T.GPU_FLAVORS:
raise T.UserError("unknown GPU flavor")
flows = [f for f in ("ss", "shape") if f in req.flows]
if not flows:
raise T.UserError("pick at least one flow")
col = get_collection(req.collection)
rows = rows_for(col, req.ids)
run = T.Run(owner, T.run_name(req.repo_id.partition("/")[2], req.test))
sets = dict(T.TEST_RUN["sets"]) if req.test else {}
p_timeout, t_timeout = (T.TEST_RUN["timeout"],) * 2 if req.test else ("6h", "12h")
# --subset also brings the export's group-aware holdout.txt, so colorway variants can't leak
subset = req.repo_id if T.SUBSET_ENABLED else None
latents = main_latents_repo(rows)
if subset and latents and (rows.latents_repo == latents).all():
return run, None, T.train_specs(flows, latents, run, req.private, req.private, sets, subset,
req.train_flavor, t_timeout)
prep_files = {g.rsplit("/", 1)[-1] for g in col.man["prep_include"]}
exported = {p.rsplit("/", 1)[-1] for f in req.formats for p in col.man["paths"]["files"].get(f, [])}
if not prep_files & exported:
raise T.UserError(f"Training needs the files TRELLIS.2 preprocesses ({', '.join(sorted(prep_files))}): "
"include them and export again.")
trains = T.train_specs(flows, run.latents_repo, run, req.private, req.private, sets, subset,
req.train_flavor, t_timeout)
prep = T.prep_args(req.repo_id, col.man["prep_include"], run.latents_repo,
req.private, T.TEST_RUN["max_objects"] if req.test else None, 0.05, 16, 8)
return run, T.prep_spec(prep, req.prep_flavor, p_timeout, run, trains, auto=True), trains
def oauth_or_401(request: Request):
info = parse_huggingface_oauth(request)
if info is None:
raise HTTPException(401, "sign in with Hugging Face first")
return info
@app.get("/api/train/options")
def train_options() -> dict:
return {stage: [{"value": v, "label": label} for label, v in T.flavor_choices(stage)] for stage in ("prep", "train")} | \
{"recommended": T.RECOMMENDED, "trainer_url": TRAINER_URL, "test": T.TEST_RUN}
@app.post("/api/train/plan")
def train_plan(req: TrainRequest, request: Request) -> dict:
info = oauth_or_401(request)
try:
run, prep, trains = build_train(req, info.user_info.preferred_username)
except T.UserError as e:
raise HTTPException(400, str(e))
jobs, total = T.costs(([prep] if prep else []) + trains)
return {"run": run.name, "prep": prep is not None, "jobs": jobs, "total_usd": total,
"latents_repo": run.latents_repo if prep else None, "lora_repo": run.lora_repo,
"trackio_space": run.trackio_space}
@app.post("/api/train")
def train(req: TrainRequest, request: Request) -> dict:
info = oauth_or_401(request)
if "jobs" not in (info.scope or "").split():
raise HTTPException(403, "Sign out and in again: your login predates training support and can't run jobs.")
if not req.agreed:
raise HTTPException(400, "confirm that the jobs are billed to your account")
token = info.access_token
try:
run, prep, trains = build_train(req, info.user_info.preferred_username)
needed = (T.hours(prep.timeout) if prep else 0) + max(T.hours(s.timeout) for s in trains)
T.preflight(token, info.access_token_expires_at.timestamp(), needed, training=True)
launched = [T.launch(token, spec) for spec in ([prep] if prep else trains)]
except T.UserError as e:
raise HTTPException(400, str(e))
return {"run": run.name, "jobs": [{"id": j.id, "url": j.url} for j in launched],
"trainer_url": f"{TRAINER_URL}?run={run.name}"}
# ---------------------------------------------------------------------------- static
if LOCAL_DIR:
app.mount("/local", StaticFiles(directory=LOCAL_DIR), name="local")
app.mount("/static", StaticFiles(directory=STATIC), name="static")
@app.get("/")
def index() -> FileResponse:
return FileResponse(os.path.join(STATIC, "index.html"))