Library media from the artifact bucket (<ARTIFACT_S3_URI>/library-media/<corpus>/media.tar), not an HF dataset — indexing PR #24 (6b52aba), via this image's aws-CLI artifact_store (GPU variant)

docker/hf-space/nginx.conf.template

@@ -262,7 +262,7 @@ server {
     }
     # Library media — the Explorer's joints viewer (model.glb + articulation.json), moving-parts and drop-test
     # clips, per asset of this corpus: static files pipeline/library_media.py fetches at container start from
-    # SimReady Central's HF dataset (supervisord program library-media). Not there (not fetched yet, or none for
+    # the artifact bucket (supervisord program library-media). Not there (not fetched yet, or none for
     # this corpus) → a JSON 404 and the Explorer shows no media; never the SPA's index.html. Range requests
     # (Safari's <video>) are nginx's own. add_header drops his server-level headers: his CORS lines re-declared.
     location /library_media/ {

docker/hf-space/supervisord-gpu.conf

@@ -122,7 +122,7 @@ stderr_logfile_maxbytes=0
 
 [program:library-media]
 ; SimReady Central's library media for this corpus (the Explorer's joints viewer, moving-parts and drop-test
-; clips), fetched once per container start from its HF dataset into .build/library_media/ — see
+; clips), fetched once per container start from the artifact bucket into .build/library_media/ — see
 ; pipeline/library_media.py and nginx.conf.template's /library_media/. Optional: it always exits 0 and is never
 ; restarted; until it has finished (or when there is none) /library_media/ answers 404 and the Explorer shows
 ; no media.

docker/hf-space/supervisord.conf

@@ -120,7 +120,7 @@ stderr_logfile_maxbytes=0
 
 [program:library-media]
 ; SimReady Central's library media for this corpus (the Explorer's joints viewer, moving-parts and drop-test
-; clips), fetched once per container start from its HF dataset into .build/library_media/ — see
+; clips), fetched once per container start from the artifact bucket into .build/library_media/ — see
 ; pipeline/library_media.py and nginx.conf.template's /library_media/. Optional: it always exits 0 and is never
 ; restarted; until it has finished (or when there is none) /library_media/ answers 404 and the Explorer shows
 ; no media.

pipeline/library_media.py

@@ -1,20 +1,21 @@
 """Library media for the Space's Explorer: the joints viewer's models, moving-parts and drop-test clips.
 
 SimReady Central builds these per asset — model.glb + articulation.json for each prop with moving joints
-(its scripts/library_media/build.py), the moving-parts clips its lab render pool renders, and the drop tests —
-and publishes them as a DATA artifact: a Hugging Face dataset (LIBRARY_MEDIA_REPO, default
-nvidia/simready-central-library-media) holding, per corpus, ``<CORPUS_NAME>/media.tar``: an uncompressed tar of
-``manifest.json`` + ``<id>/<file>`` (its members are already compressed). Run once at container start by supervisord
-(program ``library-media``), this downloads that ONE file — thousands of small files fetched one by one hit the
-Hub's per-IP rate limit on a Space (429s, then a failed fetch, seen live) — unpacks it and installs the tree as
-``.build/library_media/<INDEX_KEY>/``; nginx serves it at /library_media/ (docker/hf-space/
-nginx.conf.template), where the Explorer asks for /library_media/usdsearch-dev/manifest.json — usdsearch-dev being
-its index key in /config.js.
-
-Nothing here is needed to search. Any failure — no such repo or archive, a private repo with no token, a network
-error, an unsafe or malformed archive — leaves the previous tree (none at boot): nginx answers 404 and the Explorer
-shows no media. LIBRARY_MEDIA_REPO=off disables the fetch; LIBRARY_MEDIA_REVISION pins a dataset commit (default
-main). The token is HF_STORAGE_API_TOKEN, else HF_TOKEN, else none (the dataset is public).
+(its scripts/library_media/build.py), the moving-parts clips its lab render pool renders, and the drop tests — and
+they are kept in the artifact bucket next to the index runs, one archive per corpus:
+``<ARTIFACT_S3_URI>/library-media/<CORPUS_NAME>/media.tar``, an uncompressed tar of ``manifest.json`` +
+``<id>/<file>`` (its members are already compressed). Outside ``<corpus>/``, so nothing that lists a corpus's runs
+sees it; corpus-level, not per run, because the media are keyed by the assets' URLs, not by an index build.
+
+Run once at container start by supervisord (program ``library-media``), this downloads that ONE object with the
+same aws CLI and credentials as the index restore (artifact_store), unpacks it and installs the tree as
+``.build/library_media/<INDEX_KEY>/``; nginx serves it at /library_media/ (docker/hf-space/nginx.conf.template), where
+the Explorer asks for /library_media/usdsearch-dev/manifest.json — usdsearch-dev being its index key in /config.js.
+NOT under ``.build/restore/``: restore.download empties that whole directory at every start.
+
+Nothing here is needed to search. Any failure — no archive for this corpus, no bucket credentials, a network error,
+an unsafe or malformed archive — leaves the previous tree (none at boot): nginx answers 404 and the Explorer shows no
+media. LIBRARY_MEDIA=off disables the fetch.
 """
 
 from __future__ import annotations
@@ -22,20 +23,25 @@ from __future__ import annotations
 import json
 import os
 import shutil
+import subprocess
 import sys
 import tarfile
 import time
 from pathlib import Path
 
-from .config import BUILD_DIR
+from .config import BUILD_DIR, Config
 
-LIBRARY_MEDIA_REPO_DEFAULT = "nvidia/simready-central-library-media"
 ARCHIVE = "media.tar"
+PREFIX = "library-media"
 # The Explorer's index key for this Space (/config.js in nginx.conf.template): the manifest's URL names it.
 INDEX_KEY = "usdsearch-dev"
-# NOT under .build/restore/: the index restore (restore.download) empties that whole directory at every start.
 MEDIA_ROOT = BUILD_DIR / "library_media"
-DISABLED = {"", "off", "none", "false", "0"}
+DISABLED = {"off", "none", "false", "0"}
+
+
+def archive_uri(cfg: Config, corpus: str) -> str:
+    """Where this corpus's media archive lives in the artifact bucket."""
+    return f"{cfg.require('ARTIFACT_S3_URI').rstrip('/')}/{PREFIX}/{corpus}/{ARCHIVE}"
 
 
 def valid_manifest(path: Path) -> int | None:
@@ -73,48 +79,65 @@ def install(src: Path, dest: Path) -> None:
     shutil.rmtree(old, ignore_errors=True)
 
 
+def cli_download(cfg: Config, uri: str, dest: Path, attempts: int = 3) -> bool:
+    """``aws s3 cp`` ``uri`` to ``dest``; False when there is no such object (this image's artifact_store is the
+    aws-CLI one, before indexing PR #25's ``download_file``)."""
+    from .artifact_store import artifact_env, aws_cmd
+    dest.parent.mkdir(parents=True, exist_ok=True)
+    for attempt in range(1, attempts + 1):
+        r = subprocess.run(aws_cmd(cfg) + ["cp", uri, str(dest), "--only-show-errors"],
+                           capture_output=True, text=True, env=artifact_env(), timeout=1800)
+        if r.returncode == 0:
+            return True
+        err = (r.stderr or r.stdout).strip()
+        if "(404)" in err or "Not Found" in err or "does not exist" in err:
+            return False
+        print(f"==> library media: aws s3 cp failed (attempt {attempt}/{attempts}, exit {r.returncode}): {err[:300]}")
+        if attempt < attempts:
+            time.sleep(5 * attempt)
+    raise RuntimeError(f"aws s3 cp {uri} failed {attempts} times")
+
+
 def fetch(env: dict[str, str], root: Path = MEDIA_ROOT, download=None) -> int:
     """Fetch this corpus's media into ``root/INDEX_KEY``; returns the number of assets installed (0: none)."""
-    repo = env.get("LIBRARY_MEDIA_REPO", LIBRARY_MEDIA_REPO_DEFAULT).strip()
     corpus = env.get("CORPUS_NAME", "").strip()
-    if repo.lower() in DISABLED:
-        print("==> library media: off (LIBRARY_MEDIA_REPO)")
+    if env.get("LIBRARY_MEDIA", "").strip().lower() in DISABLED:
+        print("==> library media: off (LIBRARY_MEDIA)")
         return 0
     if not corpus or "/" in corpus or corpus.startswith("."):
         print(f"==> library media: no usable CORPUS_NAME ({corpus!r}), skipped")
         return 0
     if download is None:
-        from huggingface_hub import hf_hub_download as download
-    token = env.get("HF_STORAGE_API_TOKEN", "").strip() or env.get("HF_TOKEN", "").strip() or None
-    revision = env.get("LIBRARY_MEDIA_REVISION", "").strip() or None
+        download = cli_download
+    cfg = Config(values=dict(env))
+    uri = archive_uri(cfg, corpus)
     root.mkdir(parents=True, exist_ok=True)
     tmp = root / f".{INDEX_KEY}.download"
     shutil.rmtree(tmp, ignore_errors=True)
     t0 = time.time()
     try:
-        archive = Path(download(repo_id=repo, repo_type="dataset", revision=revision, filename=f"{corpus}/{ARCHIVE}",
-                                local_dir=str(tmp / "dl"), token=token))
+        archive = tmp / ARCHIVE
+        if not download(cfg, uri, archive):
+            print(f"==> library media: no {uri} — none served")
+            return 0
         unpack(archive, tmp / "tree")
         count = valid_manifest(tmp / "tree" / "manifest.json")
         if count is None:
-            print(f"==> library media: {repo}/{corpus}/{ARCHIVE} has no usable manifest.json — none served")
+            print(f"==> library media: {uri} has no usable manifest.json — none served")
             return 0
         install(tmp / "tree", root / INDEX_KEY)
     finally:
         shutil.rmtree(tmp, ignore_errors=True)
     size = sum(p.stat().st_size for p in (root / INDEX_KEY).rglob("*") if p.is_file())
-    print(f"==> library media: {count} assets from {repo}/{corpus}/{ARCHIVE} ({size / 1e6:.0f} MB) "
-          f"in {time.time() - t0:.0f} s")
+    print(f"==> library media: {count} assets from {uri} ({size / 1e6:.0f} MB) in {time.time() - t0:.0f} s")
     return count
 
 
 def main() -> int:
-    # supervisord runs this as uid 1000 with root's HOME: huggingface_hub's default home (~/.cache/huggingface, its
-    # token file and caches) is unreadable there, so it gets one under .build/ unless the image sets its own.
-    os.environ.setdefault("HF_HOME", str(BUILD_DIR / "hf-home"))
     try:
         fetch(dict(os.environ))
-    except Exception as exc:  # noqa: BLE001 — media are optional: never a failed program, never a retry loop
+    except (Exception, SystemExit) as exc:  # noqa: BLE001 — media are optional: never a failed program, never a retry loop
+        # (SystemExit too: artifact_store raises it when the bucket credentials are missing.)
         print(f"==> library media: not fetched ({type(exc).__name__}: {str(exc)[:300]}) — none served", file=sys.stderr)
     return 0