Indexing main 955a553a: usdsearch fd61a332 packages, package downloads, usdsearch validator

Files deploy-space(-gpu) stages from NVIDIA-dev/simready-central-indexing-internal main 955a553a (PRs #39, #41, #43); same run 20261005-ee37df6; Space secrets unchanged.

Dockerfile

@@ -284,15 +284,14 @@ COPY docker/hf-space/supervisord-gpu.conf /etc/supervisor/conf.d/hf-space.conf
 
 COPY docker/hf-space/nginx.conf.template /etc/nginx/hf-space.conf.template
 
-# SimReady Central's validator (supervisord's [program:deeptag-validate], nginx's
-# /vlm_validate/search_result): one vendored file, run by the app venv, which already
-# has its fastapi, uvicorn, httpx, pydantic and pillow (deepsearch-api's own).
-COPY docker/hf-space/deeptag-validate/app.py /app/deeptag-validate/app.py
-
 COPY services/explorer/dist /usr/share/nginx/explorer
 
 
 COPY --chown=1000:1000 pipeline /app/hf-space-pipeline/pipeline
+# package-download (supervisord's program of that name) runs from this copy on the
+# venv deepsearch-api already brought (fastapi, uvicorn, grpcio, the storage_api
+# stubs, search_utils): fail the build, not the Space, if that ever stops importing.
+RUN cd /app/hf-space-pipeline && env -u PYTHONPATH /app/.venv/bin/python -c "import pipeline.package_download.app, pipeline.package_download.source, search_utils.url_mapping"
 
 ARG ARTIFACT_S3_URI
 ARG ARTIFACT_S3_REGION=us-east-2
@@ -314,7 +313,8 @@ ARG ES_NAME
 ARG S3_STORAGE_BUCKET_NAME
 ARG S3_STORAGE_REGION_NAME=us-west-2
 ARG STORAGE_BACKEND_TYPE=s3
-# DOWNLOAD_ENABLED / HF_STORAGE_API_TOKEN / SUBMIT_CONTENT_URL / STORAGE_API_* / DOWNLOAD_* — see
+# DOWNLOAD_ENABLED / HF_STORAGE_API_TOKEN / SUBMIT_CONTENT_URL / STORAGE_API_* / DOWNLOAD_* /
+# PACKAGE_DOWNLOAD_* — see
 # docker/Dockerfile.hf-space's own copies of these comments for the full
 # explanation.
 # USDSEARCH_URL_MAPPING_MODE/_RULES — hf_space_build_args() derives them from
@@ -342,6 +342,10 @@ ENV HF_STORAGE_API_TOKEN= \
     ASSERT_ADMIN_USER=false \
     USDSEARCH_VISION_VALIDATION_ENABLED=false \
     USDSEARCH_LLM_PARSING_ENABLED=false \
+    # Validation limits are per deepsearch-api worker: 8 checks in flight and a 125k-verdict cache in
+    # each of the 4 workers (DEEPSEARCH_API_WORKERS below), 32 and 500k in all as on the cpu image.
+    USDSEARCH_VISION_VALIDATION_MAX_CONCURRENT=8 \
+    USDSEARCH_VISION_VALIDATION_CACHE_SIZE=125000 \
     USDSEARCH_LLM_PARSING_FIELDS_FILEPATH=/app/search_fields.yaml \
     # Business telemetry — same as docker/Dockerfile.hf-space, see its comment.
     OTEL_SDK_DISABLED=false \
@@ -362,6 +366,10 @@ ENV HF_STORAGE_API_TOKEN= \
     DOWNLOAD_MAX_CONCURRENT=2 \
     DOWNLOAD_QUEUE_TIMEOUT=120 \
     DOWNLOAD_MAX_BUNDLE_BYTES=2000000000 \
+    PACKAGE_DOWNLOAD_PORT=8020 \
+    PACKAGE_DOWNLOAD_MAX_CONCURRENT=4 \
+    PACKAGE_DOWNLOAD_QUEUE_TIMEOUT=120 \
+    PACKAGE_DOWNLOAD_MAX_BUNDLE_BYTES=2000000000 \
     STORAGE_API_THUMBNAIL_METADATA_FIELDS="[\"hf:thumbnail_url\"]" \
     SIGLIP2_TRITON_SERVER_URL=localhost:8001 \
     TRITON_SERVER_URL=localhost:8001 \

THIRD_PARTY_NOTICE.md

@@ -31,7 +31,7 @@ the Hugging Face datasets the assets are downloaded from).
 Generated by `scripts/generate_third_party_notice.sh` in the internal
 source repository (simready-central-indexing-internal); do not hand-edit.
 
-**153** unique `(package, version)` entries.
+**152** unique `(package, version)` entries.
 
 | Package | Version | License | Homepage | Sources | License text |
 | --- | --- | --- | --- | --- | --- |
@@ -71,8 +71,7 @@ source repository (simready-central-indexing-internal); do not hand-edit.
 | `filelock` | 3.29.0 | MIT | <https://github.com/tox-dev/py-filelock> | app-venv | [`licenses/filelock-3.29.0.txt`](licenses/filelock-3.29.0.txt) |
 | `fire` | 0.7.1 | Apache-2.0 | <https://github.com/google/python-fire> | app-venv | [`licenses/fire-0.7.1.txt`](licenses/fire-0.7.1.txt) |
 | `frozenlist` | 1.8.0 | Apache-2.0 | <https://github.com/aio-libs/frozenlist> | app-venv | [`licenses/frozenlist-1.8.0.txt`](licenses/frozenlist-1.8.0.txt) |
-| `fsspec` | 2026.3.0 | BSD-3-Clause | <https://github.com/fsspec/filesystem_spec> | app-venv | [`licenses/fsspec-2026.3.0.txt`](licenses/fsspec-2026.3.0.txt) |
-| `fsspec` | 2026.4.0 | BSD-3-Clause | <https://github.com/fsspec/filesystem_spec> | siglip2-triton | [`licenses/fsspec-2026.4.0.txt`](licenses/fsspec-2026.4.0.txt) |
+| `fsspec` | 2026.9.0 | BSD-3-Clause | <https://github.com/fsspec/filesystem_spec> | app-venv, siglip2-triton | [`licenses/fsspec-2026.9.0.txt`](licenses/fsspec-2026.9.0.txt) |
 | `googleapis-common-protos` | 1.74.0 | Apache 2.0 | <https://github.com/googleapis/google-cloud-python/tree/main/packages/googleapis-common-protos> | app-venv | [`licenses/googleapis-common-protos-1.74.0.txt`](licenses/googleapis-common-protos-1.74.0.txt) |
 | `greenlet` | 3.4.0 | MIT AND PSF-2.0 | <https://greenlet.readthedocs.io> | app-venv | [`licenses/greenlet-3.4.0.txt`](licenses/greenlet-3.4.0.txt) |
 | `grpcio` | 1.80.0 | Apache-2.0 | <https://grpc.io> | app-venv | [`licenses/grpcio-1.80.0.txt`](licenses/grpcio-1.80.0.txt) |
@@ -103,7 +102,7 @@ source repository (simready-central-indexing-internal); do not hand-edit.
 | `langgraph` | 1.2.11 | MIT | <https://docs.langchain.com/oss/python/langgraph/overview> | app-venv | [`licenses/langgraph-1.2.11.txt`](licenses/langgraph-1.2.11.txt) |
 | `langgraph-checkpoint` | 4.1.1 | MIT | <https://github.com/langchain-ai/langgraph/tree/main/libs/checkpoint> | app-venv | [`licenses/langgraph-checkpoint-4.1.1.txt`](licenses/langgraph-checkpoint-4.1.1.txt) |
 | `langgraph-prebuilt` | 1.1.0 | MIT | <https://github.com/langchain-ai/langgraph/tree/main/libs/prebuilt> | app-venv | [`licenses/langgraph-prebuilt-1.1.0.txt`](licenses/langgraph-prebuilt-1.1.0.txt) |
-| `langgraph-sdk` | 0.4.2 | MIT | <https://github.com/langchain-ai/langgraph/tree/main/libs/sdk-py> | app-venv | [`licenses/langgraph-sdk-0.4.2.txt`](licenses/langgraph-sdk-0.4.2.txt) |
+| `langgraph-sdk` | 0.4.6 | MIT | <https://github.com/langchain-ai/langgraph/tree/main/libs/sdk-py> | app-venv | [`licenses/langgraph-sdk-0.4.6.txt`](licenses/langgraph-sdk-0.4.6.txt) |
 | `langsmith` | 0.8.18 | MIT | <https://smith.langchain.com/> | app-venv | [`licenses/langsmith-0.8.18.txt`](licenses/langsmith-0.8.18.txt) |
 | `markdown-it-py` | 4.0.0 | MIT License | <https://github.com/executablebooks/markdown-it-py> | app-venv | [`licenses/markdown-it-py-4.0.0.txt`](licenses/markdown-it-py-4.0.0.txt) |
 | `markdown-it-py` | 4.2.0 | MIT License | <https://github.com/executablebooks/markdown-it-py> | siglip2-triton | [`licenses/markdown-it-py-4.2.0.txt`](licenses/markdown-it-py-4.2.0.txt) |
@@ -233,6 +232,6 @@ inferred from each package's `/usr/share/doc/<pkg>/copyright`.
 | `python3.12-venv` | 3.12.3-1ubuntu0.17 | Python-2.0 | <https://packages.ubuntu.com/search?keywords=python3.12-venv> | hf-space-gpu | [`licenses/os-python3.12-venv-3.12.3-1ubuntu0.17.txt`](licenses/os-python3.12-venv-3.12.3-1ubuntu0.17.txt) |
 | `python3-pip-whl` | 24.0+dfsg-1ubuntu1.3 | Expat, Apache-2.0, MPL-2, LGPL-2.1+, BSD-3, Python, Apache-2.0 OR BSD-2, ISC, BSD-2 | <https://packages.ubuntu.com/search?keywords=python3-pip-whl> | hf-space-gpu | [`licenses/os-python3-pip-whl-24.0-dfsg-1ubuntu1.3.txt`](licenses/os-python3-pip-whl-24.0-dfsg-1ubuntu1.3.txt) |
 | `python3-setuptools-whl` | 68.1.2-2ubuntu1.2 | Apache-2.0 | <https://packages.ubuntu.com/search?keywords=python3-setuptools-whl> | hf-space-gpu | [`licenses/os-python3-setuptools-whl-68.1.2-2ubuntu1.2.txt`](licenses/os-python3-setuptools-whl-68.1.2-2ubuntu1.2.txt) |
-| `redis-server` | 5:7.0.15-1ubuntu0.24.04.4 | BSD-3-clause, BSD-2-clause, MIT | <https://packages.ubuntu.com/search?keywords=redis-server> | hf-space-gpu | [`licenses/os-redis-server-5-7.0.15-1ubuntu0.24.04.4.txt`](licenses/os-redis-server-5-7.0.15-1ubuntu0.24.04.4.txt) |
-| `redis-tools` | 5:7.0.15-1ubuntu0.24.04.4 | BSD-3-clause, BSD-2-clause, MIT | <https://packages.ubuntu.com/search?keywords=redis-tools> | hf-space-gpu | [`licenses/os-redis-tools-5-7.0.15-1ubuntu0.24.04.4.txt`](licenses/os-redis-tools-5-7.0.15-1ubuntu0.24.04.4.txt) |
+| `redis-server` | 5:7.0.15-1ubuntu0.24.04.5 | BSD-3-clause, BSD-2-clause, MIT | <https://packages.ubuntu.com/search?keywords=redis-server> | hf-space-gpu | [`licenses/os-redis-server-5-7.0.15-1ubuntu0.24.04.5.txt`](licenses/os-redis-server-5-7.0.15-1ubuntu0.24.04.5.txt) |
+| `redis-tools` | 5:7.0.15-1ubuntu0.24.04.5 | BSD-3-clause, BSD-2-clause, MIT | <https://packages.ubuntu.com/search?keywords=redis-tools> | hf-space-gpu | [`licenses/os-redis-tools-5-7.0.15-1ubuntu0.24.04.5.txt`](licenses/os-redis-tools-5-7.0.15-1ubuntu0.24.04.5.txt) |
 | `supervisor` | 4.2.5-1ubuntu0.1 | BSD-derived, RDFLIB, GPL-2+, Medusa, BSD-Derived | <https://packages.ubuntu.com/search?keywords=supervisor> | hf-space-gpu | [`licenses/os-supervisor-4.2.5-1ubuntu0.1.txt`](licenses/os-supervisor-4.2.5-1ubuntu0.1.txt) |

docker/hf-space/deeptag-validate/app.py

@@ -1,836 +0,0 @@
-# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-# SPDX-License-Identifier: Apache-2.0
-"""
-deeptag-validate — VLM relevance-validation shim.
-
-Two routes, one scorer:
-
-- `POST /vlm_validate/search_result` — the original per-image contract: the
-  browser uploads one base64 thumbnail per call (~100 requests per search).
-- `POST /vlm_validate/batch_stream` — the streamed batch contract (task #27):
-  ONE request per result page carrying only ids + image references; the shim
-  fetches each thumbnail ITSELF through the gateway's `/images` (LAN hop,
-  shared 7-day nginx cache the browsers already warmed, index-scoped by the
-  request's `index` slug) and streams verdicts back as NDJSON lines the
-  moment each lands — shim-cache hits first, then misses dispatched in
-  REQUEST (retrieval) order. Client disconnect cancels every queued engine
-  call (the whole point of the per-request task set — an abandoned query
-  must not strand work on the shared engine).
-
-Bridges the Explorer's `/vlm_validate/*` contract to the self-hosted
-unified vlm-parser-reranker engine (Qwen3.6-35B-A3B-FP8 on one RTX PRO 6000, served alias
-"reranker" — the same
-engine also serves query parsing under the alias "parser"). The engine is used
-as a relevance scorer via the first-token logprob trick: it reads the verdict
-distribution from one forward pass (enable_thinking=false; Qwen thinks by
-default, so a missed flag silently corrupts verdicts), ~20x faster than
-generating a verdict. We return the SAME shape and scale as usdsearch's own
-validator (llm_client's log-prob method), minus the `reasoning` the UI no
-longer shows:
-
-    {"is_match": bool, "confidence": float, "similarity_score": float, "model": str}
-
-ONE verdict method on every route, the Qwen Yes/No log-prob verdict: a Yes/No
-prompt (JUDGE_PROMPT for text queries, JUDGE_IMAGE_PROMPT for an image alone,
-JUDGE_IMAGE_TEXT_PROMPT for an image with words), P(Yes) read from the
-first-token log-probs (position scan, Yes/No mass renormalized), is_match =
-P(Yes) > 0.5, confidence = P(Yes) in [0,1] and similarity_score = P(Yes) x 100
-in [0,100] — usdsearch's scale, so the Explorer reads every validator's score
-the same way (2026-10-06: the shim sent P(Yes) in [0,1] until then, and the
-Explorer's per-value guess showed a usdsearch reject at P(Yes) 0.0076 as 76 %).
-With JUDGE_ASSET_CONTEXT on (2026-10-07), a words-only check also reads the
-candidate's file path and the index's caption, between the query and the
-question (_select_prompt); image checks never do.
-is_match coincides with the 50 % badge boundary the Explorer hardcodes (a
-false is_match renders the orange "VLM rejected, likely false positive" chip,
-and searchOutcome.js's WEAK state means "no hit at or above 50%"). This is the
-verdict the 2026-09-14/15 evaluations measured
-(docs/decisions/2026-09-search-quality.md). The graded 0/1/2 method and the
-Gemma-4 engine it targeted were evaluated on 2026-09-15 and rejected (user
-decision).
-
-Validator-contract semantics (the authoritative engine + client contracts
-live in INTEGRATION.md of https://gitlab-master.nvidia.com/omniverse/deeptag/parser-reranker):
-- ABSTAIN (Yes/No probability mass below threshold at every position) → retry
-  the call once; a persistent abstain is returned as is_match=true with
-  abstain=true and confidence 0.0 — the candidate KEEPS its retrieval-order
-  place, unscored. An abstain must never be treated as "No": rejecting on an
-  unreliable read silently deletes good results.
-- Transient failure (connect error / engine 5xx) → back off and retry once,
-  then fail-soft: 503/502, which the Explorer reads as "VLM unavailable": it
-  shows the VLM-unavailable banner and stops validating (verdicts already
-  collected keep their badges), leaving search intact. Scores are never
-  fabricated.
-
-Throughput: the serving owner's operational contract caps validation fan-out
-at 24 in-flight requests (engine-side 42.9 verdicts/s at that fan-out,
-measured on the earlier L40S engine; end-to-end the shim sustains less because it also fetches the
-thumbnails; more in-flight degrades p95 and adds nothing — the engine also
-serves query parsing), so this shim is tuned
-client-side:
-- ONE pooled keep-alive HTTP client (lifespan-managed) — no per-call TCP
-  handshake tax (~an RTT each on ~100 calls per search);
-- images reach the engine at most VALIDATE_MAX_EDGE (448) px on the longest
-  edge (2026-10-07): Isaac 6.2 stores every SimReady thumbnail at 1024x1024,
-  1,078 prompt tokens per verdict at that size against 250 at 448 (robots stay
-  256/512 px; anything at or under the cap passes through untouched). A JPEG
-  decodes at a reduced DCT scale first (draft), so the resize costs ~3 ms of
-  CPU per 1024-px image. Measured in docs/decisions/2026-09-search-quality.md:
-  65 vs 38 verdicts/s at saturation, 97 % of Yes/No verdicts unchanged;
-- an asyncio.Semaphore caps in-flight reranker calls at the contract's
-  fan-out ceiling, converting bursts into queueing at the sweet spot;
-- max_tokens 2: Qwen3.6 emits the verdict at position 0 (no wrapper tokens —
-  verified by probe); the position SCAN below is kept anyway, per the
-  integration contract, so a future wrapper-emitting model still extracts.
-
-Environment:
-    RERANKER_BASE_URL         OpenAI-compatible engine base URL
-                              (default http://simready-central-prod-cluster.nvidia.com/vlm-parser-reranker/v1)
-    RERANKER_MODEL            served model name (default "reranker")
-    RERANKER_API_KEY          bearer key for an engine that wants one (default
-                              none: the cluster engine is open). Sent on the
-                              engine's chat calls only, never on an image fetch.
-    RERANKER_NAME             display name in responses (default "Qwen3.6-35B-A3B-FP8";
-                              the `model` field in every verdict)
-    RERANKER_COVER_MIN        Yes/No verdict-token mass below this → abstain
-                              (default 0.2).
-    RERANKER_TIMEOUT_S        per-call timeout (default 30, per the serving
-                              contract; a 50-hit page drained in ≈1.2 s, 1.46 s p95, on the earlier L40S)
-    RERANKER_MAX_TOKENS       decode steps per call (default 2; raise if a
-                              future model buries the verdict deeper)
-    RERANKER_MAX_CONCURRENCY  in-flight cap toward the engine (default 48 for
-                              the RTX PRO 6000 engine with async scheduling and
-                              two API servers: 3 clients x c48 = 120.9 verdicts/s
-                              at p95 1.41 s; 24 was the L40-class ceiling)
-    VALIDATE_MAX_EDGE         downscale images whose longest edge exceeds this
-                              many pixels before sending (default 448 since
-                              2026-10-07; 4096 = no resize from 2026-09-21,
-                              512 before 2026-09-15). The verdict cache key
-                              includes it: changing it starts the shim's cache
-                              cold.
-    RERANKER_BATCH_CONCURRENCY  per-batch fan-out cap INSIDE the global
-                              semaphore (default 24) — two users' concurrent
-                              batches interleave at the engine instead of one
-                              queueing whole behind the other.
-    SHIM_GATEWAY_BASE_URL     the gateway's public listener, for shim-side
-                              thumbnail fetches (default http://gateway:8080).
-    IMAGES_TIMEOUT_S          per-thumbnail fetch timeout on the dedicated
-                              image client (default 5; slow thumbnails must
-                              not starve the engine connection pool).
-    BATCH_MAX_ITEMS           batch size ceiling (default 200; larger → 413).
-    BATCH_HEARTBEAT_S         blank-line heartbeat cadence on an otherwise
-                              quiet stream. COUPLED CONSTANT — the gateway's
-                              batch_stream proxy_read_timeout (60 s, hard-
-                              coded) and the Explorer's stream stall guard
-                              (45 s) are both sized off the default 10:
-                              raising it past ~40 trips the client guard,
-                              past 60 the gateway severs healthy streams.
-    VALIDATE_CACHE_SIZE       verdict-cache entries (default 4000000, ~0.9 GB
-                              at 240 bytes an entry; two entries per verdict;
-                              0 disables; 20000 until 2026-10-07).
-                              Verdicts are deterministic (temperature 0), so one
-                              (model, max-edge, query, image-bytes) tuple always
-                              scores the same — caching makes repeat waves
-                              (re-searches, pagination bounces, other USERS
-                              asking the same query) free. Complements the
-                              Explorer's per-browser verdict cache: this one is
-                              shared across every client of this shim. Keyed on
-                              the prompt too (a different query never
-                              coalesces).
-
-Tests (tests/test_scoring.py: the scorer, the payload builder, the verdict
-contract and the cache keys as pure functions; tests/test_routes.py: both
-routes through fastapi.testclient with the engine call and the image fetch
-monkeypatched, i.e. the prompt selection, the two cache paths, the NDJSON line
-protocol and the per-item vs fatal error split; no engine, no network), from
-the repo root:
-
-    cd deeptag-validate && pip install -r requirements-dev.txt && python -m pytest -q
-    # or, with no install step:
-    uv run --with pytest --with fastapi --with httpx --with pillow pytest deeptag-validate/tests
-
-CI runs the first form as the `test-deeptag-validate` job, which gates
-`build-deeptag-validate` exactly like the simready-api suite gates its image.
-"""
-import asyncio
-import base64
-import hashlib
-import io
-import json
-import logging
-import math
-import os
-import re
-from collections import OrderedDict
-from contextlib import asynccontextmanager
-from typing import List, Optional
-
-import httpx
-from fastapi import FastAPI, HTTPException
-from fastapi.responses import StreamingResponse
-from PIL import Image
-from pydantic import BaseModel
-
-logger = logging.getLogger("deeptag-validate")
-
-# The unified vlm-parser-reranker engine's "reranker" alias, OpenAI-compatible. Defaults to
-# the cluster engine's ingress; override per deployment. It does not gate on an API key.
-RERANKER_BASE_URL = os.environ.get("RERANKER_BASE_URL", "http://simready-central-prod-cluster.nvidia.com/vlm-parser-reranker/v1").rstrip("/")
-RERANKER_MODEL = os.environ.get("RERANKER_MODEL", "reranker")
-# A hosted engine (the HF Spaces' NVCF function / Inference Endpoint) wants a bearer key. It rides the
-# engine's POSTs alone: the same pooled client also fetches http image URLs a request names, and a
-# default header there would hand the key to any origin.
-RERANKER_API_KEY = os.environ.get("RERANKER_API_KEY", "").strip()
-_ENGINE_HEADERS = {"Authorization": f"Bearer {RERANKER_API_KEY}"} if RERANKER_API_KEY else None
-RERANKER_NAME = os.environ.get("RERANKER_NAME", "Qwen3.6-35B-A3B-FP8")
-
-# Yes/No verdict-token probability mass below this is unreliable → abstain
-# (treat as unscored, never as "No").
-COVER_MIN = float(os.environ.get("RERANKER_COVER_MIN", "0.2"))
-TIMEOUT_S = float(os.environ.get("RERANKER_TIMEOUT_S", "30"))
-# Qwen3.6 emits the verdict at position 0; one spare decode step is plenty and
-# more only burns shared engine compute (it also serves query parsing).
-RERANKER_MAX_TOKENS = int(os.environ.get("RERANKER_MAX_TOKENS", "2"))
-# The serving contract's fan-out ceiling: past 24 in-flight, p95 degrades and
-# throughput gains nothing.
-RERANKER_MAX_CONCURRENCY = int(os.environ.get("RERANKER_MAX_CONCURRENCY", "48"))
-# Vision-token cost scales with pixels; 448 px (longest edge) is the one view
-# the production Yes/No verdict was measured at — the same edge the server-side
-# verdict stage judges (docs/operations.md "Measured quality and the ordering
-# rule"). The cache key carries the edge, so changing it never re-serves a
-# verdict judged at another size.
-VALIDATE_MAX_EDGE = int(os.environ.get("VALIDATE_MAX_EDGE", "448"))
-
-# The asset's own facts in the words-only judge prompt (2026-10-07): "off" (the
-# 2026-09 prompt), "path" (the candidate's file path) or "path+caption" (+ the
-# index's one-line description). The client sends each hit's `path` and
-# `caption`; a hit without a path is judged without facts, and image queries
-# never use them (only words-only judging was evaluated). Measured in
-# docs/decisions/2026-09-search-quality.md.
-ASSET_CONTEXT_MODES = ("off", "path", "path+caption")
-JUDGE_ASSET_CONTEXT = os.environ.get("JUDGE_ASSET_CONTEXT", "off").strip().lower()
-if JUDGE_ASSET_CONTEXT not in ASSET_CONTEXT_MODES:
-    logger.warning(
-        "JUDGE_ASSET_CONTEXT=%r is not one of %s: judging without asset facts",
-        JUDGE_ASSET_CONTEXT, ", ".join(ASSET_CONTEXT_MODES),
-    )
-    JUDGE_ASSET_CONTEXT = "off"
-# Prompt-length bounds for the facts: a path keeps its tail (the file and its
-# folders name the asset), a caption its head.
-ASSET_PATH_MAX = 300
-ASSET_CAPTION_MAX = 200
-
-# Streamed batch validation (task #27). The per-batch cap nests INSIDE the
-# global engine semaphore: a 50-item batch occupies at most this many of the
-# 24 engine slots at once, so concurrent users' batches interleave fairly.
-RERANKER_BATCH_CONCURRENCY = int(os.environ.get("RERANKER_BATCH_CONCURRENCY", "24"))
-# Shim-side thumbnail fetches go through the gateway's public listener — the
-# same /images route (and shared nginx cache) the browsers use, index-scoped
-# by the x-usdsearch-storage-backend header. NEVER a client-supplied URL: the
-# base is pinned here and references travel as urlencoded query params only.
-SHIM_GATEWAY_BASE_URL = os.environ.get("SHIM_GATEWAY_BASE_URL", "http://gateway:8080").rstrip("/")
-IMAGES_TIMEOUT_S = float(os.environ.get("IMAGES_TIMEOUT_S", "5"))
-BATCH_MAX_ITEMS = int(os.environ.get("BATCH_MAX_ITEMS", "200"))
-BATCH_HEARTBEAT_S = float(os.environ.get("BATCH_HEARTBEAT_S", "10"))
-
-# Cross-user verdict cache: temperature-0 verdicts are pure functions of
-# (model, resize edge, query, image bytes), so identical pairs — from ANY
-# client — re-serve without an engine call. Insertion-order eviction. An entry
-# costs ~240 bytes (measured on python:3.11), and each fresh verdict writes two
-# (its reference key and its content key), so 4,000,000 entries hold 2,000,000
-# verdicts in ~0.9 GB: about half a day at the capacity plan's Phase 1 load
-# (2026-10-07; 20000 before was ~30 s of traffic at 600 checks/s). In-process
-# only: every shim restart starts cold.
-VALIDATE_CACHE_SIZE = int(os.environ.get("VALIDATE_CACHE_SIZE", "4000000"))
-_verdict_cache: "OrderedDict[str, object]" = OrderedDict()
-
-
-def _cache_key(query: str, image_url: str, query_image_digest: str = "") -> str:
-    """Content-keyed verdict-cache entry: the prompt, the PREPARED candidate
-    image and, for an image query, the query image's digest (every image query
-    shares one prompt, so without it two different query images would
-    coalesce)."""
-    h = hashlib.sha256()
-    for part in (RERANKER_MODEL, str(VALIDATE_MAX_EDGE), query, image_url, query_image_digest or ""):
-        h.update(part.encode())
-        h.update(b"\x00")
-    return h.hexdigest()
-
-
-def _ref_key(prompt: str, query_image_digest: str, index: str, ref: str) -> str:
-    """Reference-keyed verdict-cache entry for the batch path: keyed on the
-    IMAGE REFERENCE (image_key/asset_url + index slug) instead of prepared
-    bytes, so a cache hit costs no thumbnail fetch at all. The content key
-    (`_cache_key`) is still written alongside on every fresh verdict, so the
-    two paths coalesce. The query image participates as a PRE-COMPUTED digest
-    (image queries share one prompt and must not collide across different
-    reference images; hashing the multi-MB data URL per item would stall the
-    event loop; the route hashes it once)."""
-    h = hashlib.sha256()
-    for part in (RERANKER_MODEL, str(VALIDATE_MAX_EDGE), "ref", prompt, query_image_digest or "", index or "", ref):
-        h.update(part.encode())
-        h.update(b"\x00")
-    return h.hexdigest()
-
-
-def _cache_get(key: str):
-    p = _verdict_cache.get(key)
-    if p is not None:
-        _verdict_cache.move_to_end(key)
-    return p
-
-
-def _cache_put(key: str, score) -> None:
-    if VALIDATE_CACHE_SIZE <= 0:
-        return
-    _verdict_cache[key] = score
-    _verdict_cache.move_to_end(key)
-    while len(_verdict_cache) > VALIDATE_CACHE_SIZE:
-        _verdict_cache.popitem(last=False)
-
-# Firm, one-word judge prompt (a weak prompt makes GLM describe the image → the
-# logprob trick breaks). Text goes BEFORE the image in the message content.
-JUDGE_PROMPT_HEAD = 'You are a visual relevance judge for a 3D-asset search. Query: "{query}". '
-JUDGE_QUESTION = "Does the rendered asset match the query? Answer with exactly one word: Yes or No."
-JUDGE_PROMPT = JUDGE_PROMPT_HEAD + JUDGE_QUESTION
-# The asset facts (JUDGE_ASSET_CONTEXT), between the query and the question, as
-# evaluated on 2026-10-07.
-JUDGE_PATH_FACT = 'The candidate asset\'s file is "{path}". '
-JUDGE_CAPTION_FACT = 'Catalog description: "{caption}". '
-# An image alone (find-similar): usdsearch's own image-query question
-# (llm_client LOGPROB_JUDGE_PROMPT_IMAGE_QUERY without its caption clause),
-# word for word; the query image goes first. Without a dedicated prompt an
-# image search would be scored against 'Query: ""' — pure noise. On 40 known
-# pairs (2026-10-06) it matched the shim's earlier wording: every same-object
-# pair Yes, every unrelated pair No.
-JUDGE_IMAGE_PROMPT = (
-    "You are a visual relevance judge for a 3D-asset search. The first image is "
-    "the query image; the remaining images are rendered views of one asset. Does "
-    "the rendered asset match the query? Answer with exactly one word: Yes or No."
-)
-# An image WITH words: both count, the query image first. Not usdsearch's
-# caption clause ('… the query image and the text query is "…"; the remaining
-# images are rendered views of one asset'), which let the model judge the query
-# image itself against the words: on the same 40 pairs it passed 3-4 unrelated
-# candidates, this wording none (the words alone, the shim's earlier rule: 1).
-JUDGE_IMAGE_TEXT_PROMPT = (
-    'You are a visual relevance judge for a 3D-asset search. Query: "{query}", '
-    "together with the first image, which shows what the user is looking for. "
-    "The other image shows a candidate asset. Does the candidate asset match the "
-    "query? Answer with exactly one word: Yes or No."
-)
-YES_TOKENS = {"yes", "y", "true"}
-NO_TOKENS = {"no", "n", "false"}
-
-# One pooled keep-alive client for the process (created in lifespan): validation
-# is ~100 calls per search against a remote engine, and a per-request client
-# pays a fresh TCP handshake (an RTT) on every call.
-_reranker_client: Optional[httpx.AsyncClient] = None
-# Separate, short-timeout client for shim-side thumbnail fetches (batch path):
-# 50 image GETs must never compete with engine calls for the reranker pool,
-# and one slow origin must cost 5 s, not the engine's 30.
-_images_client: Optional[httpx.AsyncClient] = None
-# In-flight cap at the engine's measured saturation point — a burst beyond it
-# queues here instead of doubling everyone's latency engine-side.
-_reranker_sem = asyncio.Semaphore(RERANKER_MAX_CONCURRENCY)
-
-
-@asynccontextmanager
-async def _lifespan(app: FastAPI):
-    global _reranker_client, _images_client
-    _reranker_client = httpx.AsyncClient(
-        timeout=TIMEOUT_S,
-        limits=httpx.Limits(max_connections=64, max_keepalive_connections=64),
-    )
-    _images_client = httpx.AsyncClient(
-        # pool=None: waiting for a connection SLOT must queue, not error — a
-        # scalar timeout also bounds pool acquisition, and N concurrent
-        # batches × 12 workers over a finite pool would otherwise fabricate
-        # PoolTimeout "image fetch failed" verdicts for perfectly good
-        # thumbnails under load.
-        timeout=httpx.Timeout(connect=IMAGES_TIMEOUT_S, read=IMAGES_TIMEOUT_S, write=IMAGES_TIMEOUT_S, pool=None),
-        limits=httpx.Limits(max_connections=64, max_keepalive_connections=64),
-    )
-    try:
-        yield
-    finally:
-        await _reranker_client.aclose()
-        _reranker_client = None
-        await _images_client.aclose()
-        _images_client = None
-
-
-app = FastAPI(title="deeptag-validate", version="1.1.0", lifespan=_lifespan)
-
-
-class ValidateRequest(BaseModel):
-    query_text: Optional[str] = None
-    query_image: Optional[str] = None
-    images: List[str] = []
-    # The asset's facts for words-only judging (JUDGE_ASSET_CONTEXT).
-    path: Optional[str] = None
-    caption: Optional[str] = None
-
-
-def p_yes_from_logprobs(content):
-    """Scan token positions for the one carrying Yes/No mass (scan-mode is the
-    contract: robust to wrapper tokens some models emit before the verdict);
-    return (p_yes, cover) or (None, 0) if uncertain."""
-    for pos in content or []:
-        yes = no = 0.0
-        for entry in pos.get("top_logprobs", []) or []:
-            token = (entry.get("token") or "").strip().lower()
-            prob = math.exp(entry["logprob"])
-            if token in YES_TOKENS:
-                yes += prob
-            elif token in NO_TOKENS:
-                no += prob
-        if yes + no > COVER_MIN:
-            return yes / (yes + no), yes + no
-    return None, 0.0
-
-
-def _shrink_bytes(raw: bytes, src: str, was_data_url: bool) -> str:
-    """Sync (thread-pooled) half of image prep: decode, downscale if the longest
-    edge exceeds VALIDATE_MAX_EDGE (LANCZOS, JPEG q90), return a data URL. An
-    already-small data URL passes through UNTOUCHED; an already-small fetched
-    image is re-encoded as a data URL so the engine doesn't re-fetch it."""
-    with Image.open(io.BytesIO(raw)) as img:
-        w, h = img.size
-        if max(w, h) <= VALIDATE_MAX_EDGE:
-            logger.debug("image %dx%d <= %dpx: pass-through", w, h, VALIDATE_MAX_EDGE)
-            if was_data_url:
-                return src
-            fmt = (img.format or "png").lower()
-            mime = "image/jpeg" if fmt in ("jpg", "jpeg") else f"image/{fmt}"
-            return f"data:{mime};base64," + base64.b64encode(raw).decode()
-        # A JPEG decodes straight at a reduced DCT scale (1/2 for a 1024-px
-        # thumbnail; never below the cap): about half the shim CPU of a full
-        # decode before the resize. Other formats ignore it.
-        img.draft(None, (VALIDATE_MAX_EDGE, VALIDATE_MAX_EDGE))
-        rgb = img.convert("RGB")
-        rgb.thumbnail((VALIDATE_MAX_EDGE, VALIDATE_MAX_EDGE), Image.LANCZOS)
-        buf = io.BytesIO()
-        rgb.save(buf, format="JPEG", quality=90)
-        logger.debug(
-            "image resized %dx%d -> %dx%d (%d -> %d bytes)",
-            w, h, rgb.width, rgb.height, len(raw), buf.tell(),
-        )
-        return "data:image/jpeg;base64," + base64.b64encode(buf.getvalue()).decode()
-
-
-async def _prepare_image(src: str) -> str:
-    """Downscale-and-inline an image reference before it reaches the engine.
-    Defensive: anything unparseable is sent through untouched — a weird image
-    must degrade to the engine's own handling, never break validation."""
-    try:
-        if src.startswith("data:"):
-            _, _, b64 = src.partition(",")
-            raw = base64.b64decode(b64)
-            was_data_url = True
-        elif src.startswith(("http://", "https://")):
-            resp = await _reranker_client.get(src)
-            resp.raise_for_status()
-            raw = resp.content
-            was_data_url = False
-        else:
-            # Bare base64: the Explorer strips the data: prefix off its query
-            # image. Sent as is, the engine reads it as an image URL and 400s
-            # every image-only verdict; inline it as a data URL instead.
-            # Non-base64 strings still pass through (strict decode raises).
-            raw = base64.b64decode(src, validate=True)
-            was_data_url = False
-        return await asyncio.to_thread(_shrink_bytes, raw, src, was_data_url)
-    except Exception as exc:  # noqa: BLE001 — image prep must never break validation
-        logger.warning("image prep failed (%s: %s); sending original", type(exc).__name__, exc)
-        return src
-
-
-def _judge_payload(prompt: str, image_url: str, query_image_url: str = None) -> dict:
-    # Text before image(s) per the judge contract. For image queries the query
-    # image precedes the candidate (the image prompts' "first image" wording
-    # depends on this order). Plain sampling parameters only:
-    # no structured-outputs grammar (that belonged to the retired graded
-    # method), so an engine without a grammar backend serves this unchanged.
-    content = [{"type": "text", "text": prompt}]
-    if query_image_url:
-        content.append({"type": "image_url", "image_url": {"url": query_image_url}})
-    content.append({"type": "image_url", "image_url": {"url": image_url}})
-    return {
-        "model": RERANKER_MODEL,
-        "temperature": 0,
-        "max_tokens": RERANKER_MAX_TOKENS,
-        "logprobs": True,
-        "top_logprobs": 20,
-        "chat_template_kwargs": {"enable_thinking": False},
-        "messages": [{"role": "user", "content": content}],
-    }
-
-
-async def _call_reranker(payload: dict) -> dict:
-    # Transient-failure semantics from the validator contract: back off and
-    # retry ONCE on connect errors / engine 5xx before failing soft. The
-    # semaphore holds in-flight calls at the engine's saturation point.
-    for attempt in (1, 2):
-        try:
-            async with _reranker_sem:
-                resp = await _reranker_client.post(
-                    f"{RERANKER_BASE_URL}/chat/completions", json=payload, headers=_ENGINE_HEADERS
-                )
-            resp.raise_for_status()
-            return resp.json()
-        except (httpx.ConnectError, httpx.ConnectTimeout, httpx.ReadTimeout):
-            if attempt == 1:
-                await asyncio.sleep(0.25)
-                continue
-            # Engine down / slow → fail-soft. The Explorer reads 503 as
-            # "VLM unavailable": banner, validation stops; search is unaffected.
-            raise HTTPException(status_code=503, detail="reranker unavailable")
-        except httpx.HTTPStatusError as exc:
-            if attempt == 1 and exc.response.status_code >= 500:
-                await asyncio.sleep(0.25)
-                continue
-            raise EngineRejected(exc.response.status_code, f"reranker error: {exc}")
-        except httpx.HTTPError as exc:
-            raise HTTPException(status_code=502, detail=f"reranker error: {exc}")
-
-
-# The configuration refusal's message: fixed, never the engine's own error (it names the engine's URL).
-IMAGE_QUERY_REFUSED = "the validation engine refused an image query (it may take one image per prompt)"
-
-
-class EngineRejected(HTTPException):
-    """The engine ANSWERED with an error status (a 502 to the client, as before; the batch route keeps
-    treating it per item). `engine_status` lets the per-image route tell a refused request shape apart."""
-
-    def __init__(self, engine_status: int, detail: str):
-        super().__init__(status_code=502, detail=detail)
-        self.engine_status = engine_status
-
-
-def _abstain_verdict() -> dict:
-    # Persistent abstain → keep the candidate in retrieval order, unscored
-    # (is_match=true so the Explorer does not reject it; confidence 0.0 and
-    # abstain=true make the non-verdict explicit rather than a fabricated
-    # score).
-    return {"is_match": True, "confidence": 0.0, "similarity_score": 0.0, "abstain": True, "model": RERANKER_NAME}
-
-
-def _verdict_from_score(score: float) -> dict:
-    """The ONE place a verdict dict is built from a cached or fresh SCORE
-    RECORD (the P(Yes) float), for both routes and both cache paths — a
-    boundary that lived in three copies would silently regress on cache hits.
-    similarity_score is P(Yes) x 100 (usdsearch's scale); confidence stays
-    P(Yes)."""
-    return {
-        "is_match": score > 0.5,
-        "confidence": round(score, 4),
-        "similarity_score": round(score * 100, 2),
-        "model": RERANKER_NAME,
-    }
-
-
-def _extract_score(data: dict):
-    """Pull the P(Yes) score record out of one engine response, or None
-    (abstain)."""
-    content = ((data.get("choices") or [{}])[0].get("logprobs") or {}).get("content")
-    p_yes, _cover = p_yes_from_logprobs(content)
-    return p_yes
-
-
-def _fact_text(value, limit: int, keep_tail: bool = False) -> str:
-    """One asset fact as prompt text: whitespace collapsed, double quotes made
-    single (the fact sits inside a quoted span), at most `limit` characters."""
-    text = " ".join(str(value or "").split()).replace('"', "'")
-    if len(text) > limit:
-        text = text[-limit:] if keep_tail else text[:limit]
-    return text
-
-
-_ISAAC_ROOT = re.compile(r"/Isaac/[^/]+/Isaac/(.+)$")
-# A Hugging Face dataset file (the hf-* indexes, the HF Spaces): below the dataset, and below
-# `resolve|blob/<revision>/` where the key pins one, e.g.
-# hf://huggingface.co/datasets/nvidia/simready-assets/resolve/2464ca32…/fixture_island_a01/simready_usd/x.usd
-# → fixture_island_a01/simready_usd/x.usd.
-_HF_DATASET_ROOT = re.compile(
-    r"^(?:[a-z][a-z0-9+.-]*://)?huggingface\.co/datasets/[^/]+/[^/]+/(?:(?:resolve|blob)/[^/]+/)?(.+)$", re.I
-)
-
-
-def _asset_path(path) -> str:
-    """The path as the judge reads it: below the release-agnostic Isaac content
-    root (`Robots/Unitree/H2/H2.usda`, as evaluated), or below a Hugging Face
-    dataset's root, else without its leading slash."""
-    p = str(path or "").strip()
-    m = _ISAAC_ROOT.search(p) or _HF_DATASET_ROOT.match(p)
-    return _fact_text(m.group(1) if m else p.lstrip("/"), ASSET_PATH_MAX, keep_tail=True)
-
-
-def _asset_facts(path, caption) -> str:
-    """The words-only prompt's asset facts under JUDGE_ASSET_CONTEXT, or "":
-    the file path, then (path+caption) the caption. No path → no facts, so
-    only the evaluated combinations are ever sent."""
-    if JUDGE_ASSET_CONTEXT == "off":
-        return ""
-    p = _asset_path(path)
-    if not p:
-        return ""
-    facts = JUDGE_PATH_FACT.format(path=p)
-    caption_text = _fact_text(caption, ASSET_CAPTION_MAX) if JUDGE_ASSET_CONTEXT == "path+caption" else ""
-    if caption_text:
-        facts += JUDGE_CAPTION_FACT.format(caption=caption_text)
-    return facts
-
-
-def _select_prompt(query_text: str, has_query_image: bool, path=None, caption=None) -> str:
-    """The judge prompt for one candidate, over the stripped query text: an
-    image with words → JUDGE_IMAGE_TEXT_PROMPT, an image alone →
-    JUDGE_IMAGE_PROMPT, words alone → JUDGE_PROMPT, with the candidate's facts
-    before the question when JUDGE_ASSET_CONTEXT is on (a request with neither
-    words nor image keeps the historical 'Query: ""' prompt)."""
-    text = (query_text or "").strip()
-    if has_query_image:
-        return JUDGE_IMAGE_TEXT_PROMPT.format(query=text) if text else JUDGE_IMAGE_PROMPT
-    facts = _asset_facts(path, caption)
-    if not facts:
-        return JUDGE_PROMPT.format(query=text)
-    return JUDGE_PROMPT_HEAD.format(query=text) + facts + JUDGE_QUESTION
-
-
-def _query_image_digest(query_image: Optional[str]) -> str:
-    """Hash the (possibly multi-MB) query image ONCE per request — the cache
-    keys run per item, and per-item sha256 over a full data URL would stall
-    the loop."""
-    return hashlib.sha256(query_image.encode()).hexdigest() if query_image else ""
-
-
-async def _score_prepared(prompt: str, image_url: str, query_image_url: str = None):
-    """One verdict for a PREPARED (downscaled, inlined) image. Returns
-    (score | None, verdict_dict); score None means persistent abstain (the
-    dict already carries abstain=true; the CALLER decides caching, and
-    abstains are never cached). The score is the cache record (see
-    _verdict_from_score). Raises HTTPException 502/503 when the engine is
-    unreachable (fail-soft to the Explorer's "VLM unavailable")."""
-    data = await _call_reranker(_judge_payload(prompt, image_url, query_image_url))
-    score = _extract_score(data)
-    if score is None:
-        # Abstain semantics: retry once — never treat an unreliable read as "No".
-        data = await _call_reranker(_judge_payload(prompt, image_url, query_image_url))
-        score = _extract_score(data)
-    if score is None:
-        return None, _abstain_verdict()
-    return score, _verdict_from_score(score)
-
-
-@app.get("/health")
-def health():
-    return {"status": "ok", "reranker": RERANKER_BASE_URL, "model": RERANKER_MODEL, "asset_context": JUDGE_ASSET_CONTEXT}
-
-
-@app.post("/vlm_validate/search_result")
-async def validate(req: ValidateRequest):
-    if not req.images:
-        raise HTTPException(status_code=400, detail="no image provided")
-    # The same judge as the batch route: a query image (the Explorer sends it
-    # bare base64 when it finishes a cut batch stream here) goes to the engine
-    # first, under the image prompts.
-    prompt = _select_prompt(req.query_text, bool(req.query_image), req.path, req.caption)
-    query_image_url = await _prepare_image(req.query_image) if req.query_image else None
-    image_url = await _prepare_image(req.images[0])
-    # Serve deterministic repeats from the verdict cache — no engine call. The
-    # key hashes the PREPARED image (post-resize), so the same asset arriving
-    # as a data URL or a fetched http URL coalesces once prepared.
-    ckey = _cache_key(prompt, image_url, _query_image_digest(req.query_image))
-    cached = _cache_get(ckey) if VALIDATE_CACHE_SIZE > 0 else None
-    if cached is not None:
-        return {**_verdict_from_score(cached), "cached": True}
-    try:
-        score, verdict = await _score_prepared(prompt, image_url, query_image_url)
-    except EngineRejected as exc:
-        # An engine that takes one image per prompt (the dev HF Space's NVCF function) answers an image
-        # query (query image + candidate) with a 4xx. Nothing is down: answer usdsearch's configuration
-        # refusal, which the Space's Explorer reads as "this search goes unvalidated" instead of the
-        # VLM-unavailable banner. Central's Explorer reads any 503 as before.
-        if query_image_url and 400 <= exc.engine_status < 500 and exc.engine_status not in (408, 429):
-            raise HTTPException(
-                status_code=503,
-                detail={"error_type": "configuration", "retryable": False, "message": IMAGE_QUERY_REFUSED},
-            ) from exc
-        raise
-    if score is not None:
-        _cache_put(ckey, score)
-    return verdict
-
-
-class BatchItem(BaseModel):
-    id: str
-    image_key: Optional[str] = None
-    asset_url: Optional[str] = None
-    # The hit's source.path and the index's caption for it: facts for
-    # words-only judging (JUDGE_ASSET_CONTEXT).
-    path: Optional[str] = None
-    caption: Optional[str] = None
-
-
-class BatchValidateRequest(BaseModel):
-    query_text: Optional[str] = None
-    query_image: Optional[str] = None
-    # The index SLUG (x-usdsearch-storage-backend value) the shim forwards on
-    # its /images fetches — routing, api-key injection, and the shared image
-    # cache all key off it upstream. Empty → the gateway's default index.
-    index: Optional[str] = None
-    # In the client's DISPLAY (retrieval) order — dispatch preserves it, so
-    # the most-likely matches score first and the ranked flow fills top-down.
-    items: List[BatchItem] = []
-
-
-def _ndjson(obj) -> str:
-    return json.dumps(obj, separators=(",", ":")) + "\n"
-
-
-@app.post("/vlm_validate/batch_stream")
-async def validate_batch(req: BatchValidateRequest):
-    """Streamed batch validation (NDJSON). Line protocol:
-
-        {"id", "cached", "verdict": {is_match, confidence, similarity_score,
-                                     model[, abstain]}}   — one per scored hit
-        {"id", "error": "..."}          — per-hit failure (bad thumbnail etc.);
-                                          NEVER poisons the rest of the batch
-        ""                              — blank heartbeat every BATCH_HEARTBEAT_S
-        {"fatal": "..."}                — engine unreachable; treat as 502/503
-        {"done": {completed, errored, cached}}  — ALWAYS last on a clean finish;
-                                          its absence means the stream was cut
-                                          (the client re-POSTs the missing ids)
-
-    Shim-cache hits flush first (no engine, no image fetch — the reference-
-    keyed entries); misses dispatch in request order under the per-batch cap.
-    Client disconnect cancels every pending worker; `async with _reranker_sem`
-    releases on cancellation, so an abandoned query frees its engine slots."""
-    if not req.items:
-        raise HTTPException(status_code=400, detail="no items")
-    if len(req.items) > BATCH_MAX_ITEMS:
-        raise HTTPException(status_code=413, detail=f"batch exceeds {BATCH_MAX_ITEMS} items")
-    if not (req.query_text or req.query_image):
-        raise HTTPException(status_code=400, detail="no query")
-    index = (req.index or "").strip()
-    qimg_digest = _query_image_digest(req.query_image)
-    # An image query (alone or with words): the judge sees the query image
-    # first, then each candidate. Prepared (downscaled) once. With asset facts
-    # on, the prompt (and so every cache key) is per item.
-    def prompt_for(it: BatchItem) -> str:
-        return _select_prompt(req.query_text, bool(req.query_image), it.path, it.caption)
-
-    query_image_url = await _prepare_image(req.query_image) if req.query_image else None
-
-    # Phase 0 — reference-keyed cache pass: free verdicts, emitted first.
-    cached_lines = []  # (kind, line) — kind: "verdict" | "error"
-    misses = []
-    for it in req.items:
-        ref = it.image_key or it.asset_url
-        if not ref:
-            cached_lines.append(("error", _ndjson({"id": it.id, "error": "no image reference"})))
-            continue
-        cached = _cache_get(_ref_key(prompt_for(it), qimg_digest, index, ref)) if VALIDATE_CACHE_SIZE > 0 else None
-        if cached is not None:
-            cached_lines.append(
-                ("verdict", _ndjson({
-                    "id": it.id,
-                    "cached": True,
-                    "verdict": _verdict_from_score(cached),
-                }))
-            )
-        else:
-            misses.append(it)
-
-    queue: asyncio.Queue = asyncio.Queue()
-    batch_sem = asyncio.Semaphore(RERANKER_BATCH_CONCURRENCY)
-
-    async def worker(it: BatchItem):
-        async with batch_sem:
-            ref = it.image_key or it.asset_url
-            prompt = prompt_for(it)
-            # Pinned base + urlencoded params ONLY — a reference can never
-            # change the request target (no SSRF surface); image_key (a
-            # specific matched view) wins over asset_url, mirroring the
-            # Explorer's own /images precedence.
-            params = {"image_key": it.image_key} if it.image_key else {"asset_url": it.asset_url}
-            headers = {"x-usdsearch-storage-backend": index} if index else {}
-            try:
-                resp = await _images_client.get(f"{SHIM_GATEWAY_BASE_URL}/images", params=params, headers=headers)
-                resp.raise_for_status()
-                raw = resp.content
-            except Exception as exc:  # noqa: BLE001 — one bad thumbnail must not poison the batch
-                await queue.put(("error", _ndjson({"id": it.id, "error": f"image fetch failed: {type(exc).__name__}"})))
-                return
-            try:
-                # Raw bytes straight into the thread-pooled shrink: no base64
-                # round-trip on the loop, and the data URL's mime comes from
-                # PIL's own sniff (img.format), never the upstream's
-                # content-type header (S3-ish origins say octet-stream, which
-                # OpenAI-compatible engines 400 on).
-                image_url = await asyncio.to_thread(_shrink_bytes, raw, "", False)
-            except Exception as exc:  # noqa: BLE001 — undecodable bytes must not reach the engine
-                await queue.put(("error", _ndjson({"id": it.id, "error": f"image decode failed: {type(exc).__name__}"})))
-                return
-            try:
-                score, verdict = await _score_prepared(prompt, image_url, query_image_url)
-            except HTTPException as exc:
-                if exc.status_code == 503:
-                    # Engine UNREACHABLE — the whole batch is over.
-                    await queue.put(("fatal", _ndjson({"fatal": exc.detail})))
-                else:
-                    # Engine rejected THIS item (4xx→502: e.g. an image it
-                    # can't decode) — per-hit, never batch-fatal.
-                    await queue.put(("error", _ndjson({"id": it.id, "error": f"engine rejected item: {exc.detail}"})))
-                return
-            except Exception as exc:  # noqa: BLE001 — a dead worker with no line would
-                # strand `remaining` and heartbeat the stream FOREVER (the
-                # blank lines reset every downstream timeout). Anything
-                # unexpected (non-JSON 200 from a proxy, logprob shape drift)
-                # settles the hit as a per-hit error instead.
-                logger.warning("batch worker failed (%s: %s)", type(exc).__name__, exc)
-                await queue.put(("error", _ndjson({"id": it.id, "error": f"scoring failed: {type(exc).__name__}"})))
-                return
-            if score is not None:
-                _cache_put(_ref_key(prompt, qimg_digest, index, ref), score)
-                _cache_put(_cache_key(prompt, image_url, qimg_digest), score)
-            await queue.put(("verdict", _ndjson({"id": it.id, "cached": False, "verdict": verdict})))
-
-    async def stream():
-        completed = sum(1 for kind, _ in cached_lines if kind == "verdict")
-        errored = len(cached_lines) - completed
-        cached_count = completed
-        # Workers are created INSIDE the generator: a client that aborts
-        # before the body is ever iterated (POST → immediate abort) must not
-        # leak a batch onto the shared engine — a never-started generator
-        # never runs its finally, so task lifetime has to live in this frame.
-        # Dispatch in REQUEST order — the per-batch semaphore wakes FIFO, so
-        # completion approximates retrieval order and the flow's top fills
-        # first.
-        tasks = [asyncio.create_task(worker(it)) for it in misses]
-        try:
-            for _, ln in cached_lines:
-                yield ln
-            remaining = len(tasks)
-            while remaining > 0:
-                try:
-                    kind, ln = await asyncio.wait_for(queue.get(), timeout=BATCH_HEARTBEAT_S)
-                except asyncio.TimeoutError:
-                    yield "\n"  # heartbeat — keeps proxies from severing a quiet stream
-                    continue
-                yield ln
-                if kind == "fatal":
-                    return  # finally cancels the rest; no `done` line on a fatal
-                remaining -= 1
-                if kind == "error":
-                    errored += 1
-                else:
-                    completed += 1
-            yield _ndjson({"done": {"completed": completed, "errored": errored, "cached": cached_count}})
-        finally:
-            # Client gone, fatal, or clean finish — nothing may keep burning
-            # shared engine slots. Cancellation releases the semaphores.
-            for t in tasks:
-                t.cancel()
-
-    return StreamingResponse(stream(), media_type="application/x-ndjson")

docker/hf-space/hf-storage-api/SOURCE.txt

@@ -1,3 +1,3 @@
 # Generated by scripts/extract_hf_storage_api.sh — do not hand-edit.
-source_image=nvcr.io/m3sujtetvf5w/usdsearch/hf-storage-api:71b7537
-extracted_at=2026-09-30T07:11:27Z
+source_image=nvcr.io/m3sujtetvf5w/usdsearch/hf-storage-api:417d9af
+extracted_at=2026-10-08T22:52:12Z

docker/hf-space/hf-storage-api/constraints-hf-storage-api.txt

@@ -10,10 +10,10 @@ httpx==0.28.1
 idna==3.20
 packaging==26.3
 protobuf==7.36.2
-pydantic==2.13.5
-pydantic_core==2.46.5
+pydantic==2.14.0
+pydantic_core==2.50.0
 pydantic-settings==2.15.0
-python-dotenv==1.2.3
+python-dotenv==1.2.4
 PyYAML==6.0.3
 typing_extensions==4.16.0
 typing-inspection==0.4.4

docker/hf-space/hf-storage-api/hf_storage_api/addressing.py

@@ -32,6 +32,13 @@ INBOUND_SCHEMES = frozenset({"hf", "https", "http"})
 
 DEFAULT_SCHEME = "https"
 
+# The only characters `build()` escapes: the ones that do not survive a round
+# trip through `parse()` raw — `%` because `parse()` unquotes, `#` and `?`
+# because `urlsplit` cuts the path at them. Escaping anything more (`@`, space,
+# non-ASCII) would change addresses USD Search has already indexed, and those
+# round-trip fine as they are. `%` goes first so the others are not re-escaped.
+_ADDRESS_ESCAPES = (("%", "%25"), ("#", "%23"), ("?", "%3F"))
+
 
 @dataclass(frozen=True)
 class Address:
@@ -136,10 +143,17 @@ def build(
     a fetchable HF URL — HF requires a ``/resolve/<rev>/`` or ``/blob/<rev>/`` segment.
     It still round-trips through :func:`parse`, which is what this service needs of it;
     see the plan's §6.1 on pairing ``https`` with ``URI_REVISION_MODE``.
+
+    ``path`` is escaped just enough to round-trip (see ``_ADDRESS_ESCAPES``): a file
+    named ``Package%401.0.0.json`` is emitted as ``Package%25401.0.0.json``, which
+    is also the form the Hub itself needs.
     """
     prefix = REPO_TYPE_PREFIX.get(repo_type, "")
     ref = f"resolve/{revision}" if revision else ""
-    parts = [p for p in (prefix, repo_id, ref, path.strip("/")) if p]
+    path = path.strip("/")
+    for char, escaped in _ADDRESS_ESCAPES:
+        path = path.replace(char, escaped)
+    parts = [p for p in (prefix, repo_id, ref, path) if p]
     out = f"{scheme}://{host}/" + "/".join(parts)
     if is_folder and not out.endswith("/"):
         out += "/"

docker/hf-space/hf-storage-api/hf_storage_api/catalog/loader.py

@@ -10,6 +10,14 @@ blipped.
 Every transport is polled, not just ``url``: a catalog file is mounted from a
 ConfigMap or written by the catalog-sync sidecar, and both change under a running
 process (sidecar plan §3).
+
+The primary catalog may be joined by *extra* catalog files
+(``HF_ADAPTER_CATALOG_EXTRA_PATHS``) — in practice a YAML catalog that walks some
+repos, plus a URL list the sidecar keeps in step with a GitHub submission file.
+Each source is read, discovered and guarded on its own and keeps its own last
+good copy, so a broken sync leaves the YAML repos serving and a changed URL list
+does not re-walk them. The results are merged into one catalog: still one access
+boundary, still one repo per key.
 """
 
 from __future__ import annotations
@@ -17,6 +25,7 @@ from __future__ import annotations
 import asyncio
 import hashlib
 import logging
+from dataclasses import dataclass
 from pathlib import Path
 
 import httpx
@@ -24,7 +33,7 @@ import yaml
 
 from ..hf.client import HFClient
 from ..settings import Settings
-from .model import Catalog, LayoutConfig, RepoConfig
+from .model import Catalog, LayoutConfig, RepoConfig, looks_like_sha
 from .urllist import UrlListError, looks_like_url_list, normalize_asset_urls, parse_url_list
 
 logger = logging.getLogger(__name__)
@@ -65,41 +74,71 @@ def parse_text(text: str, *, fmt: str = "auto", layout: LayoutConfig | None = No
     return parse_catalog(text)
 
 
+@dataclass
+class _Source:
+    """One catalog input and what it looked like last time it was read."""
+
+    path: Path | None  # None: the primary catalog, fetched from `catalog_url`
+    primary: bool
+    etag: str | None = None
+    # A cheap (mtime, size) gate in front of an authoritative content digest.
+    stat: tuple[int, int] | None = None
+    digest: str | None = None
+    # Parsed, discovered and guarded — what this source contributes right now.
+    last_good: Catalog | None = None
+
+    @property
+    def name(self) -> str:
+        return str(self.path) if self.path is not None else "catalog URL"
+
+
 class CatalogLoader:
     """Loads the catalog and keeps it fresh, holding the last good copy."""
 
     def __init__(self, settings: Settings, hf: HFClient) -> None:
         self._settings = settings
         self._hf = hf
-        self._etag: str | None = None
         self._last_good: Catalog | None = None
         self._task: asyncio.Task | None = None
-        # What the catalog file looked like last time we read it: a cheap
-        # (mtime, size) gate in front of an authoritative content digest.
-        self._stat: tuple[int, int] | None = None
-        self._digest: str | None = None
+        primary_path = None if settings.catalog_source == "url" else settings.catalog_path
+        self._sources = [_Source(primary_path, primary=True)] + [
+            _Source(path, primary=False) for path in settings.extra_catalog_paths
+        ]
 
     @property
     def current(self) -> Catalog | None:
         return self._last_good
 
     async def load(self) -> Catalog:
-        """Load once. Raises on the first load; later failures keep the last good."""
-        try:
-            text = await self._fetch_text()
-            if text is None:  # 304 Not Modified
-                return self._last_good  # type: ignore[return-value]
-            catalog = self._parse(text)
-        except Exception as exc:
-            if self._last_good is None:
-                raise
-            logger.warning("catalog refresh failed (%s); keeping the last good copy", exc)
-            return self._last_good
+        """Load once. Raises on the first load; later failures keep the last good.
 
-        if self._settings.catalog_source == "discover" or any(r.discover is not None for r in catalog.repos):
-            catalog = await self._discover(catalog)
+        "Last good" is kept per source: one broken input holds its own previous
+        contribution while the others still update.
+        """
+        parts: list[Catalog] = []
+        changed = False
+        for source in self._sources:
+            try:
+                part = await self._load_source(source)
+            except Exception as exc:
+                if source.last_good is None:
+                    raise
+                logger.warning(
+                    "catalog refresh failed for %s (%s); keeping its last good copy", source.name, exc
+                )
+                part = None
+            if part is not None:
+                changed = True
+            parts.append(source.last_good)  # type: ignore[arg-type] - set by now
 
+        if not changed and self._last_good is not None:
+            return self._last_good
+
+        catalog = merge_catalogs(parts)
+        # Per-source checks cannot see across sources: the same repo named by the
+        # YAML catalog and by the synced list is only visible here.
         validate_catalog(catalog)
+        self._check_revisions_are_addressable(catalog)
         self._last_good = catalog
         logger.info(
             "catalog loaded: %d repo(s), %d asset(s)",
@@ -108,24 +147,50 @@ class CatalogLoader:
         )
         return catalog
 
-    async def _fetch_text(self) -> str | None:
+    def _check_revisions_are_addressable(self, catalog: Catalog) -> None:
+        """Several revisions of one repo are told apart only by the
+        ``/resolve/<sha>/`` segment in their addresses — which ``never`` drops."""
+        if self._settings.uri_revision_mode != "never":
+            return
+        keys = [repo.key for repo in catalog.repos]
+        if len(keys) != len(set(keys)):
+            raise CatalogError(
+                "the catalog serves a repo at several revisions, which needs the revision in every "
+                "address — HF_ADAPTER_URI_REVISION_MODE=never removes it"
+            )
+
+    async def _load_source(self, source: _Source) -> Catalog | None:
+        """Refresh one source. ``None`` means "unchanged since last time"."""
+        text = await self._fetch_text(source)
+        if text is None:
+            return None
+        catalog = self._parse(text)
+
+        discover = source.primary and self._settings.catalog_source == "discover"
+        if discover or any(r.discover is not None for r in catalog.repos):
+            catalog = await self._discover(catalog)
+
+        validate_catalog(catalog)
+        source.last_good = catalog
+        return catalog
+
+    async def _fetch_text(self, source: _Source) -> str | None:
         """Fetch the raw catalog. ``None`` means "unchanged since last time"."""
-        source = self._settings.catalog_source
-        if source in ("file", "discover"):
-            return self._read_file()
+        if source.path is not None:
+            return self._read_file(source)
 
-        headers = {"If-None-Match": self._etag} if self._etag else {}
+        headers = {"If-None-Match": source.etag} if source.etag else {}
         async with httpx.AsyncClient(timeout=self._settings.http_timeout) as client:
             response = await client.get(self._settings.catalog_url or "", headers=headers)
-        if response.status_code == 304 and self._last_good is not None:
+        if response.status_code == 304 and source.last_good is not None:
             logger.debug("catalog unchanged (304)")
             return None
         response.raise_for_status()
-        self._etag = response.headers.get("ETag")
+        source.etag = response.headers.get("ETag")
         return response.text
 
-    def _read_file(self) -> str | None:
-        """Read the catalog file, or ``None`` when its contents have not changed.
+    def _read_file(self, source: _Source) -> str | None:
+        """Read a catalog file, or ``None`` when its contents have not changed.
 
         Two gates, because they answer different questions. ``stat()`` is what
         makes polling free — it is the only syscall in the steady state. The
@@ -137,23 +202,20 @@ class CatalogLoader:
         bytes that failed to parse must not be retried every interval, or one
         bad file becomes one log line per tick forever.
         """
-        path = self._settings.catalog_path
-        if path is None:  # pragma: no cover - Settings validates this
-            raise CatalogError("catalog_path is unset")
-        path = Path(path)
+        path = Path(source.path)  # type: ignore[arg-type] - only called for files
 
         info = path.stat()
         stamp = (info.st_mtime_ns, info.st_size)
-        if self._last_good is not None and stamp == self._stat:
+        if source.last_good is not None and stamp == source.stat:
             return None
-        self._stat = stamp
+        source.stat = stamp
 
         text = path.read_text(encoding="utf-8")
         digest = hashlib.sha256(text.encode("utf-8")).hexdigest()
-        if self._last_good is not None and digest == self._digest:
+        if source.last_good is not None and digest == source.digest:
             logger.debug("catalog file touched but unchanged (%s)", path)
             return None
-        self._digest = digest
+        source.digest = digest
         return text
 
     def _parse(self, text: str) -> Catalog:
@@ -219,9 +281,10 @@ class CatalogLoader:
         was silently ignored until the pod restarted — the asset set stayed at
         whatever it was at boot while everything upstream reported success.
 
-        ``discover`` re-walks the repo only when the file that names the repos
-        changed. A repo-wide tree read on a timer is a much bigger promise than
-        this loop should make on its own (plan §5.2).
+        ``discover`` re-walks a repo only when the file that names it changed —
+        an extra catalog file changing does not re-walk the primary's repos. A
+        repo-wide tree read on a timer is a much bigger promise than this loop
+        should make on its own (plan §5.2).
 
         Polling rather than watching: ``stat()`` behaves the same on an
         ``emptyDir`` file replaced by ``os.replace``, on a ConfigMap's symlink
@@ -259,6 +322,26 @@ class CatalogLoader:
             self._task = None
 
 
+def merge_catalogs(parts: list[Catalog]) -> Catalog:
+    """Join several catalogs into one, each repo keeping its own endpoint and layout.
+
+    ``endpoint`` and ``layout`` are catalog-level defaults, so a repo relying on
+    its own catalog's defaults would silently pick up the first catalog's instead
+    — a URL list inferring a different manifest, say. Pinning them per repo
+    first makes the merge lossless. Duplicates are left for
+    :func:`validate_catalog` to reject.
+    """
+    if len(parts) == 1:
+        return parts[0]
+    repos = [
+        repo.model_copy(update={"endpoint": repo.endpoint or part.endpoint, "layout": part.layout_for(repo)})
+        for part in parts
+        for repo in part.repos
+    ]
+    first = parts[0]
+    return Catalog(endpoint=first.endpoint, layout=first.layout, repos=repos)
+
+
 def validate_catalog(catalog: Catalog) -> None:
     """Reject a catalog that would serve nothing, or serve it ambiguously.
 
@@ -269,14 +352,25 @@ def validate_catalog(catalog: Catalog) -> None:
     """
     if not catalog.repos:
         raise CatalogError("catalog lists no repos")
-    seen: set[str] = set()
+    revisions: dict[str, list[str]] = {}
     for repo in catalog.repos:
         if "/" not in repo.repo_id:
             raise CatalogError(f"repo_id must be <owner>/<name>: {repo.repo_id!r}")
-        if repo.key in seen:
-            raise CatalogError(f"duplicate repo in catalog: {repo.repo_id!r}")
-        seen.add(repo.key)
+        if repo.revision in revisions.setdefault(repo.key, []):
+            raise CatalogError(f"duplicate repo in catalog: {repo.repo_id!r} at revision {repo.revision!r}")
+        revisions[repo.key].append(repo.revision)
         if not repo.assets and repo.discover is None:
             raise CatalogError(
                 f"repo {repo.repo_id!r} lists no assets and no discover block — it would serve nothing"
             )
+    # One repo at several revisions is served side by side, each address carrying
+    # its own `/resolve/<sha>/` — which is what tells them apart. A moving ref has
+    # no fixed sha to put there, so it can only be served alone.
+    for key, revs in revisions.items():
+        moving = [r for r in revs if not looks_like_sha(r)]
+        if len(revs) > 1 and moving:
+            raise CatalogError(
+                f"{key.split(':', 1)[1]} appears at {len(revs)} revisions ({', '.join(sorted(revs))}); "
+                f"serving several revisions of one repo needs each pinned to a commit sha, "
+                f"but {', '.join(repr(r) for r in moving)} is a moving ref"
+            )

docker/hf-space/hf-storage-api/hf_storage_api/catalog/model.py

@@ -16,6 +16,12 @@ from pydantic import BaseModel, Field, model_validator
 
 DEFAULT_HOST = "huggingface.co"
 
+
+def looks_like_sha(revision: str) -> bool:
+    """Is this a pinned commit sha rather than a moving ref like ``main``?"""
+    return len(revision) == 40 and all(c in "0123456789abcdef" for c in revision.lower())
+
+
 # Defaults below are written for `catalog_kind="manifest"` (nvidia/simready-assets).
 # `catalog_kind="wrapp"` (nvidia/srp-staging-controlled) and `catalog_kind=
 # "package_entry"` (nvidia/SimReady-Internal) each swap in their own defaults for
@@ -166,8 +172,14 @@ class Catalog(BaseModel):
         return repo.layout or self.layout
 
     def repo(self, repo_type: str, repo_id: str) -> RepoConfig | None:
+        """The first entry for this repo. Use :meth:`repos_for` where a repo may be
+        served at several pinned revisions."""
+        return next(iter(self.repos_for(repo_type, repo_id)), None)
+
+    def repos_for(self, repo_type: str, repo_id: str) -> list[RepoConfig]:
+        """Every entry for this repo — one per revision served."""
         key = f"{repo_type}:{repo_id}"
-        return next((r for r in self.repos if r.key == key), None)
+        return [r for r in self.repos if r.key == key]
 
     def host(self, repo: RepoConfig) -> str:
         endpoint = repo.endpoint or self.endpoint

docker/hf-space/hf-storage-api/hf_storage_api/catalog/urllist.py

@@ -154,18 +154,9 @@ def parse_url_list(text: str, *, layout: LayoutConfig | None = None) -> Catalog:
     if not grouped:
         raise UrlListError("the URL list is empty")
 
-    # One repo at two revisions cannot be served under one URI namespace: the
-    # revision is configuration, not part of the address, so both would collide on
-    # the same base_key. The `@<rev>` escape hatch is plan §6.1 / checklist 4.4.
-    by_repo: dict[tuple[str, str, str], list[str]] = {}
-    for endpoint, repo_type, repo_id, revision in grouped:
-        by_repo.setdefault((endpoint, repo_type, repo_id), []).append(revision)
-    for (_, _, repo_id), revisions in by_repo.items():
-        if len(revisions) > 1:
-            raise UrlListError(
-                f"{repo_id} appears at {len(revisions)} revisions ({', '.join(sorted(revisions))}); "
-                f"one URI namespace can only serve one revision per repo"
-            )
+    # One repo at several revisions becomes one RepoConfig per revision. Pinned
+    # shas are served side by side, told apart by the `/resolve/<sha>/` in their
+    # addresses; `validate_catalog` refuses the mix when one of them is a moving ref.
 
     # The list names the manifest, so it also tells us the layout's marker — but
     # only adopt it when the list agrees with itself and differs from the default.

docker/hf-space/hf-storage-api/hf_storage_api/catalog_sync/publish.py

@@ -57,7 +57,7 @@ def review(
     Parsing is :func:`hf_storage_api.catalog.loader.parse_text` — the server's,
     not a copy — so this rejects exactly what the server would reject at load
     time: a truncated body, a URL pointing at a USD instead of a manifest, one
-    repo pinned at two revisions.
+    repo at two revisions when one of them is a moving ref.
     """
     if not _has_content(text):
         # Caught before parsing, purely for the error message. An emptied URL list
@@ -103,7 +103,8 @@ def publish(
     """Write ``text`` to ``target`` so that no reader can ever see it half-written."""
     digest = hashlib.sha256(text.encode("utf-8")).hexdigest()
     assets = sum(len(repo.assets) for repo in catalog.repos)
-    repo_ids = tuple(repo.repo_id for repo in catalog.repos)
+    # One entry per repo, not per revision served.
+    repo_ids = tuple(dict.fromkeys(repo.repo_id for repo in catalog.repos))
     previous = read_meta(target)
 
     if previous.get("digest") == digest and target.exists():

docker/hf-space/hf-storage-api/hf_storage_api/hf/client.py

@@ -15,6 +15,7 @@ import logging
 import re
 from collections.abc import AsyncIterator, Sequence
 from typing import Any
+from urllib.parse import quote
 
 import httpx
 
@@ -28,6 +29,18 @@ logger = logging.getLogger(__name__)
 _API_SEGMENT = {"dataset": "datasets", "model": "models", "space": "spaces"}
 _URL_SEGMENT = {"dataset": "datasets", "model": "", "space": "spaces"}
 
+
+def _quote_path(path: str) -> str:
+    """Percent-encode a repo path for a Hub URL.
+
+    The Hub decodes the URL path, so a raw `%40` in a file name resolves to `@`
+    and 404s. Every SimReady package carries one
+    (`.metadata/com.nvidia.simready.conformance.Package%401.0.0.json`), and `#`,
+    `?` or a space would break the URL outright.
+    """
+    return quote(path, safe="/")
+
+
 RETRY_STATUSES = frozenset({429, 500, 502, 503, 504})
 
 # HF does not send `Retry-After` — it sends `RateLimit` in the
@@ -106,7 +119,7 @@ class HFClient:
 
     def resolve_url(self, repo_type: str, repo_id: str, revision: str, path: str) -> str:
         segment = _URL_SEGMENT.get(repo_type, "datasets")
-        parts = [self._endpoint, segment, repo_id, "resolve", revision, path]
+        parts = [self._endpoint, segment, repo_id, "resolve", revision, _quote_path(path)]
         return "/".join([p for p in parts if p])
 
     def _headers(self, token: str | None) -> dict[str, str]:
@@ -177,7 +190,7 @@ class HFClient:
         token: str | None = None,
     ) -> list[TreeEntry]:
         """List a subtree. Follows the Hub's link-header pagination."""
-        url = self.api_url(repo_type, repo_id, "tree", revision, path)
+        url = self.api_url(repo_type, repo_id, "tree", revision, _quote_path(path))
         params: dict[str, Any] = {}
         if recursive:
             params["recursive"] = "true"

docker/hf-space/hf-storage-api/hf_storage_api/services/capabilities.py

@@ -55,14 +55,15 @@ class CapabilitiesService:
     async def ListRoutes(self, request, context):
         """Wildcard patterns for the addresses this server serves.
 
-        Unused by USD Search's Python client, but it is one of only two RPCs that
-        could feed ``omni.client``'s ``on_registered`` callback, and it exists
-        only in v1alpha — see plan §2.4. Cheap insurance against a silent Kit
-        registration failure.
+        Unused by USD Search's Python client, but ``omni.client`` routes by
+        these, and it exists only in v1alpha — see plan §2.4. ``**``, not ``*``:
+        ovrtx's bundled ``omni.client`` (2.74.0) matches ``*`` within one path
+        segment, so a ``*`` route sends no nested address to this provider and
+        every asset fails with ``ERROR_NOT_SUPPORTED`` before any RPC is made.
         """
         return capabilities_pb2.ListRoutesResponse(
             items=[
-                capabilities_pb2.Route(wildcard_pattern=f"{address}*")
+                capabilities_pb2.Route(wildcard_pattern=f"{address}**")
                 for address in self._store.top_level_addresses()
             ]
         )

docker/hf-space/hf-storage-api/hf_storage_api/services/filefolder.py

@@ -37,21 +37,22 @@ class FileFolderService:
     @handle_hf_errors_stream
     async def ListStat(self, request, context):
         uri = request.folder.uri
-        repo = self._store.repo_for_root(uri)
+        repos = self._store.repos_for_root(uri)
 
-        if repo is not None:
+        if repos:
             # Repo root: the crawl surface, flat. Returning no subfolder_addresses
             # stops the client's breadth-first walk here, which is what keeps the
             # indexed set to one entry per asset.
             semaphore = asyncio.Semaphore(self._settings.enumerate_concurrency)
 
-            async def resolve(asset_name: str):
+            async def resolve(repo, asset_name: str):
                 async with semaphore:
-                    return await self._store.folder(repo, asset_name)
+                    return repo, await self._store.folder(repo, asset_name)
 
             entries: list[filefolder_service_pb2.ListItem] = []
-            for completed in asyncio.as_completed([resolve(name) for name in repo.assets]):
-                loaded = await completed
+            pending = [resolve(repo, name) for repo in repos for name in repo.assets]
+            for completed in asyncio.as_completed(pending):
+                repo, loaded = await completed
                 if loaded is None:
                     continue
                 entry = loaded.entry(loaded.root_usd)

docker/hf-space/hf-storage-api/hf_storage_api/services/fileobject.py

@@ -1,7 +1,7 @@
 # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
 
-"""``FileObjectService`` — ``Stat``, ``Enumerate``, ``ReadFromAddress``.
+"""``FileObjectService`` — ``Stat``, ``Enumerate``, ``ReadFromAddress``, ``Read``.
 
 Two surfaces (plan §6.2):
 
@@ -94,25 +94,27 @@ class FileObjectService:
         to come out of this one call (plan §2.4).
         """
         address = request.resource_address
-        repo = self._store.repo_for_root(address)
-        if repo is None:
+        repos = self._store.repos_for_root(address)
+        if not repos:
             # Scoped to something other than a served root. Prefer NOT_FOUND:
             # any other status aborts the crawl (contract doc §3.4).
             await context.abort(grpc.StatusCode.NOT_FOUND, address)
 
         semaphore = asyncio.Semaphore(self._settings.enumerate_concurrency)
 
-        async def resolve(asset_name: str):
+        async def resolve(repo, asset_name: str):
             async with semaphore:
-                return asset_name, await self._store.folder(repo, asset_name)
+                return repo, asset_name, await self._store.folder(repo, asset_name)
 
         batch: list[fileobject_pb2.AddressInfo] = []
         # Stream as results arrive rather than resolving everything first: a
         # crawl that emits nothing for minutes is indistinguishable from a hang.
-        tasks = [asyncio.create_task(resolve(name)) for name in repo.assets]
+        # Every revision served under this root: each asset's address carries its
+        # own `/resolve/<sha>/`, so they cannot collide.
+        tasks = [asyncio.create_task(resolve(repo, name)) for repo in repos for name in repo.assets]
         try:
             for completed in asyncio.as_completed(tasks):
-                asset_name, loaded = await completed
+                repo, asset_name, loaded = await completed
                 if loaded is None:
                     logger.warning("catalogued asset %s is absent upstream; skipping", asset_name)
                     continue
@@ -156,7 +158,30 @@ class FileObjectService:
         yield fileobject_service_pb2.ReadFromAddressResponse(
             resource_info=_resource_info(self._store, entry, address)
         )
+        async for chunk in self._chunks(resolved):
+            yield fileobject_service_pb2.ReadFromAddressResponse(chunk=fileobject_pb2.Chunk(chunk=chunk))
 
+    @handle_hf_errors_stream
+    async def Read(self, request, context):
+        """Read by ``ResourceIdentity``, as ovrtx's bundled ``omni.client``
+        (2.74.0) does after the ``Stat`` that minted the identity; Kit and USD
+        Search's client use ``ReadFromAddress`` instead.
+
+        The identity is the content oid, so it is mapped back to the address it
+        was minted for (contract doc §5.1) and re-authorised like any address.
+        """
+        identity = request.resource_identity.encoded_identity
+        address = self._store.address_for_identity(identity)
+        found = await self._store.stat(address) if address else None
+        if found is None:
+            await context.abort(grpc.StatusCode.NOT_FOUND, identity)
+        resolved, entry = found
+
+        yield fileobject_service_pb2.ReadResponse(metadata=_metadata(entry))
+        async for chunk in self._chunks(resolved):
+            yield fileobject_service_pb2.ReadResponse(chunk=fileobject_pb2.Chunk(chunk=chunk))
+
+    async def _chunks(self, resolved):
         revision = await self._store.revision(resolved.repo)
         stream = self._store.hf.stream(
             resolved.repo.repo_type,
@@ -168,7 +193,7 @@ class FileObjectService:
         )
         try:
             async for chunk in stream:
-                yield fileobject_service_pb2.ReadFromAddressResponse(chunk=fileobject_pb2.Chunk(chunk=chunk))
+                yield chunk
         except asyncio.CancelledError:
             # The client hung up: close the upstream response instead of draining
             # it into a dead RPC. With multi-hundred-MB assets that is a real
@@ -176,13 +201,6 @@ class FileObjectService:
             await stream.aclose()
             raise
 
-    async def Read(self, request, context):
-        # The client downloads by address, not by ResourceIdentity, so this is
-        # unused. Implement only if the Phase-1.4 Kit log shows it (plan §2.4).
-        await context.abort(
-            grpc.StatusCode.UNIMPLEMENTED, "Read by identity is not supported; use ReadFromAddress"
-        )
-
     # -- write path: read-only service (plan §6.8) ----------------------------
 
     async def FetchWriteTypeInfo(self, request, context):

docker/hf-space/hf-storage-api/hf_storage_api/services/metadata_service.py

@@ -6,6 +6,7 @@
 from __future__ import annotations
 
 import logging
+import posixpath
 
 import grpc
 from nvidia.omniverse.storage.metadata.v1alpha import metadata_pb2
@@ -39,9 +40,14 @@ class MetadataService:
         # Under `catalog_kind="wrapp"` the per-asset filenames use the short stem
         # from the root USD (e.g. `sm_IV_pole_a01_01`), not the full folder path
         # that `asset_name` is under a versioned layout (plan §6.3).
-        stem = loaded.root_usd.rsplit("/", 1)[-1].removesuffix(".usd")
+        stem = posixpath.splitext(loaded.root_usd.rsplit("/", 1)[-1])[0]
         manifest_path = f"{asset_name}/{layout.render(layout.manifest, asset_name, stem=stem)}"
         metadata_path = f"{asset_name}/{layout.render(layout.metadata, asset_name, stem=stem)}"
+        if layout.catalog_kind == "manifest" and metadata_path not in loaded.files:
+            # Same drift as the root USD's (store.py:_data_dir_usd): the
+            # descriptive metadata sits next to the root USD as `<stem>.json`,
+            # not at the `{data_dir}/{asset_name}.json` the convention names.
+            metadata_path = f"{posixpath.splitext(loaded.root_usd)[0]}.json"
 
         manifest = await self._sidecar(resolved, manifest_path, loaded)
         asset_metadata = await self._sidecar(resolved, metadata_path, loaded)

docker/hf-space/hf-storage-api/hf_storage_api/settings.py

@@ -41,6 +41,11 @@ class Settings(BaseSettings):
     catalog_path: Path | None = None
     catalog_url: str | None = None
     catalog_refresh_seconds: float = 900.0
+    # More catalog files, merged with the one above — e.g. a URL list the
+    # catalog-sync sidecar writes, next to a YAML catalog from a ConfigMap. Each is
+    # polled and guarded on its own. Comma-separated rather than `list[Path]`:
+    # pydantic-settings decodes complex types from env as JSON.
+    catalog_extra_paths: str = ""
 
     # A URL list has nowhere to put a layout, so a non-default one comes from the
     # environment instead. `manifest` is inferred from the list itself.
@@ -104,6 +109,10 @@ class Settings(BaseSettings):
             raise ValueError("chunk_size must stay well below the client's 4 MiB default limit")
         return self
 
+    @property
+    def extra_catalog_paths(self) -> list[Path]:
+        return [Path(item.strip()) for item in self.catalog_extra_paths.split(",") if item.strip()]
+
     def layout_defaults(self):
         """Layout for formats that cannot carry one (plan §6.3)."""
         from .catalog.model import LayoutConfig

docker/hf-space/hf-storage-api/hf_storage_api/store.py

@@ -24,6 +24,7 @@ from dataclasses import dataclass
 from . import addressing
 from .cache import MISSING, Missing, SingleFlightCache
 from .catalog.model import Catalog, LayoutConfig, RepoConfig
+from .catalog.model import looks_like_sha as _looks_like_sha
 from .hf.client import HFClient, HFNotFound
 from .hf.models import TreeEntry
 from .settings import Settings
@@ -135,7 +136,9 @@ class AssetStore:
         return None
 
     def top_level_addresses(self) -> list[str]:
-        return [
+        """One root per repo, however many revisions of it are served: the root
+        carries no revision, and its listing spans all of them."""
+        addresses = (
             addressing.build(
                 host=self._catalog.host(repo),
                 repo_type=repo.repo_type,
@@ -144,14 +147,22 @@ class AssetStore:
                 scheme=self._settings.uri_scheme,
             )
             for repo in self._catalog.repos
-        ]
+        )
+        return list(dict.fromkeys(addresses))
+
+    def repos_for_root(self, address: str) -> list[RepoConfig]:
+        """Every revision served under this repo root; empty if it is not one.
 
-    def repo_for_root(self, address: str) -> RepoConfig | None:
+        An address *inside* the repo is not its root: ``ListStat`` and
+        ``Enumerate`` answer a root with the flat crawl surface, so matching a
+        nested folder here would hand ovrtx's client the crawl surface for an
+        asset's own folder — it stats a file by listing its parent, finds the
+        file missing, and fails every load.
+        """
         parsed = self._parse(address)
-        if parsed is None:
-            return None
-        repo = self._catalog.repo(parsed.repo_type, parsed.repo_id)
-        return repo if repo and self._host_matches(parsed, repo) else None
+        if parsed is None or parsed.path != "":
+            return []
+        return self._repos_named(parsed)
 
     def resolve(self, address: str) -> Resolved | None:
         """Map an address to its catalogued asset folder, using the catalog only.
@@ -164,26 +175,31 @@ class AssetStore:
         parsed = self._parse(address)
         if parsed is None:
             return None
-        repo = self._catalog.repo(parsed.repo_type, parsed.repo_id)
-        if repo is None or not self._host_matches(parsed, repo):
-            return None
-        layout = self._catalog.layout_for(repo)
-        if parsed.revision and _looks_like_sha(repo.revision) and parsed.revision != repo.revision:
-            # Pinned repo: a wrong revision is answerable from the catalog alone,
-            # so it costs no upstream request.
-            return None
-
-        for asset_name in repo.assets:
-            if addressing.is_within(parsed.path, asset_name):
-                return Resolved(
-                    repo=repo,
-                    layout=layout,
-                    asset_name=asset_name,
-                    path=parsed.path,
-                    is_folder=parsed.is_folder,
-                    address_revision=parsed.revision,
+        matches: list[Resolved] = []
+        for repo in self._repos_named(parsed):
+            if parsed.revision and _looks_like_sha(repo.revision) and parsed.revision != repo.revision:
+                # Pinned repo: a wrong revision is answerable from the catalog
+                # alone, so it costs no upstream request. With several revisions
+                # served, this is also what picks the right one.
+                continue
+            asset_name = next((a for a in repo.assets if addressing.is_within(parsed.path, a)), None)
+            if asset_name is not None:
+                matches.append(
+                    Resolved(
+                        repo=repo,
+                        layout=self._catalog.layout_for(repo),
+                        asset_name=asset_name,
+                        path=parsed.path,
+                        is_folder=parsed.is_folder,
+                        address_revision=parsed.revision,
+                    )
                 )
-        return None
+        if len(matches) > 1:
+            # A revision-less address for a folder served at several revisions
+            # names no one of them; guessing would serve whichever came first.
+            logger.debug("%s matches %d revisions; it must name one", address, len(matches))
+            return None
+        return matches[0] if matches else None
 
     def is_repo_root(self, address: str) -> bool:
         """Is this the root of a served repo, with or without a trailing slash?"""
@@ -212,6 +228,14 @@ class AssetStore:
             revision=self._emit_revision(repo, revision),
         )
 
+    def _repos_named(self, parsed) -> list[RepoConfig]:
+        """Every revision served for the repo an address names, at any depth."""
+        return [
+            repo
+            for repo in self._catalog.repos_for(parsed.repo_type, parsed.repo_id)
+            if self._host_matches(parsed, repo)
+        ]
+
     def _host_matches(self, parsed, repo: RepoConfig) -> bool:
         """The address must name the host this repo is configured against.
 
@@ -364,7 +388,11 @@ class AssetStore:
 
         ``catalog_kind="manifest"``: ``root_usds.json`` where it exists, else the
         naming convention (plan §3.3 — the index file was absent on 2 of 4
-        sampled assets)."""
+        sampled assets). When the convention's file is not in the folder either,
+        the USD sitting directly in ``data_dir`` — verified 2026-10-08: both
+        nvidia/simready-assets@2464ca32 and nvidia/srp-staging-controlled ship
+        ``simready_usd/sm_<asset>_01.usd`` with no index on most assets, and the
+        convention's ``simready_usd/<asset>.usd`` on none."""
         if layout.catalog_kind == "wrapp":
             manifest_path = f"{asset_name}/{layout.output_manifest}"
             payload = (
@@ -428,7 +456,10 @@ class AssetStore:
                 # entries[] are relative to the asset folder.
                 return f"{asset_name}/{str(entries[0]).lstrip('/')}"
             logger.warning("%s has an empty entries[]; falling back to the naming convention", index_path)
-        return f"{asset_name}/{layout.render(layout.content_fallback, asset_name)}"
+        convention = f"{asset_name}/{layout.render(layout.content_fallback, asset_name)}"
+        if convention in files:
+            return convention
+        return _data_dir_usd(asset_name, layout, files) or convention
 
     async def _sidecar_json(self, repo: RepoConfig, revision: str, path: str) -> dict | None:
         """Fetch and parse a small JSON sidecar. Immutable within a revision."""
@@ -528,10 +559,6 @@ class AssetStore:
         }
 
 
-def _looks_like_sha(revision: str) -> bool:
-    return len(revision) == 40 and all(c in "0123456789abcdef" for c in revision.lower())
-
-
 def _top_level_usd(asset_name: str, files: Mapping[str, TreeEntry]) -> str | None:
     """The one USD file (``.usd``/``.usda``/``.usdc``/``.usdz``) sitting directly
     at the asset folder's root, if there is exactly one candidate set to choose
@@ -547,6 +574,30 @@ def _top_level_usd(asset_name: str, files: Mapping[str, TreeEntry]) -> str | Non
     return top_level_usds[0] if top_level_usds else None
 
 
+def _data_dir_usd(asset_name: str, layout: LayoutConfig, files: Mapping[str, TreeEntry]) -> str | None:
+    """The USD file directly inside the asset's ``data_dir``, for ``manifest``
+    assets that carry neither a root-USD index nor the conventional
+    ``<asset>.usd``.
+
+    Several candidates happen — a variant, a payload — so the SimReady naming
+    ``sm_<asset>_NN.*`` wins over the others, then alphabetical order, which puts
+    ``_01`` first.
+    """
+    data_dir = layout.render("{data_dir}", asset_name).strip("/")
+    folder = f"{asset_name}/{data_dir}" if data_dir else asset_name
+    candidates = sorted(
+        path
+        for path in files
+        if path.startswith(f"{folder}/")
+        and "/" not in path[len(folder) + 1 :]
+        and path.endswith((".usd", ".usda", ".usdc", ".usdz"))
+    )
+    if not candidates:
+        return None
+    preferred = f"{folder}/sm_{asset_name.rsplit('/', 1)[-1]}_"
+    return next((path for path in candidates if path.startswith(preferred)), candidates[0])
+
+
 def _wrapp_folder_basename(asset_name: str) -> str:
     """The ``.wrapp`` file's own name embeds the asset folder's basename, not
     the full ``asset_name`` — which under ``catalog_kind: wrapp`` is

docker/hf-space/nginx.conf.template

@@ -73,11 +73,10 @@ server {
         proxy_pass http://127.0.0.1:8000;
     }
 
-    # SimReady Central's validator (supervisord's deeptag-validate, 127.0.0.1:8020) answers here in
-    # place of deepsearch-api's: the same judge plus each hit's path and caption and a verdict cache.
     location /vlm_validate/search_result {
         if ($request_method = OPTIONS) { add_header Content-Length 0 always; add_header Content-Type "text/plain" always; return 204; }
-        proxy_pass http://127.0.0.1:8020;
+        rewrite ^/vlm_validate/search_result(.*)$ /v3/deepsearch/vlm_validate/search_result$1 break;
+        proxy_pass http://127.0.0.1:8000;
     }
 
     location /llm_parse/ {
@@ -86,11 +85,14 @@ server {
         proxy_pass http://127.0.0.1:8000;
     }
 
+    # package-download (127.0.0.1:8020, pipeline/package_download/), not deepsearch-api: it answers
+    # /download/asset with the asset's whole SimReady package folder, same response contract, and
+    # forwards scope=dependencies (and anything outside a catalogued package) to deepsearch-api's
+    # own /v3/download/asset. It serves the /download/... paths itself, so no rewrite.
     location /download {
         if ($request_method = OPTIONS) { add_header Content-Length 0 always; add_header Content-Type "text/plain" always; return 204; }
-        rewrite ^/download(.*)$ /v3/download$1 break;
-        proxy_pass http://127.0.0.1:8000;
-        # Bundling an asset + all its dependencies can take longer than a single thumbnail.
+        proxy_pass http://127.0.0.1:8020;
+        # Fetching a whole package (or a forwarded dependency bundle) can take longer than a single thumbnail.
         proxy_read_timeout 300s;
     }
 
@@ -263,12 +265,13 @@ server {
         try_files /category_hierarchy.json @slim_not_found;
     }
     # One request per result page (QA 6881558: Hugging Face limits each user to ~2,000 requests a minute,
-    # and a page of per-image checks took 50): SimReady Central's validator (supervisord deeptag-validate,
-    # 127.0.0.1:8020) streams one NDJSON verdict per hit, fetching each thumbnail through /images above
-    # (SHIM_GATEWAY_BASE_URL). Unbuffered so verdicts land as they come; heartbeats keep the stream open.
+    # and a page of per-image checks took 50): deepsearch-api's /vlm_validate/search_result_batch streams one
+    # NDJSON verdict per hit. The Explorer calls this path; the request and the lines are the same.
+    # Unbuffered so verdicts land as they come; heartbeats keep the stream open.
     location = /vlm_validate/batch_stream {
         if ($request_method = OPTIONS) { add_header Content-Length 0 always; add_header Content-Type "text/plain" always; return 204; }
-        proxy_pass http://127.0.0.1:8020;
+        rewrite ^ /v3/deepsearch/vlm_validate/search_result_batch break;
+        proxy_pass http://127.0.0.1:8000;
         proxy_buffering off;
         proxy_read_timeout 120s;
         gzip off;

docker/hf-space/siglip2-model/SOURCE.txt

@@ -1,4 +1,4 @@
 # Generated by scripts/extract_siglip2_model.sh — do not hand-edit.
-source_repo=/tmp/usdsearch-rc7
-source_commit=b045c1eb8d86f997c51ec454ba0506247af9c511
-extracted_at=2026-10-02T14:40:11Z
+source_repo=/tmp/claude-1000/-home-arozantsev-Documents-projects-DeepSearch-simready-central-indexing-internal/4b40209a-9038-43e7-b70a-ed77e9f3ffdf/scratchpad/usdsearch-d94e39e4
+source_commit=d94e39e458b982ec1854ceea6a6e948f995944f5
+extracted_at=2026-10-08T10:11:36Z

docker/hf-space/siglip2-model/pyproject.toml

@@ -11,4 +11,5 @@ dependencies = [
     "wheel>=0.46.3",
     "urllib3>=2.8.0",
     "starlette>=1.3.1",
+    "fsspec>=2026.9.0",
 ]

docker/hf-space/siglip2-model/uv.lock

@@ -65,11 +65,11 @@ wheels = [
 
 [[package]]
 name = "fsspec"
-version = "2026.4.0"
+version = "2026.9.0"
 source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/d5/8d/1c51c094345df128ca4a990d633fe1a0ff28726c9e6b3c41ba65087bba1d/fsspec-2026.4.0.tar.gz", hash = "sha256:301d8ac70ae90ef3ad05dcf94d6c3754a097f9b5fe4667d2787aa359ec7df7e4", size = 312760, upload-time = "2026-04-29T20:42:38.635Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/77/cd/9be253869fc42e764de7f3dedd6969af7d44ff9c3375214a3442a6f3fc08/fsspec-2026.9.0.tar.gz", hash = "sha256:0f08147951c8cb31d844c3547d631053b127863b60be04cf06e121333ee0e2fe", size = 333545, upload-time = "2026-09-18T17:50:42.825Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/d5/0c/043d5e551459da400957a1395e0febbf771446ff34291afcbe3d8be2a279/fsspec-2026.4.0-py3-none-any.whl", hash = "sha256:11ef7bb35dab8a394fde6e608221d5cf3e8499401c249bebaeaad760a1a8dec2", size = 203402, upload-time = "2026-04-29T20:42:36.842Z" },
+    { url = "https://files.pythonhosted.org/packages/6c/c0/a98505f18594f1bce828bb159cec0fcf9860562f1a2c85913409fc8f3d9e/fsspec-2026.9.0-py3-none-any.whl", hash = "sha256:8dd6e646e99ea382bd85f97a45e6b526a442d79423a7dc673f1e2756d05fcb5f", size = 221738, upload-time = "2026-09-18T17:50:41.341Z" },
 ]
 
 [[package]]
@@ -462,6 +462,7 @@ name = "siglip2-triton-server"
 version = "0.1.0"
 source = { virtual = "." }
 dependencies = [
+    { name = "fsspec" },
     { name = "numpy" },
     { name = "pillow" },
     { name = "starlette" },
@@ -473,6 +474,7 @@ dependencies = [
 
 [package.metadata]
 requires-dist = [
+    { name = "fsspec", specifier = ">=2026.9.0" },
     { name = "numpy" },
     { name = "pillow", specifier = ">=12.3.0" },
     { name = "starlette", specifier = ">=1.3.1" },

docker/hf-space/supervisord-gpu.conf

@@ -110,7 +110,22 @@ command=/bin/sh -c '\
         exec sleep infinity; \
     fi; \
     exec /app/.venv/bin/python -m hf_storage_api'
-environment=PYTHONPATH="/app/hf-storage-api/src",HF_ADAPTER_GRPC_HOST="127.0.0.1",HF_ADAPTER_GRPC_PORT="50051",HF_ADAPTER_CATALOG_SOURCE="file",HF_ADAPTER_CATALOG_PATH="/app/hf-space-pipeline/.build/restore/hf-storage-api-catalog.yaml",HF_ADAPTER_CATALOG_FORMAT="yaml",HF_ADAPTER_HF_ENDPOINT="https://huggingface.co",HF_ADAPTER_URI_SCHEME="hf",HF_ADAPTER_ALLOW_ANONYMOUS="true",HF_ADAPTER_LOG_LEVEL="INFO",HF_ADAPTER_HF_TOKEN="%(ENV_HF_STORAGE_API_TOKEN)s"
+environment=PYTHONPATH="/app/hf-storage-api/src",HF_ADAPTER_GRPC_HOST="127.0.0.1",HF_ADAPTER_GRPC_PORT="50051",HF_ADAPTER_CATALOG_SOURCE="file",HF_ADAPTER_CATALOG_PATH="/app/hf-space-pipeline/.build/restore/hf-storage-api-catalog.yaml",HF_ADAPTER_CATALOG_FORMAT="yaml",HF_ADAPTER_HF_ENDPOINT="https://huggingface.co",HF_ADAPTER_URI_SCHEME="hf",HF_ADAPTER_ALLOW_ANONYMOUS="true",HF_ADAPTER_LOG_LEVEL="INFO",HF_ADAPTER_ADVERTISE_FILEFOLDER="true",HF_ADAPTER_HF_TOKEN="%(ENV_HF_STORAGE_API_TOKEN)s"
+directory=/app
+user=1000
+autorestart=true
+startsecs=5
+stdout_logfile=/dev/stdout
+stdout_logfile_maxbytes=0
+stderr_logfile=/dev/stderr
+stderr_logfile_maxbytes=0
+
+[program:package-download]
+; See docker/hf-space/supervisord.conf's own copy of this program for the full explanation (identical here
+; apart from `env -u PYTHONPATH`, like this file's other venv programs).
+command=/usr/bin/env -u PYTHONPATH /bin/sh -c '\
+    until [ -f /app/hf-space-pipeline/.build/restore/.download-complete ]; do sleep 2; done; \
+    cd /app/hf-space-pipeline && exec /app/.venv/bin/python -m pipeline.package_download'
 directory=/app
 user=1000
 autorestart=true
@@ -155,25 +170,6 @@ stdout_logfile_maxbytes=0
 stderr_logfile=/dev/stderr
 stderr_logfile_maxbytes=0
 
-[program:deeptag-validate]
-; SimReady Central's validator (docker/hf-space/deeptag-validate/app.py, vendored from
-; simready-central deeptag-validate/, !582) answers /vlm_validate/search_result in place of
-; deepsearch-api's: the same Yes/No log-prob judge on the same engine, plus each hit's path
-; and caption in words-only checks (JUDGE_ASSET_CONTEXT) and an in-memory verdict cache
-; (VALIDATE_CACHE_SIZE, ~120 MB at 500k entries). /vlm_validate/batch_stream (one request per page)
-; fetches each thumbnail through nginx's /images (SHIM_GATEWAY_BASE_URL). The engine settings are deepsearch-api's
-; USDSEARCH_* variables, read when the shell execs (an unset one never stops supervisord).
-command=/usr/bin/env -u PYTHONPATH /bin/sh -c 'RERANKER_BASE_URL="$USDSEARCH_LLM_BASE_URL" RERANKER_MODEL="$USDSEARCH_VISION_VALIDATION_MODEL" RERANKER_NAME="$USDSEARCH_VISION_VALIDATION_MODEL" RERANKER_API_KEY="$USDSEARCH_LLM_API_KEY" exec /app/.venv/bin/python -m uvicorn app:app --host 127.0.0.1 --port 8020 --log-level warning'
-environment=JUDGE_ASSET_CONTEXT="path+caption",VALIDATE_CACHE_SIZE="500000",VALIDATE_MAX_EDGE="448",RERANKER_MAX_CONCURRENCY="32",RERANKER_BATCH_CONCURRENCY="16",RERANKER_COVER_MIN="0.2",SHIM_GATEWAY_BASE_URL="http://127.0.0.1:%(ENV_PORT)s"
-directory=/app/deeptag-validate
-user=1000
-autorestart=true
-startsecs=3
-stdout_logfile=/dev/stdout
-stdout_logfile_maxbytes=0
-stderr_logfile=/dev/stderr
-stderr_logfile_maxbytes=0
-
 [program:nginx]
 command=/bin/sh -c "until curl -sf http://localhost:8000/health >/dev/null 2>&1 && curl -sf http://localhost:8010/docs >/dev/null 2>&1; do sleep 2; done; dl=false; case '%(ENV_DOWNLOAD_ENABLED)s' in [Tt]rue|1) if [ -f /app/hf-space-pipeline/.build/restore/hf-storage-api-catalog.yaml ]; then dl=true; else echo 'nginx: no hf-storage-api catalog, rendering download:false' >&2; fi;; esac; sed -e 's/__PORT__/%(ENV_PORT)s/' -e 's|__CORPUS_NAME__|%(ENV_CORPUS_NAME)s|g' -e 's/__DOWNLOAD_ENABLED__/'$dl'/' -e 's|__SUBMIT_CONTENT_URL__|%(ENV_SUBMIT_CONTENT_URL)s|g' /etc/nginx/hf-space.conf.template > /etc/nginx/conf.d/hf-space.conf && exec nginx -g 'daemon off;'"
 user=root

docker/hf-space/supervisord.conf

@@ -79,10 +79,15 @@ stderr_logfile_maxbytes=0
 ; Read-only Omniverse Storage API gRPC server backed by a HuggingFace
 ; dataset (docker/hf-space/hf-storage-api/, vendored+regenerated by
 ; scripts/extract_hf_storage_api.sh — see that script and
-; docs/hf-space-deployment.md for why). Backs deepsearch-api's
-; existing GET /v3/download/asset route (search_utils.storage_client's
-; STORAGE_API_GRPC_ENDPOINT, baked in Dockerfile.hf-space) for a
-; storage_api-backend corpus. Bound to loopback only, like every other
+; docs/hf-space-deployment.md for why). Backs package-download's
+; whole-package downloads (below; FileFolder ListStat + ReadFromAddress —
+; hence HF_ADAPTER_ADVERTISE_FILEFOLDER, off by default upstream) and
+; deepsearch-api's own GET /v3/download/asset route (search_utils.
+; storage_client's STORAGE_API_GRPC_ENDPOINT, baked in Dockerfile.hf-space;
+; it ignores FileFolder, STORAGE_API_IGNORE_FILEFOLDER_API=true) for a
+; storage_api-backend corpus. Needs hf-storage-api >= 417d9af for that
+; (sub-folder ListStat and percent-encoded Hub paths, hf-storage-api-service#1
+; — see docs/development.md). Bound to loopback only, like every other
 ; backend process in this container — nginx is the sole externally-proxied
 ; port, and gRPC never needs to cross it (same shape as the GPU variant's
 ; own local siglip2-triton on :8001).
@@ -108,7 +113,31 @@ command=/bin/sh -c '\
         exec sleep infinity; \
     fi; \
     exec /app/.venv/bin/python -m hf_storage_api'
-environment=PYTHONPATH="/app/hf-storage-api/src",HF_ADAPTER_GRPC_HOST="127.0.0.1",HF_ADAPTER_GRPC_PORT="50051",HF_ADAPTER_CATALOG_SOURCE="file",HF_ADAPTER_CATALOG_PATH="/app/hf-space-pipeline/.build/restore/hf-storage-api-catalog.yaml",HF_ADAPTER_CATALOG_FORMAT="yaml",HF_ADAPTER_HF_ENDPOINT="https://huggingface.co",HF_ADAPTER_URI_SCHEME="hf",HF_ADAPTER_ALLOW_ANONYMOUS="true",HF_ADAPTER_LOG_LEVEL="INFO",HF_ADAPTER_HF_TOKEN="%(ENV_HF_STORAGE_API_TOKEN)s"
+environment=PYTHONPATH="/app/hf-storage-api/src",HF_ADAPTER_GRPC_HOST="127.0.0.1",HF_ADAPTER_GRPC_PORT="50051",HF_ADAPTER_CATALOG_SOURCE="file",HF_ADAPTER_CATALOG_PATH="/app/hf-space-pipeline/.build/restore/hf-storage-api-catalog.yaml",HF_ADAPTER_CATALOG_FORMAT="yaml",HF_ADAPTER_HF_ENDPOINT="https://huggingface.co",HF_ADAPTER_URI_SCHEME="hf",HF_ADAPTER_ALLOW_ANONYMOUS="true",HF_ADAPTER_LOG_LEVEL="INFO",HF_ADAPTER_ADVERTISE_FILEFOLDER="true",HF_ADAPTER_HF_TOKEN="%(ENV_HF_STORAGE_API_TOKEN)s"
+directory=/app
+user=1000
+autorestart=true
+startsecs=5
+stdout_logfile=/dev/stdout
+stdout_logfile_maxbytes=0
+stderr_logfile=/dev/stderr
+stderr_logfile_maxbytes=0
+
+[program:package-download]
+; Whole-package downloads (pipeline/package_download/, docs/plans/hf-space-package-download.md). nginx's
+; /download location proxies here instead of deepsearch-api: GET /download/asset returns the asset's whole
+; SimReady package folder (package marker, .metadata/, sidecar JSONs, thumbnails — everything on HF), read
+; through hf-storage-api above (its FileFolder service is on for this: HF_ADAPTER_ADVERTISE_FILEFOLDER), with
+; usdsearch's response contract. scope=dependencies, and any URL outside a catalogued package, is forwarded
+; unchanged to deepsearch-api's own /v3/download/asset. Loopback only. Holds no HF token.
+;
+; Waits for the seed's restore marker (.download-complete: the whole artifact set, catalog included, is on
+; disk — the catalog file alone can be seen half-written), then loads the same catalog hf-storage-api serves
+; from. Without one (an s3-backend corpus, or a run that has none) it forwards every request, so /download
+; behaves as it did before this program existed.
+command=/bin/sh -c '\
+    until [ -f /app/hf-space-pipeline/.build/restore/.download-complete ]; do sleep 2; done; \
+    cd /app/hf-space-pipeline && exec /app/.venv/bin/python -m pipeline.package_download'
 directory=/app
 user=1000
 autorestart=true
@@ -159,25 +188,6 @@ stdout_logfile_maxbytes=0
 stderr_logfile=/dev/stderr
 stderr_logfile_maxbytes=0
 
-[program:deeptag-validate]
-; SimReady Central's validator (docker/hf-space/deeptag-validate/app.py, vendored from
-; simready-central deeptag-validate/, !582) answers /vlm_validate/search_result in place of
-; deepsearch-api's: the same Yes/No log-prob judge on the same engine, plus each hit's path
-; and caption in words-only checks (JUDGE_ASSET_CONTEXT) and an in-memory verdict cache
-; (VALIDATE_CACHE_SIZE, ~120 MB at 500k entries). /vlm_validate/batch_stream (one request per page)
-; fetches each thumbnail through nginx's /images (SHIM_GATEWAY_BASE_URL). The engine settings are deepsearch-api's
-; USDSEARCH_* variables, read when the shell execs (an unset one never stops supervisord).
-command=/usr/bin/env -u PYTHONPATH /bin/sh -c 'RERANKER_BASE_URL="$USDSEARCH_LLM_BASE_URL" RERANKER_MODEL="$USDSEARCH_VISION_VALIDATION_MODEL" RERANKER_NAME="$USDSEARCH_VISION_VALIDATION_MODEL" RERANKER_API_KEY="$USDSEARCH_LLM_API_KEY" exec /app/.venv/bin/python -m uvicorn app:app --host 127.0.0.1 --port 8020 --log-level warning'
-environment=JUDGE_ASSET_CONTEXT="path+caption",VALIDATE_CACHE_SIZE="500000",VALIDATE_MAX_EDGE="448",RERANKER_MAX_CONCURRENCY="32",RERANKER_BATCH_CONCURRENCY="16",RERANKER_COVER_MIN="0.2",SHIM_GATEWAY_BASE_URL="http://127.0.0.1:%(ENV_PORT)s"
-directory=/app/deeptag-validate
-user=1000
-autorestart=true
-startsecs=3
-stdout_logfile=/dev/stdout
-stdout_logfile_maxbytes=0
-stderr_logfile=/dev/stderr
-stderr_logfile_maxbytes=0
-
 [program:nginx]
 ; nginx is the *only* process bound to the externally-proxied port, and
 ; Hugging Face's own Space "ready" detection is just "does something answer

docker/hf-space/wheels/SOURCE.txt

@@ -1,3 +1,3 @@
 # Generated by scripts/extract_usdsearch_wheels.sh — do not hand-edit.
-source_image=nvcr.io/m3sujtetvf5w/usdsearch/usdsearch:1.5.0-rc.7
-extracted_at=2026-10-02T14:39:50Z
+source_image=gitlab-master.nvidia.com:5005/omniverse/deeptag/usdsearch/usdsearch:fd61a332
+extracted_at=2026-10-09T14:24:46Z

docker/hf-space/wheels/constraints.txt

@@ -37,7 +37,7 @@ fastapi-swagger==0.4.48
 filelock==3.29.0
 fire==0.7.1
 frozenlist==1.8.0
-fsspec==2026.3.0
+fsspec==2026.9.0
 googleapis-common-protos==1.74.0
 greenlet==3.4.0
 grpcio==1.80.0
@@ -67,7 +67,7 @@ langchain-text-splitters==1.1.2
 langgraph==1.2.11
 langgraph-checkpoint==4.1.1
 langgraph-prebuilt==1.1.0
-langgraph-sdk==0.4.2
+langgraph-sdk==0.4.6
 langsmith==0.8.18
 markdown-it-py==4.0.0
 MarkupSafe==3.0.3

docker/hf-space/wheels/deepsearch_api-0.1.0-py3-none-any.whl

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:57dcbdda688fd56743accba5eca5c5483677c8ce4408c765d31ea6991b795cae
-size 214351
+oid sha256:380e2f6e9f261b3aedc41c6f3647017002cf68209fed3577eef9e3157209a272
+size 221113

docker/hf-space/wheels/search_utils-0.1.0-py3-none-any.whl

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:123f44763c3c3e1561c1e3fec9dd4fac24305befa0a310c218725259a5af2e00
-size 286723
+oid sha256:35f1cc7cf94f7bf845036cd7332ce2888944eb1c1309532b8863f9e7097c3092
+size 286722

docker/hf-space/wheels/siglip2_triton_client-0.6.1-py3-none-any.whl

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:437965e33a16a83dd5540a5aad40ddbb3008226197cb92df6702095c55ed8ea6
-size 5482684
+oid sha256:daba44a4ac31b836f233e1e7945ab5aaa8b8a2ebcf2702375166f87248f49fc0
+size 5482694

licenses/fsspec-2026.4.0.txt

@@ -1,29 +0,0 @@
-BSD 3-Clause License
-
-Copyright (c) 2018, Martin Durant
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are met:
-
-* Redistributions of source code must retain the above copyright notice, this
-  list of conditions and the following disclaimer.
-
-* Redistributions in binary form must reproduce the above copyright notice,
-  this list of conditions and the following disclaimer in the documentation
-  and/or other materials provided with the distribution.
-
-* Neither the name of the copyright holder nor the names of its
-  contributors may be used to endorse or promote products derived from
-  this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
-FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
-DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
-SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
-CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
-OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

pipeline/bake.py

@@ -408,20 +408,31 @@ def run(cfg: Config, push: bool = False) -> int:
 
 def download_smoke(base_url: str, timeout_s: float = 180.0) -> bool:
     """A storage_api corpus's download route answers for a real asset: search
-    one hit, then ask /download/asset for its manifest (sizes only, no bytes).
-    Polls, because hf-storage-api starts only once the seed restored its
-    catalog."""
+    one hit, then ask /download/asset for its manifest (sizes only, no bytes) —
+    once as the default whole-package download (package-download, through
+    hf-storage-api's folder listing: scope "package", the asset itself among
+    the files) and once as scope=dependencies (forwarded to deepsearch-api's
+    own asset + dependencies bundle). Polls, because hf-storage-api and
+    package-download start only once the seed restored the catalog."""
     import time
     import httpx
+
+    def preview(asset_url: str, **params):
+        resp = httpx.get(f"{base_url}/download/asset",
+                         params={"asset_url": asset_url, "manifest_only": "true", **params}, timeout=60.0)
+        return resp.json() if resp.status_code == 200 else None
+
     deadline = time.monotonic() + timeout_s
     while time.monotonic() < deadline:
         try:
             hits = httpx.get(f"{base_url}/search", params={"query": "box", "limit": 1}, timeout=15.0).json()
             asset_url = hits[0]["url"] if hits else None
             if asset_url:
-                resp = httpx.get(f"{base_url}/download/asset", params={"asset_url": asset_url, "manifest_only": "true"},
-                                 timeout=60.0)
-                if resp.status_code == 200 and (resp.json().get("file_count") or 0) > 0:
+                package = preview(asset_url)
+                deps = preview(asset_url, scope="dependencies")
+                if (package and package.get("scope") == "package" and (package.get("file_count") or 0) > 0
+                        and asset_url in {f["url"] for f in package["files"]}
+                        and deps and (deps.get("file_count") or 0) > 0):
                     return True
         except (httpx.HTTPError, ValueError, KeyError, IndexError, TypeError, AttributeError):
             pass
@@ -641,9 +652,9 @@ def verify_baked(cfg: Config, tags: Dict[str, str], manifest: Dict) -> int:
             download_ok = True
             if healthy and manifest.get("storage_backend") == "storage_api":
                 download_ok = download_smoke("http://localhost:17860")
-                print(f"    /download/asset manifest for a real asset: {download_ok}")
+                print(f"    /download/asset package + dependencies manifests for a real asset: {download_ok}")
                 if not download_ok:
-                    print("    FAIL: hf-space image's download route (hf-storage-api) does not answer")
+                    print("    FAIL: hf-space image's download route (package-download / hf-storage-api) does not answer")
                     ok = False
 
             # Returned asset URLs are this run's public huggingface.co links

pipeline/config.py

@@ -51,6 +51,13 @@ HF_SPACE_GPU_HARDWARE_DEFAULT = "t4-medium"
 STORAGE_BACKENDS = ("s3", "storage_api")
 DEFAULT_STORAGE_BACKEND = "s3"
 
+# Content roots the validator's path fact reads below (usdsearch logprob_asset_path_roots): a Hugging Face dataset
+# at any revision (hf://huggingface.co/datasets/<org>/<ds>/resolve/<rev>/...) and Isaac content at any release.
+VALIDATION_ASSET_PATH_ROOTS = [
+    r"(?i)^(?:[a-z][a-z0-9+.-]*://)?huggingface\.co/datasets/[^/]+/[^/]+/(?:(?:resolve|blob)/[^/]+/)?",
+    r"/Isaac/[^/]+/Isaac/",
+]
+
 REPO_ROOT = Path(__file__).resolve().parent.parent
 BUILD_DIR = REPO_ROOT / ".build"
 DEFAULT_ENV_FILE = REPO_ROOT / "config" / "pipeline.env"
@@ -386,6 +393,16 @@ class Config:
             "USDSEARCH_VISION_VALIDATION_LOGPROB_TOP_LOGPROBS": "20",
             "USDSEARCH_VISION_VALIDATION_LOGPROB_MAX_TOKENS": "1",
             "USDSEARCH_VISION_VALIDATION_MAX_TRIES": "2",
+            # As SimReady Central's validator judged on the Spaces until usdsearch shipped it: views at 448 px,
+            # a text query's judge reads each hit's path (below its dataset or Isaac content root) and vision
+            # caption, and an image + words query asks whether the candidate matches both (usdsearch !279).
+            "USDSEARCH_VISION_VALIDATION_IMAGE_MAX_PX": "448",
+            "USDSEARCH_VISION_VALIDATION_LOGPROB_ASSET_FACTS": json.dumps(["path", "vlm_metadata_caption"]),
+            "USDSEARCH_VISION_VALIDATION_LOGPROB_ASSET_PATH_ROOTS": json.dumps(VALIDATION_ASSET_PATH_ROOTS),
+            "USDSEARCH_VISION_VALIDATION_LOGPROB_IMAGE_TEXT_QUERY": "true",
+            # The Redis verdict cache (on: REDIS_URL is set) holds 250k verdicts (~0.5 GB); the Space's Redis
+            # has no maxmemory.
+            "USDSEARCH_VISION_VALIDATION_CACHE_MAX_ENTRIES": "250000",
         }
 
     # ── VLM metadata extraction from thumbnails (indexing-time plugin) ──────

pipeline/package_download/__init__.py

@@ -0,0 +1,19 @@
+"""Whole-package downloads for the HF Space (supervisord program ``package-download``).
+
+nginx's ``/download`` location sends ``GET /download/asset?asset_url=…[&manifest_only=true]`` here instead of
+to deepsearch-api. The response keeps usdsearch's download contract (the JSON preview, the ZIP with a
+``manifest.json`` at its root, the ``X-Download-Summary`` header, 404/413/503), but the ZIP holds the asset's whole
+SimReady package folder — the nearest ancestor carrying ``com.nvidia.simready.packaging.json``, i.e. the asset
+folder in hf-storage-api's own catalog — rather than the asset plus its USD dependency graph. So the package
+marker, ``.metadata/*.json``, the ``sm_*.json`` sidecar, validation reports, licenses and thumbnails come along,
+laid out exactly as on Hugging Face, contents at the archive root.
+
+Bytes come through the in-container hf-storage-api gRPC server (FileFolderService.ListStat + FileObjectService.
+ReadFromAddress), the same server deepsearch-api's own download already reads through: one HF token, one cache,
+one access boundary (only catalogued folders resolve). This process holds no HF token.
+
+``scope=dependencies``, and any URL that is not inside a catalogued package, is forwarded unchanged to
+deepsearch-api's ``/v3/download/asset`` — the old asset + dependencies bundle.
+
+Design and decision log: docs/plans/hf-space-package-download.md.
+"""

pipeline/package_download/__main__.py

@@ -0,0 +1,35 @@
+"""``python -m pipeline.package_download`` — supervisord's ``package-download`` program."""
+
+from __future__ import annotations
+
+import logging
+import sys
+
+import uvicorn
+
+from .app import create_app
+from .catalog import Catalog
+from .settings import Settings
+
+
+def main() -> int:
+    logging.basicConfig(
+        level=logging.INFO,
+        stream=sys.stdout,
+        format="%(levelname)s %(asctime)s %(name)s - %(message)s",
+    )
+    settings = Settings.from_env()
+    catalog = None
+    if not settings.catalog_path.is_file():
+        # An s3-backend corpus (or a run without a catalog): nothing is a package,
+        # so every request goes to deepsearch-api — the route behaves as before.
+        logging.getLogger("package_download").warning(
+            "no hf-storage-api catalog at %s: forwarding every download to deepsearch-api", settings.catalog_path
+        )
+        catalog = Catalog({})
+    uvicorn.run(create_app(settings, catalog=catalog), host=settings.host, port=settings.port, log_config=None)
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

pipeline/package_download/app.py

@@ -0,0 +1,417 @@
+"""HTTP surface: ``GET /download/asset`` (also ``/v3/download/asset``), ``/health``, ``/metrics``.
+
+Drop-in for usdsearch's download route (see the package docstring); nginx proxies ``/download`` here.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import contextlib
+import dataclasses
+import json
+import logging
+import math
+import shutil
+import tempfile
+import time
+from typing import Any, Awaitable, Callable, Literal, Optional
+
+import httpx
+from fastapi import FastAPI, Query, Request
+from fastapi.responses import FileResponse, JSONResponse, Response, StreamingResponse
+from starlette.background import BackgroundTask
+
+from . import bundle, telemetry
+from .catalog import Catalog
+from .settings import Settings
+
+logger = logging.getLogger("package_download")
+
+# Response headers worth passing back from a forwarded deepsearch-api download.
+_FORWARD_RESPONSE_HEADERS = (
+    "content-type",
+    "content-length",
+    "content-encoding",
+    "content-disposition",
+    "x-download-summary",
+    "access-control-expose-headers",
+    "retry-after",
+)
+_HOP_BY_HOP = {"host", "connection", "keep-alive", "transfer-encoding", "upgrade", "content-length", "te", "trailer"}
+
+# SourceError.kind -> (status, client-facing detail). "denied" is the backend's
+# own HF token being refused, not the caller's access: answered as 502, never 403,
+# which the Explorer would show as "Access denied" for a public asset.
+_SOURCE_STATUS = {
+    "not_found": (404, "Asset not found"),
+    "denied": (502, "The storage backend could not read this package (check HF_STORAGE_API_TOKEN)."),
+    "busy": (503, "The storage backend is rate limited; retry shortly."),
+    "unavailable": (503, "The storage backend is unavailable; retry shortly."),
+    "error": (502, "The storage backend failed."),
+}
+
+
+UrlMapping = tuple  # (to_internal(url), to_public_document(doc), to_public(url))
+
+
+def _default_url_mapping() -> UrlMapping:
+    """deepsearch-api's own inbound/outbound URL translation (same USDSEARCH_URL_MAPPING_* env)."""
+    try:
+        from search_utils.url_mapping import rewrite_response_document, to_internal_url, to_public_url
+    except ImportError:  # local tests without the usdsearch wheels
+        logger.warning("search_utils not installed: asset URLs are not translated")
+        return (lambda url: url), (lambda doc: doc), (lambda url: url)
+    return to_internal_url, rewrite_response_document, to_public_url
+
+
+def _public_folder(to_public: Callable[[str], str], address: str) -> str:
+    """Public link for the package folder. rewrite_response_document only knows its
+    own URL fields, and the rules map to HF's file viewer (`/blob/<rev>/`); a folder's
+    page is `/tree/<rev>/`."""
+    public = to_public(address)
+    return public.replace("/blob/", "/tree/", 1) if public != address else public
+
+
+class _Metrics:
+    def __init__(self):
+        try:
+            from prometheus_client import CollectorRegistry, Counter, Histogram, generate_latest
+        except ImportError:
+            self.registry = None
+            return
+        self.registry = CollectorRegistry()
+        self._generate = generate_latest
+        self.requests = Counter(
+            "package_download_requests_total",
+            "Download requests by route taken and outcome",
+            ["route", "mode", "outcome"],
+            registry=self.registry,
+        )
+        self.bytes = Counter("package_download_bytes_total", "Package bytes written into ZIPs", registry=self.registry)
+        self.seconds = Histogram(
+            "package_download_seconds",
+            "Wall clock of package downloads (headers sent)",
+            ["mode"],
+            buckets=(0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60, 120, 300),
+            registry=self.registry,
+        )
+
+    def observe(self, route: str, mode: str, outcome: str, seconds: Optional[float] = None, nbytes: int = 0):
+        if self.registry is None:
+            return
+        self.requests.labels(route, mode, outcome).inc()
+        if route == "package" and seconds is not None:
+            self.seconds.labels(mode).observe(seconds)
+        if nbytes:
+            self.bytes.inc(nbytes)
+
+    def render(self) -> Response:
+        if self.registry is None:
+            return Response("prometheus_client not installed\n", media_type="text/plain")
+        return Response(self._generate(self.registry), media_type="text/plain; version=0.0.4")
+
+
+class _CleanupFileResponse(FileResponse):
+    """Serves the ZIP, then removes its temp dir and frees the slot — whatever happens.
+
+    Not Starlette's ``background=``: that only runs after a successful send, so a
+    client hanging up mid-download would leak the archive (usdsearch's
+    ``_TempDirFileResponse``, same reason).
+    """
+
+    def __init__(self, *args, cleanup_dir: str, on_close: Callable[[], None], **kwargs):
+        super().__init__(*args, **kwargs)
+        self._cleanup_dir = cleanup_dir
+        self._on_close = on_close
+
+    async def __call__(self, scope, receive, send) -> None:
+        try:
+            await super().__call__(scope, receive, send)
+        finally:
+            shutil.rmtree(self._cleanup_dir, ignore_errors=True)
+            self._on_close()
+
+
+class _HTTPError(Exception):
+    def __init__(self, status: int, detail: str, headers: Optional[dict] = None):
+        self.status, self.detail, self.headers = status, detail, headers or {}
+
+
+def _source_error(exc: bundle.SourceError) -> _HTTPError:
+    status, detail = _SOURCE_STATUS.get(exc.kind, _SOURCE_STATUS["error"])
+    if status != 503:
+        return _HTTPError(status, detail)
+    wait = "30" if exc.retry_after is None else str(math.ceil(exc.retry_after))
+    return _HTTPError(status, detail, {"Retry-After": wait})
+
+
+class _ClientGone(Exception):
+    pass
+
+
+async def _client_gone(request: Request) -> None:
+    """Returns once the client has hung up. uvicorn never cancels a handler itself; it
+    only answers ``receive()`` with ``http.disconnect`` (after the request's own body)."""
+    while (await request.receive())["type"] != "http.disconnect":
+        pass
+
+
+async def _unless_client_gone(request: Request, work: Awaitable[Response]) -> Response:
+    """``work``'s response — unless the client hangs up first (or nginx gives up on it):
+    then ``work`` is cancelled and awaited, so every read it started has stopped and its
+    slot and temp dir are freed, and _ClientGone is raised."""
+    task = asyncio.ensure_future(work)
+    gone = asyncio.ensure_future(_client_gone(request))
+    try:
+        await asyncio.wait({task, gone}, return_when=asyncio.FIRST_COMPLETED)
+    finally:
+        gone.cancel()
+        if not task.done():
+            task.cancel()
+            await asyncio.wait({task})
+    if task.cancelled():
+        raise _ClientGone
+    return task.result()
+
+
+def create_app(
+    settings: Optional[Settings] = None,
+    *,
+    catalog: Optional[Catalog] = None,
+    source: Any = None,
+    url_mapping: Optional[UrlMapping] = None,
+    fallback_transport: Optional[httpx.AsyncBaseTransport] = None,
+) -> FastAPI:
+    settings = settings or Settings.from_env()
+    to_internal, to_public, to_public_url = url_mapping or _default_url_mapping()
+
+    def public(doc: dict) -> dict:
+        doc = to_public(doc)
+        doc["package_root"] = _public_folder(to_public_url, doc["package_root"])
+        return doc
+
+    metrics = _Metrics()
+
+    @contextlib.asynccontextmanager
+    async def lifespan(app: FastAPI):
+        nonlocal catalog, source
+        if catalog is None:
+            catalog = Catalog.load(settings.catalog_path)
+        owned_source = source is None
+        if owned_source:
+            from .source import StorageApiSource
+
+            source = StorageApiSource(
+                settings.storage_api_endpoint,
+                list_timeout=settings.list_timeout,
+                read_timeout=settings.read_timeout,
+            )
+        app.state.fallback = httpx.AsyncClient(
+            base_url=settings.fallback_url,
+            timeout=httpx.Timeout(30.0, read=settings.fallback_timeout),
+            transport=fallback_transport,
+        )
+        logger.info(
+            "package-download: %d catalogued package(s), storage API %s, fallback %s",
+            len(catalog), settings.storage_api_endpoint, settings.fallback_url,
+        )
+        try:
+            yield
+        finally:
+            await app.state.fallback.aclose()
+            if owned_source:
+                await source.close()
+
+    app = FastAPI(title="package-download", docs_url=None, redoc_url=None, openapi_url=None, lifespan=lifespan)
+    # One slot per ZIP being assembled *and* sent: the archive occupies temp_dir until
+    # the client has it (usdsearch's DOWNLOAD_MAX_CONCURRENT, same reasoning).
+    app.state.slots = slots = asyncio.Semaphore(settings.max_concurrent) if settings.max_concurrent > 0 else None
+    # Reads from hf-storage-api in flight across every request — each one a live HF download.
+    app.state.connections = connections = (
+        asyncio.Semaphore(settings.hf_max_connections) if settings.hf_max_connections > 0 else None
+    )
+    # Listings in flight, capped apart from the reads: a listing (every manifest_only preview)
+    # never queues behind a ZIP's file reads.
+    app.state.listings = listings = (
+        asyncio.Semaphore(settings.hf_max_listings) if settings.hf_max_listings > 0 else None
+    )
+    retry = bundle.RetryPolicy(settings.retry_attempts, settings.retry_base_delay, settings.retry_max_delay)
+    # hf-storage-api already retries its HF tree fetch (5 tries, waiting out HF's rate-limit reset).
+    list_retry = dataclasses.replace(retry, attempts=min(retry.attempts, 2))
+    stored_extensions = settings.stored_extension_set()
+
+    @app.get("/health")
+    async def health():
+        return {"status": "ok", "packages": len(catalog) if catalog is not None else 0}
+
+    @app.get("/metrics")
+    async def prometheus():
+        return metrics.render()
+
+    async def forward(request: Request) -> Response:
+        """Hand the request to deepsearch-api's own /v3/download/asset, unchanged."""
+        params = [(k, v) for k, v in request.query_params.multi_items() if k != "scope"]
+        headers = {k: v for k, v in request.headers.items() if k.lower() not in _HOP_BY_HOP}
+        client: httpx.AsyncClient = request.app.state.fallback
+        try:
+            upstream = await client.send(
+                client.build_request("GET", "/v3/download/asset", params=params, headers=headers), stream=True
+            )
+        except httpx.HTTPError as exc:
+            logger.warning("forwarding to deepsearch-api failed: %s", exc)
+            metrics.observe("forward", "zip", "502")
+            return JSONResponse({"detail": "The dependency download service is unavailable."}, status_code=502)
+        out_headers = {k: v for k, v in upstream.headers.items() if k.lower() in _FORWARD_RESPONSE_HEADERS}
+        return StreamingResponse(
+            upstream.aiter_raw(),
+            status_code=upstream.status_code,
+            headers=out_headers,
+            background=BackgroundTask(upstream.aclose),
+        )
+
+    async def acquire_slot() -> Callable[[], None]:
+        if slots is None:
+            return lambda: None
+        try:
+            await asyncio.wait_for(slots.acquire(), timeout=settings.queue_timeout)
+        except (asyncio.TimeoutError, TimeoutError):
+            raise _HTTPError(503, "Too many downloads in progress; retry shortly.", {"Retry-After": "30"})
+        released = False
+
+        def release() -> None:
+            nonlocal released
+            if not released:
+                released = True
+                slots.release()
+
+        return release
+
+    async def list_package(package) -> list:
+        async with bundle.limited(listings):
+            return await source.list_package(package)
+
+    async def package_download(asset_url: str, package, manifest_only: bool, stats: dict) -> Response:
+        try:
+            listed = await bundle.retrying(lambda: list_package(package), list_retry, package.root_address)
+        except bundle.SourceError as exc:
+            logger.warning("listing %s failed: %s", package.root_address, exc)
+            raise _source_error(exc)
+        stats["n_listed"] = len(listed)
+        if not any(f.path == package.asset_path for f in listed):
+            raise _HTTPError(404, f"Asset '{to_public_url(asset_url)}' not found")
+        total = sum(f.size for f in listed)
+        stats["total_bytes"] = total
+
+        if manifest_only:
+            return JSONResponse(public(bundle.preview(asset_url, package, listed)))
+
+        if settings.max_bundle_bytes > 0 and total > settings.max_bundle_bytes:
+            raise _HTTPError(
+                413, f"Download bundle is too large ({total} bytes; limit is {settings.max_bundle_bytes})."
+            )
+
+        release = await acquire_slot()
+        tmp_dir = tempfile.mkdtemp(prefix="package-download-", dir=settings.temp_dir or None)
+        try:
+            zip_path = f"{tmp_dir}/bundle.zip"
+            try:
+                result = await bundle.build_zip(
+                    zip_path,
+                    package,
+                    listed,
+                    source.read,
+                    concurrency=settings.concurrency,
+                    compression=settings.compression,
+                    stored_extensions=stored_extensions,
+                    spool_bytes=settings.spool_bytes,
+                    max_bytes=settings.max_bundle_bytes,
+                    spool_dir=tmp_dir,
+                    read_ahead_bytes=settings.read_ahead_bytes,
+                    connections=connections,
+                    retry=retry,
+                )
+            except bundle.RootMissing as exc:
+                raise _HTTPError(404, f"Asset '{to_public_url(asset_url)}' could not be retrieved: {exc}")
+            except bundle.BundleTooLarge as exc:
+                raise _HTTPError(413, str(exc))
+            except bundle.SourceError as exc:
+                raise _source_error(exc)
+            stats.update(n_files=len(result.files), n_failed=len(result.errors), total_bytes=result.total_bytes,
+                         fetch_ms=result.fetch_ms)
+            doc = public(bundle.manifest(asset_url, package, listed, result))
+            await asyncio.get_running_loop().run_in_executor(
+                None, bundle.finish_zip, zip_path, bundle.manifest_name(listed, package), doc
+            )
+        except BaseException:
+            # No response will carry these away (includes the client hanging up mid-bundle:
+            # _unless_client_gone cancels this).
+            shutil.rmtree(tmp_dir, ignore_errors=True)
+            release()
+            raise
+
+        return _CleanupFileResponse(
+            zip_path,
+            media_type="application/zip",
+            filename=f"{package.name}.zip",
+            headers={
+                "X-Download-Summary": json.dumps(doc["summary"]),
+                "Access-Control-Expose-Headers": "X-Download-Summary, Content-Disposition",
+            },
+            cleanup_dir=tmp_dir,
+            on_close=release,
+        )
+
+    async def download_asset(
+        request: Request,
+        asset_url: str = Query(description="Storage URL of the asset (public or internal form)."),
+        manifest_only: bool = Query(False, description="Return the JSON preview instead of the ZIP."),
+        scope: Literal["package", "dependencies"] = Query(
+            "package",
+            description="`package` (default): the asset's whole SimReady package folder. "
+            "`dependencies`: the asset + its USD dependency graph (deepsearch-api's own bundle).",
+        ),
+    ) -> Response:
+        mode = "manifest" if manifest_only else "zip"
+        internal_url = to_internal(asset_url)
+        package = catalog.package_for(internal_url) if scope == "package" else None
+        if package is None:
+            metrics.observe("forward", mode, "forwarded")
+            return await forward(request)
+
+        started = time.perf_counter()
+        stats: dict = {}
+        status, http_status, error_type = "success", 200, None
+        try:
+            return await _unless_client_gone(request, package_download(internal_url, package, manifest_only, stats))
+        except _ClientGone:
+            # Nobody is listening: the status is for the logs (nginx's "client closed request").
+            status, http_status = "cancelled", 499
+            logger.info("client went away; download of %s stopped", package.root_address)
+            return Response(status_code=499)
+        except _HTTPError as exc:
+            status, http_status, error_type = "error", exc.status, "HTTPException"
+            return JSONResponse({"detail": exc.detail}, status_code=exc.status, headers=exc.headers)
+        except Exception as exc:
+            status, http_status, error_type = "error", 500, type(exc).__name__
+            logger.exception("package download of %s failed", package.root_address)
+            return JSONResponse({"detail": "Internal error while building the package download."}, status_code=500)
+        finally:
+            elapsed = time.perf_counter() - started
+            sent = stats.get("total_bytes", 0) if status == "success" and not manifest_only else 0
+            metrics.observe("package", mode, str(http_status), elapsed, sent)
+            telemetry.emit(
+                asset_url=internal_url,
+                manifest_only=manifest_only,
+                scope=bundle.SCOPE,
+                package=package.name,
+                status=status,
+                error_type=error_type,
+                http_status=http_status,
+                processing_time_ms=elapsed * 1000,
+                user_agent=request.headers.get("user-agent"),
+                **stats,
+            )
+
+    for path in ("/download/asset", "/v3/download/asset"):
+        app.add_api_route(path, download_asset, methods=["GET"])
+    return app

pipeline/package_download/bundle.py

@@ -0,0 +1,468 @@
+"""Package listing -> usdsearch-contract preview/manifest, and the ZIP itself.
+
+Contract (usdsearch ``routers_v3/download.py`` at the Space's wheel commit, and what the Explorer bundle reads —
+docs/plans/hf-space-package-download.md §1.2/§1.4): the preview's ``total_size``/``file_count``, the
+``skipped[]`` list, and the ``X-Download-Summary`` keys. Every summary key is kept; the ``skipped_*`` counters
+are always 0 here — a package is listed, not resolved, so nothing is ever skipped — which keeps the Explorer's
+"some files skipped" toast for real failures (``download_failed``) only. ``scope``, ``package_files``,
+``package_root`` and ``root_path`` are additive.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import contextlib
+import json
+import logging
+import math
+import os
+import random
+import re
+import tempfile
+import threading
+import time
+import zipfile
+from collections import deque
+from concurrent.futures import ThreadPoolExecutor
+from dataclasses import dataclass, field
+from typing import AsyncIterator, Awaitable, Callable, Deque, Dict, List, Optional, Tuple, TypeVar
+
+from .catalog import Package
+
+logger = logging.getLogger("package_download")
+T = TypeVar("T")
+
+SCOPE = "package"
+MANIFEST_NAME = "manifest.json"
+# Used instead when a package carries its own root-level manifest.json, so the
+# package itself still unpacks byte-identical.
+MANIFEST_FALLBACK_NAME = "download-manifest.json"
+SKIP_COUNTERS = (
+    "skipped_not_found",
+    "skipped_access_denied",
+    "skipped_no_access_or_missing",
+    "skipped_deleted",
+    "skipped_resolved_elsewhere",
+)
+
+
+@dataclass(frozen=True)
+class FileEntry:
+    path: str  # repo-relative, as on Hugging Face (not percent-decoded)
+    size: int
+
+
+class SourceError(Exception):
+    """A storage-side failure, classified so the HTTP layer needs no gRPC knowledge.
+
+    kind: "not_found" | "denied" (the backend's HF token was refused) | "busy"
+    (rate limited) | "unavailable" | "error". ``retryable``: a transient failure
+    worth another try (see `classify`); ``retry_after``: seconds the backend asked
+    the caller to wait, when it said.
+    """
+
+    def __init__(self, kind: str, message: str, *, retryable: bool = False, retry_after: Optional[float] = None):
+        super().__init__(message)
+        self.kind = kind
+        self.retryable = retryable
+        self.retry_after = retry_after
+
+
+# gRPC status code name -> SourceError.kind
+_KINDS = {
+    "NOT_FOUND": "not_found",
+    "INVALID_ARGUMENT": "not_found",
+    # hf-storage-api maps an HF 401/403 (dead or missing token) to PERMISSION_DENIED.
+    "PERMISSION_DENIED": "denied",
+    "UNAUTHENTICATED": "denied",
+    "RESOURCE_EXHAUSTED": "busy",
+    "UNAVAILABLE": "unavailable",
+    "DEADLINE_EXCEEDED": "unavailable",
+}
+# hf-storage-api (hf/client.py `_check`) words every HF status but 401/403/404 as
+# UNAVAILABLE "<status> from <url>: …", and (errors.status_for) any other failure as
+# INTERNAL "<exception class>: …"; these httpx transport errors are the transient ones.
+_HF_STATUS = re.compile(r"(\d{3}) from ")
+_TRANSPORT_ERRORS = frozenset({
+    "ConnectError", "ConnectTimeout", "ReadError", "ReadTimeout", "WriteError", "WriteTimeout",
+    "PoolTimeout", "RemoteProtocolError", "NetworkError",
+})
+
+
+def _seconds(value) -> Optional[float]:
+    try:
+        seconds = float(value)
+    except (TypeError, ValueError):
+        return None
+    return seconds if math.isfinite(seconds) and seconds >= 0 else None
+
+
+def classify(code: str, details: str, retry_after: Optional[str] = None) -> SourceError:
+    """A gRPC failure from hf-storage-api (status code name, details, its ``retry-after`` trailer) as a SourceError.
+
+    Retryable: rate limiting, an HF 429/5xx, hf-storage-api unreachable (UNAVAILABLE with no HF
+    status: it is restarting), and a dropped HF connection. Never: DEADLINE_EXCEEDED (the whole-file
+    deadline — another try would run past nginx's own 300 s), any other HF 4xx (a per-file failure,
+    kind "error"), not found, denied, cancelled. Every retryable failure is kind "busy" or
+    "unavailable", so one that outlasts its retries fails the whole bundle with 503 + Retry-After,
+    never a 404 root or a ZIP with a hole. The details' wording is hf-storage-api's own — keep
+    in step with the vendored copy (tests pin it). It sends no ``retry-after`` today.
+    """
+    kind = _KINDS.get(code, "error")
+    retryable = kind == "busy"
+    if kind == "unavailable" and code != "DEADLINE_EXCEEDED":
+        status = _HF_STATUS.match(details)
+        http_status = int(status.group(1)) if status else None
+        if http_status == 429:
+            kind, retryable = "busy", True
+        elif http_status is not None and http_status < 500:
+            kind = "error"
+        else:
+            retryable = True
+    elif code == "INTERNAL" and details.split(":", 1)[0].strip() in _TRANSPORT_ERRORS:
+        kind, retryable = "unavailable", True  # HF's connection dropped: as transient as a 5xx
+    return SourceError(kind, f"{code}: {details}", retryable=retryable, retry_after=_seconds(retry_after))
+
+
+@dataclass(frozen=True)
+class RetryPolicy:
+    """Retries of a transient SourceError: ``attempts`` tries in all (1 = none), exponential backoff with
+    full jitter (a random wait up to ``base_delay`` × 2^n, at most ``max_delay``). A ``retry_after``
+    is waited out as given, unless it is longer than ``max_delay``: then the failure stands, with it."""
+
+    attempts: int = 1
+    base_delay: float = 0.5
+    max_delay: float = 10.0
+    sleep: Callable[[float], Awaitable[None]] = asyncio.sleep
+    rand: Callable[[], float] = random.random
+
+    def delay(self, exc: SourceError, attempt: int) -> Optional[float]:
+        """Seconds to wait after failed try ``attempt`` (0-based) before the next one, or None to give up."""
+        if not exc.retryable or attempt + 1 >= self.attempts:
+            return None
+        if exc.retry_after is not None:
+            return exc.retry_after if exc.retry_after <= self.max_delay else None
+        return self.rand() * min(self.max_delay, self.base_delay * 2 ** attempt)
+
+
+NO_RETRY = RetryPolicy()
+
+
+async def retrying(call: Callable[[], Awaitable[T]], policy: RetryPolicy, what: str) -> T:
+    """``call()``, tried again per ``policy`` while it fails with a transient SourceError.
+
+    Only SourceError is caught: a cancellation (the client gone) ends it at once, backoff included.
+    """
+    attempt = 0
+    while True:
+        try:
+            return await call()
+        except SourceError as exc:
+            delay = policy.delay(exc, attempt)
+            if delay is None:
+                raise
+            logger.warning("%s: %s; try %d of %d in %.1f s", what, exc, attempt + 2, policy.attempts, delay)
+        await policy.sleep(delay)
+        attempt += 1
+
+
+def limited(semaphore: Optional[asyncio.Semaphore]):
+    """``async with`` the semaphore, or nothing when there is no limit."""
+    return semaphore if semaphore is not None else contextlib.nullcontext()
+
+
+class _ByteBudget:
+    """One bundle's read-ahead RAM (``limit`` bytes; 0 = unlimited), granted strictly first come, first served.
+
+    A request larger than the whole budget is granted once nothing else is held — so it never deadlocks
+    the bundle, and is fetched alone — and a small request never overtakes a waiting large one.
+    (build_zip never asks for more than the budget; a request of all of it is fetched alone.)
+    """
+
+    def __init__(self, limit: int):
+        self.limit = limit
+        self.used = 0
+        self._waiters: Deque[Tuple[int, asyncio.Future]] = deque()
+
+    def cost(self, size: int) -> int:
+        return min(size, self.limit) if self.limit > 0 else 0
+
+    def _fits(self, n: int) -> bool:
+        return self.limit <= 0 or self.used == 0 or self.used + n <= self.limit
+
+    async def acquire(self, n: int) -> None:
+        if not self._waiters and self._fits(n):
+            self.used += n
+            return
+        waiter = (n, asyncio.get_running_loop().create_future())
+        self._waiters.append(waiter)
+        try:
+            await waiter[1]
+        except asyncio.CancelledError:
+            if waiter[1].done() and not waiter[1].cancelled():
+                self.used -= n  # granted just as we were cancelled: hand it back
+            elif waiter in self._waiters:
+                self._waiters.remove(waiter)
+            self._grant()
+            raise
+
+    def release(self, n: int) -> None:
+        self.used -= n
+        self._grant()
+
+    def _grant(self) -> None:
+        while self._waiters:
+            n, future = self._waiters[0]
+            if future.done():  # its task was cancelled; it cleans up after itself
+                self._waiters.popleft()
+                continue
+            if not self._fits(n):
+                return
+            self._waiters.popleft()
+            self.used += n
+            future.set_result(None)
+
+
+class BundleTooLarge(Exception):
+    pass
+
+
+class RootMissing(Exception):
+    pass
+
+
+def summary(listed: int, downloaded: int, failed: int) -> dict:
+    """The per-bundle summary, in usdsearch's key order plus the additive keys."""
+    return {
+        # usdsearch counts the root's dependencies; here every other file in the
+        # package — the Explorer's preview shows "dependencies" as file_count - 1.
+        "dependencies_reported": max(listed - 1, 0),
+        "downloaded": downloaded,
+        **{key: 0 for key in SKIP_COUNTERS},
+        "respelled": 0,
+        "download_failed": failed,
+        "scope": SCOPE,
+        "package_files": listed,
+    }
+
+
+def manifest_name(files: List[FileEntry], package: Package) -> str:
+    names = {package.relative(f.path) for f in files}
+    return MANIFEST_FALLBACK_NAME if MANIFEST_NAME in names else MANIFEST_NAME
+
+
+def preview(asset_url: str, package: Package, files: List[FileEntry]) -> dict:
+    """The ``manifest_only=true`` response: what the ZIP would hold, no bytes moved."""
+    return {
+        "root_url": asset_url,
+        "file_count": len(files),
+        "total_size": sum(f.size for f in files),
+        "files": [{"url": package.address(f.path), "size": f.size} for f in files],
+        "errors": {},
+        "summary": {**summary(len(files), downloaded=len(files), failed=0), "accessible": len(files)},
+        "skipped": [],
+        "scope": SCOPE,
+        "package_root": package.root_address,
+        "root_path": package.root_path,
+    }
+
+
+def _compress_type(arc: str, mode: str, stored_extensions: frozenset) -> int:
+    if mode == "stored":
+        return zipfile.ZIP_STORED
+    if mode == "deflate":
+        return zipfile.ZIP_DEFLATED
+    return zipfile.ZIP_STORED if os.path.splitext(arc)[1].lower() in stored_extensions else zipfile.ZIP_DEFLATED
+
+
+def safe_arcname(package: Package, path: str) -> Optional[str]:
+    """In-archive name for a listed file, or None if it would escape the archive root."""
+    if not path.startswith(package.folder + "/"):
+        return None
+    arc = package.relative(path)
+    parts = arc.split("/")
+    if not arc or arc.startswith("/") or any(p in ("", ".", "..") for p in parts):
+        return None
+    return arc
+
+
+@dataclass
+class BundleResult:
+    files: List[dict] = field(default_factory=list)  # {url, path, size} per member written
+    errors: Dict[str, str] = field(default_factory=dict)  # address -> message
+    total_bytes: int = 0
+    fetch_ms: float = 0.0
+
+
+Reader = Callable[[Package, str], AsyncIterator[bytes]]  # an async generator
+_COPY_CHUNK = 1024 * 1024
+
+
+async def build_zip(
+    zip_path: str,
+    package: Package,
+    files: List[FileEntry],
+    read: Reader,
+    *,
+    concurrency: int,
+    compression: str,
+    stored_extensions: frozenset,
+    spool_bytes: int,
+    max_bytes: int = 0,
+    spool_dir: Optional[str] = None,
+    read_ahead_bytes: int = 0,
+    connections: Optional[asyncio.Semaphore] = None,
+    retry: RetryPolicy = NO_RETRY,
+) -> BundleResult:
+    """Fetch every listed file (bounded concurrency) and write it into a ZIP at ``zip_path``.
+
+    Each file is buffered in a SpooledTemporaryFile (RAM up to ``spool_bytes``, capped
+    at ``read_ahead_bytes``, then ``spool_dir``; ``spool_bytes`` 0 spills at once), then
+    copied into the archive by a single writer thread — ZipFile is not thread-safe, and
+    deflate stays off the event loop. A file that fails is recorded in ``errors`` and
+    left out; the root asset failing raises RootMissing (usdsearch's 404). Does not
+    write the manifest — see `finish_zip`.
+
+    Each file takes, in this order: the part of its listed size its spool keeps in RAM
+    from ``read_ahead_bytes`` (this bundle's RAM for bytes fetched but not yet written;
+    0 = unlimited — what spills to disk is not charged, so large files still stream side
+    by side), a ``concurrency`` slot, then — only while its bytes stream, never during a
+    backoff — a ``connections`` slot (reads in flight across every bundle). A transient
+    failure is retried per ``retry`` from byte 0: nothing reaches the archive before a
+    file is whole. Cancelled, it stops every read and the writer before it returns.
+    """
+    result = BundleResult()
+    semaphore = asyncio.Semaphore(max(concurrency, 1))
+    budget = _ByteBudget(read_ahead_bytes)
+    # A spool's RAM before it spills: never more than the budget, so it holds no more than
+    # it is charged, and at least 1 — a SpooledTemporaryFile with max_size 0 never spills.
+    in_ram = max(min(spool_bytes, read_ahead_bytes) if read_ahead_bytes > 0 else spool_bytes, 1)
+    mode = compression.strip().lower()
+    started = time.perf_counter()
+
+    async def download(entry: FileEntry, spool) -> int:
+        async def attempt() -> int:
+            spool.seek(0)
+            spool.truncate()  # a retry starts over: nothing of a failed try is kept
+            async with limited(connections):
+                async with contextlib.aclosing(read(package, entry.path)) as chunks:
+                    async for chunk in chunks:
+                        spool.write(chunk)
+            return spool.tell()
+
+        return await retrying(attempt, retry, package.address(entry.path))
+
+    async def fetch(entry: FileEntry):
+        # (entry, spool, size, exc, cost): `cost` stays reserved until the
+        # consumer has written the spool, or dropped it.
+        cost = budget.cost(min(entry.size, in_ram))
+        await budget.acquire(cost)
+        spool = tempfile.SpooledTemporaryFile(max_size=in_ram, dir=spool_dir)
+        try:
+            async with semaphore:
+                size = await download(entry, spool)
+        except asyncio.CancelledError:
+            spool.close()
+            budget.release(cost)
+            raise
+        except Exception as exc:  # noqa: BLE001 — best-effort per file, like usdsearch
+            spool.close()
+            budget.release(cost)
+            return entry, None, 0, exc, 0
+        spool.seek(0)
+        return entry, spool, size, None, cost
+
+    tasks = [asyncio.create_task(fetch(f)) for f in files]
+    stop = threading.Event()  # tells the writer thread to let go of the member it is copying
+    try:
+        with ThreadPoolExecutor(max_workers=1, thread_name_prefix="package-zip") as writer:
+            with zipfile.ZipFile(zip_path, "w", zipfile.ZIP_DEFLATED, allowZip64=True) as zf:
+                for completed in asyncio.as_completed(tasks):
+                    entry, spool, size, exc, cost = await completed
+                    address = package.address(entry.path)
+                    if spool is None:
+                        if isinstance(exc, SourceError) and exc.kind in ("denied", "busy", "unavailable"):
+                            raise exc  # the backend itself is down; a ZIP of holes helps nobody
+                        if entry.path == package.asset_path:
+                            raise RootMissing(str(exc) or "not found")
+                        result.errors[address] = str(exc) or type(exc).__name__
+                        continue
+                    try:
+                        with spool:
+                            result.total_bytes += size
+                            if max_bytes > 0 and result.total_bytes > max_bytes:
+                                raise BundleTooLarge(f"Download bundle exceeded the size limit of {max_bytes} bytes.")
+                            arc = safe_arcname(package, entry.path)
+                            if arc is None:
+                                result.errors[address] = "path outside the package"
+                                continue
+                            info = zipfile.ZipInfo(arc, date_time=time.localtime()[:6])
+                            info.compress_type = _compress_type(arc, mode, stored_extensions)
+                            info.external_attr = 0o644 << 16
+                            await _write_member(writer, stop, zf, info, spool, size)
+                            result.files.append({"url": address, "path": arc, "size": size})
+                    finally:
+                        budget.release(cost)
+    finally:
+        for task in tasks:
+            task.cancel()
+        if tasks:
+            # Until they have really stopped: a cancelled read ends its RPC (and
+            # hf-storage-api its HF request) before the caller frees the slot and temp dir.
+            await asyncio.wait(tasks)
+        # Close the spools of fetches that finished but were never written (an
+        # exception above): a disk-spilled one would otherwise hold a temp file.
+        for task in tasks:
+            if not task.cancelled() and task.exception() is None:
+                _, spool, _, _, cost = task.result()
+                if spool is not None and not spool.closed:
+                    spool.close()
+                    budget.release(cost)
+    result.fetch_ms = (time.perf_counter() - started) * 1000
+    return result
+
+
+async def _write_member(writer: ThreadPoolExecutor, stop: threading.Event, zf, info, spool, size: int) -> None:
+    """Copy one spooled file into the archive on the writer thread.
+
+    Cancelled mid-copy, it stops the thread and waits for it to let go of the member
+    and the spool, without blocking the event loop: otherwise ZipFile.close() would
+    refuse an open member and the executor's shutdown would wait on the loop.
+    """
+    copying = writer.submit(_copy_member, zf, info, spool, size, stop)
+    try:
+        await asyncio.wrap_future(copying)
+    finally:
+        if not copying.done():
+            stop.set()
+            await asyncio.wait({asyncio.wrap_future(copying)})
+
+
+def _copy_member(zf: zipfile.ZipFile, info: zipfile.ZipInfo, src, size: int, stop: threading.Event) -> None:
+    with zf.open(info, "w", force_zip64=size >= zipfile.ZIP64_LIMIT) as dst:
+        while not stop.is_set():  # set only when the bundle is abandoned
+            chunk = src.read(_COPY_CHUNK)
+            if not chunk:
+                break
+            dst.write(chunk)
+
+
+def finish_zip(zip_path: str, name: str, manifest: dict) -> None:
+    with zipfile.ZipFile(zip_path, "a") as zf:
+        zf.writestr(name, json.dumps(manifest, indent=2))
+
+
+def manifest(asset_url: str, package: Package, listed: List[FileEntry], result: BundleResult) -> dict:
+    """The ``manifest.json`` written into the archive (usdsearch's shape + additive keys)."""
+    return {
+        "root_url": asset_url,
+        "file_count": len(result.files),
+        "total_size": result.total_bytes,
+        "files": sorted(result.files, key=lambda f: f["path"]),
+        "summary": summary(len(listed), downloaded=len(result.files), failed=len(result.errors)),
+        "skipped": [],
+        "errors": result.errors,
+        "scope": SCOPE,
+        "package_root": package.root_address,
+        "root_path": package.root_path,
+    }

pipeline/package_download/catalog.py

@@ -0,0 +1,131 @@
+"""``asset_url`` -> the SimReady package folder it belongs to, from hf-storage-api's own catalog.
+
+The catalog's ``assets:`` are exactly the package folders: pipeline/hf_catalog.py maps every matched file to the
+nearest ancestor holding the corpus's ``asset_marker`` (``com.nvidia.simready.packaging.json``). Reading the same
+file hf-storage-api serves from means the two can't disagree about what a package is, and an address outside
+every catalogued folder never becomes a package download (hf-storage-api's ``resolve()`` refuses it again
+server-side).
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from pathlib import Path, PurePosixPath
+from typing import Dict, FrozenSet, Optional, Tuple
+from urllib.parse import quote
+
+import yaml
+
+from .. import hf_addressing
+
+
+def _host(endpoint: str) -> str:
+    # hf-storage-api's Catalog.host(): the endpoint minus its scheme.
+    return endpoint.split("://", 1)[-1].rstrip("/")
+
+
+def encode_path(path: str) -> str:
+    """Percent-encode a repo path for an address handed to hf-storage-api.
+
+    Its ``addressing.parse()`` unquotes every segment, so a file literally named
+    ``Package%401.0.0.json`` must travel as ``Package%25401.0.0.json`` to arrive
+    as itself. A no-op for ordinary names.
+    """
+    return quote(path, safe="/")
+
+
+@dataclass(frozen=True)
+class Package:
+    host: str
+    repo_type: str
+    repo_id: str
+    revision: str  # the catalog's configured revision (a pinned sha for simready-assets)
+    folder: str  # repo-relative package folder, e.g. "appliance_toaster_a01"
+    asset_path: str  # repo-relative path of the requested asset inside it
+
+    @property
+    def name(self) -> str:
+        return PurePosixPath(self.folder).name
+
+    @property
+    def root_path(self) -> str:
+        """The requested asset's path inside the archive (package contents sit at its root)."""
+        return self.relative(self.asset_path)
+
+    def relative(self, path: str) -> str:
+        return path[len(self.folder) + 1:]
+
+    def address(self, path: str, *, is_folder: bool = False) -> str:
+        """The hf:// address hf-storage-api resolves for a repo path (encoded, see `encode_path`)."""
+        return hf_addressing.build(
+            host=self.host,
+            repo_type=self.repo_type,
+            repo_id=self.repo_id,
+            revision=self.revision,
+            path=encode_path(path),
+            is_folder=is_folder,
+        )
+
+    @property
+    def root_address(self) -> str:
+        return self.address(self.folder, is_folder=True)
+
+    @property
+    def repo_prefix(self) -> str:
+        """Every address hf-storage-api emits for this repo starts with this."""
+        return hf_addressing.build(
+            host=self.host, repo_type=self.repo_type, repo_id=self.repo_id, revision=self.revision, is_folder=True
+        )
+
+
+class Catalog:
+    """The repos and package folders of one hf-storage-api catalog (YAML)."""
+
+    def __init__(self, doc: dict):
+        endpoint = doc.get("endpoint") or "https://huggingface.co"
+        self._repos: Dict[Tuple[str, str, str], Tuple[str, FrozenSet[str]]] = {}
+        for repo in doc.get("repos") or []:
+            key = (_host(repo.get("endpoint") or endpoint), repo.get("repo_type") or "dataset", repo["repo_id"])
+            # "" (a file at the repo root with no marker above it) is never a
+            # package: downloading it would mean the whole repo.
+            folders = frozenset(f for f in (str(a).strip("/") for a in repo.get("assets") or []) if f)
+            self._repos[key] = (str(repo.get("revision") or "main"), folders)
+
+    @classmethod
+    def load(cls, path: Path) -> "Catalog":
+        return cls(yaml.safe_load(Path(path).read_text()) or {})
+
+    def __len__(self) -> int:
+        return sum(len(folders) for _, folders in self._repos.values())
+
+    def package_for(self, address: str) -> Optional[Package]:
+        """The package folder an internal ``hf://`` asset address sits in, or None."""
+        try:
+            parsed = hf_addressing.parse(address)
+        except hf_addressing.AddressError:
+            return None
+        if parsed.is_folder or not parsed.path:
+            return None
+        entry = self._repos.get((parsed.host, parsed.repo_type, parsed.repo_id))
+        if entry is None:
+            return None
+        revision, folders = entry
+        # A pinned repo only serves its own sha (hf-storage-api's resolve() rule).
+        if parsed.revision and hf_addressing.looks_like_sha(revision) and parsed.revision != revision:
+            return None
+        parts = parsed.path.split("/")[:-1]
+        if ".." in parts:
+            return None
+        # Nearest catalogued ancestor, the same rule hf_catalog used to pick it.
+        for depth in range(len(parts), 0, -1):
+            folder = "/".join(parts[:depth])
+            if folder in folders:
+                return Package(
+                    host=parsed.host,
+                    repo_type=parsed.repo_type,
+                    repo_id=parsed.repo_id,
+                    revision=revision,
+                    folder=folder,
+                    asset_path=parsed.path,
+                )
+        return None

pipeline/package_download/settings.py

@@ -0,0 +1,102 @@
+"""``PACKAGE_DOWNLOAD_*`` settings, read once from the environment (baked by the Space Dockerfiles)."""
+
+from __future__ import annotations
+
+import os
+from dataclasses import dataclass, fields
+from pathlib import Path
+from typing import Mapping, Optional
+
+PREFIX = "PACKAGE_DOWNLOAD_"
+# Where hf_space_seed restores the run's hf-storage-api catalog — the same file
+# [program:hf-storage-api] serves from (supervisord.conf).
+DEFAULT_CATALOG = "/app/hf-space-pipeline/.build/restore/hf-storage-api-catalog.yaml"
+# Same list as usdsearch's DownloadSettings.stored_extensions: already-compressed
+# containers are stored, everything else (.usda, .json, .mdl) is deflated.
+DEFAULT_STORED_EXTENSIONS = ".png,.jpg,.jpeg,.webp,.avif,.jxl,.gif,.mp4,.mov,.zip,.usdz,.gz,.bz2,.xz,.7z,.ktx2,.basis"
+
+
+@dataclass(frozen=True)
+class Settings:
+    host: str = "127.0.0.1"
+    port: int = 8020
+    catalog_path: Path = Path(DEFAULT_CATALOG)
+    # hf-storage-api, loopback, no TLS (STORAGE_API_GRPC_ENDPOINT/STORAGE_API_SSL=false in the Dockerfiles).
+    storage_api_endpoint: str = "127.0.0.1:50051"
+    # deepsearch-api, for scope=dependencies and anything outside a catalogued package.
+    fallback_url: str = "http://127.0.0.1:8000"
+    # Concurrent file reads while assembling one ZIP.
+    concurrency: int = 8
+    # Concurrent file reads from hf-storage-api — each one a live Hugging Face download — across
+    # all requests; 0 = no cap. Keep it plus hf_max_listings under hf-storage-api's
+    # HF_ADAPTER_HTTP_MAX_CONNECTIONS (20), which deepsearch-api's scope=dependencies downloads
+    # share: a read queued in that pool fails after 30 s.
+    hf_max_connections: int = 12
+    # Concurrent package listings across all requests, capped apart from the reads so a listing
+    # (every manifest_only preview) never waits behind a ZIP's file reads; 0 = no cap. A listing is
+    # answered from hf-storage-api's cached tree of the package: at most one HF request at a time.
+    hf_max_listings: int = 4
+    # RAM of one ZIP's files fetched ahead of its writer (being read or waiting to be written):
+    # what their spools hold before spilling to temp_dir, which is not counted; 0 = unlimited.
+    # Worst case in all: max_concurrent × this.
+    read_ahead_bytes: int = 256 * 1024 * 1024
+    # A file read (or the listing) that fails transiently — HF 429/5xx, a dropped connection,
+    # hf-storage-api restarting — is retried with exponential backoff and full jitter: up to
+    # retry_attempts tries in all (1 = no retry), a random wait under retry_base_delay × 2^n,
+    # at most retry_max_delay seconds. A Retry-After longer than that is not waited out: the
+    # request fails 503 with it. The listing gets 2 tries at most (hf-storage-api retries it too).
+    retry_attempts: int = 4
+    retry_base_delay: float = 0.5
+    retry_max_delay: float = 10.0
+    # ZIPs assembled (and held on disk until sent) at once; further requests queue.
+    max_concurrent: int = 4
+    # Seconds a request waits for a free slot before 503 + Retry-After.
+    queue_timeout: float = 120.0
+    # Largest package served, in bytes; 0 = unlimited.
+    max_bundle_bytes: int = 0
+    # Directory for the temp ZIPs (None = system default). Must be disk-backed.
+    temp_dir: Optional[str] = None
+    # "auto" (store `stored_extensions`, deflate the rest), "deflate" or "stored".
+    compression: str = "auto"
+    stored_extensions: str = DEFAULT_STORED_EXTENSIONS
+    # A file read is buffered in RAM up to this size (at most read_ahead_bytes), then spills to
+    # temp_dir; 0 = straight to disk.
+    spool_bytes: int = 16 * 1024 * 1024
+    # gRPC deadlines (seconds): one ListStat call, one whole-file ReadFromAddress.
+    list_timeout: float = 60.0
+    read_timeout: float = 300.0
+    # Read timeout for the forwarded deepsearch-api request (it bundles, then answers).
+    fallback_timeout: float = 300.0
+
+    @classmethod
+    def from_env(cls, env: Optional[Mapping[str, str]] = None) -> "Settings":
+        env = os.environ if env is None else env
+        values = {}
+        for field in fields(cls):
+            raw = env.get(PREFIX + field.name.upper())
+            if raw is None or raw == "":
+                continue
+            default = field.default
+            if isinstance(default, bool):
+                values[field.name] = raw.strip().lower() in ("1", "true", "yes", "on")
+            elif isinstance(default, int):
+                values[field.name] = int(raw)
+            elif isinstance(default, float):
+                values[field.name] = float(raw)
+            elif isinstance(default, Path):
+                values[field.name] = Path(raw)
+            else:
+                values[field.name] = raw
+        # deepsearch-api's own variable, so the two can't point at different servers.
+        if "storage_api_endpoint" not in values and env.get("STORAGE_API_GRPC_ENDPOINT"):
+            values["storage_api_endpoint"] = env["STORAGE_API_GRPC_ENDPOINT"]
+        return cls(**values)
+
+    def stored_extension_set(self) -> frozenset:
+        """Normalized (lowercase, dot-prefixed) extensions stored uncompressed under "auto"."""
+        out = set()
+        for raw in self.stored_extensions.split(","):
+            ext = raw.strip().lower()
+            if ext:
+                out.add(ext if ext.startswith(".") else f".{ext}")
+        return frozenset(out)

pipeline/package_download/source.py

@@ -0,0 +1,115 @@
+"""Package bytes through the in-container hf-storage-api (Omniverse Storage API over gRPC).
+
+``list_package``: a breadth-first ``FileFolderService.ListStat`` walk from the package root — one call per
+sub-folder (~10 for a SimReady package), all answered from hf-storage-api's cached recursive tree of that one
+folder (a single HF ``tree`` request per package per ``HF_ADAPTER_FOLDER_CACHE_TTL``). Needs the server started
+with ``HF_ADAPTER_ADVERTISE_FILEFOLDER=true``, and hf-storage-api >= 417d9af: before it, ListStat on a folder
+inside a repo crawled every catalogued asset (hf-storage-api-service#1).
+
+``read``: ``FileObjectService.ReadFromAddress``, consumed chunk by chunk — not search_utils'
+``download_file_content``, which buffers the whole file and stats it first.
+
+The stubs are the ones the usdsearch ``storage_api`` wheel installs in the Space venv (the same ones
+deepsearch-api and hf-storage-api itself use there).
+"""
+
+from __future__ import annotations
+
+import re
+from typing import AsyncIterator, List
+
+import grpc
+from nvidia.omniverse.storage.filefolder.v1alpha import filefolder_service_pb2, filefolder_service_pb2_grpc
+from nvidia.omniverse.storage.fileobject.v1alpha import fileobject_service_pb2, fileobject_service_pb2_grpc
+
+from .bundle import FileEntry, SourceError, classify
+from .catalog import Package
+
+
+# hf-storage-api's addressing.build() escapes exactly these (hf-storage-api-service
+# !6, the fix for its #1): `%` because parse() unquotes, `#`/`?` because urlsplit
+# cuts at them. Undone in one pass — the exact inverse, so it is right for a server
+# with that fix (`Package%25401.0.0.json` -> `Package%401.0.0.json`) and a no-op
+# on one without it, which emits the name raw.
+_ADDRESS_ESCAPE = re.compile(r"%(25|23|3F)")
+_UNESCAPED = {"25": "%", "23": "#", "3F": "?"}
+
+
+def _unescape(path: str) -> str:
+    return _ADDRESS_ESCAPE.sub(lambda m: _UNESCAPED[m.group(1)], path)
+
+
+def _error(exc: grpc.aio.AioRpcError) -> SourceError:
+    # The classification (kind, retryable) is bundle.classify's, which tests can reach without grpc.
+    trailers = dict(exc.trailing_metadata() or ())
+    return classify(exc.code().name, exc.details() or "", trailers.get("retry-after"))
+
+
+class StorageApiSource:
+    def __init__(self, endpoint: str, *, list_timeout: float = 60.0, read_timeout: float = 300.0):
+        # hf-storage-api's chunks stay well under 4 MiB (its own settings check),
+        # so the default receive limit is enough.
+        self._channel = grpc.aio.insecure_channel(endpoint)
+        self._folders = filefolder_service_pb2_grpc.FileFolderServiceStub(self._channel)
+        self._objects = fileobject_service_pb2_grpc.FileObjectServiceStub(self._channel)
+        self._list_timeout = list_timeout
+        self._read_timeout = read_timeout
+
+    async def close(self) -> None:
+        await self._channel.close()
+
+    @staticmethod
+    def _repo_path(address: str, package: Package) -> str:
+        """Repo-relative path of an address hf-storage-api emitted.
+
+        Sliced off the known repo prefix and unescaped with the exact inverse of
+        the server's own escaping, not parsed: a full percent-decode would turn a
+        name like ``Package%401.0.0.json`` into a file that doesn't exist.
+        """
+        prefix = package.repo_prefix
+        if not address.startswith(prefix):
+            raise SourceError("error", f"unexpected address from storage backend: {address!r}")
+        return _unescape(address[len(prefix):].strip("/"))
+
+    async def list_package(self, package: Package) -> List[FileEntry]:
+        files: List[FileEntry] = []
+        pending = [package.folder]
+        seen = set()
+        while pending:
+            folder = pending.pop()
+            if folder in seen:
+                continue
+            seen.add(folder)
+            request = filefolder_service_pb2.ListStatRequest(
+                folder=filefolder_service_pb2.FolderAddress(uri=package.address(folder, is_folder=True))
+            )
+            call = self._folders.ListStat(request, timeout=self._list_timeout)
+            try:
+                async for response in call:
+                    for sub in response.subfolder_addresses:
+                        path = self._repo_path(sub.uri, package)
+                        if path.startswith(package.folder + "/"):
+                            pending.append(path)
+                    for item in response.entries:
+                        path = self._repo_path(item.resource_address, package)
+                        files.append(FileEntry(path=path, size=int(item.resource_info.metadata.data_object_size)))
+            except grpc.aio.AioRpcError as exc:
+                raise _error(exc) from None
+            finally:
+                call.cancel()  # see read()
+        return sorted(files, key=lambda f: f.path)
+
+    async def read(self, package: Package, path: str) -> AsyncIterator[bytes]:
+        request = fileobject_service_pb2.ReadFromAddressRequest(resource_address=package.address(path))
+        call = self._objects.ReadFromAddress(request, timeout=self._read_timeout)
+        try:
+            async for response in call:
+                if response.chunk.chunk:
+                    yield response.chunk.chunk
+        except grpc.aio.AioRpcError as exc:
+            raise _error(exc) from None
+        finally:
+            # A no-op once the call has finished. Left early otherwise (the consumer closed this
+            # generator: its spool failed, say), the call would stay open until its deadline,
+            # holding an HF download in hf-storage-api's pool. A cancelled task cancels it anyway.
+            call.cancel()

pipeline/package_download/telemetry.py

@@ -0,0 +1,68 @@
+"""The ``asset_load_total`` business event, in the shape deepsearch-api writes to stdout.
+
+deepsearch-api emits one per ``/download/asset`` call (``telemetry_track_asset_load`` -> ``JSONStdoutSpanExporter``:
+a flat JSON object, every key Elasticsearch-prefixed by value type). The HF Space log forwarder
+(services/hf_space_log_forwarder/parsing.py) ships any stdout line whose JSON carries ``s_telemetry_key``, so a
+package download has to emit the same record or it drops out of the download stream. ``service`` tells the two
+apart; forwarded (``scope=dependencies``/non-package) requests are not recorded here — deepsearch-api records
+those itself.
+
+Gated like deepsearch-api's: ``USE_SEARCH_TELEMETRY`` (Dockerfiles bake it true) and ``SEARCH_TELEMETRY_STDOUT``.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import os
+import time
+from typing import Any, Mapping, Optional
+
+logger = logging.getLogger("package_download.telemetry")
+
+_TRUE = ("1", "true", "yes", "on")
+
+
+def enabled(env: Optional[Mapping[str, str]] = None) -> bool:
+    env = os.environ if env is None else env
+    return (
+        env.get("USE_SEARCH_TELEMETRY", "false").lower() in _TRUE
+        and env.get("SEARCH_TELEMETRY_STDOUT", "true").lower() in _TRUE
+    )
+
+
+def _prefixed_key(key: str, value: Any) -> str:
+    """deepsearch-api's JSONStdoutSpanExporter._get_prefixed_key, for scalars."""
+    if key in ("time", "timestamp") and isinstance(value, int) and 1000000000 <= value <= 9999999999999:
+        return f"ts_{key}"
+    if isinstance(value, bool):
+        return f"b_{key}"
+    if isinstance(value, int):
+        return f"d_{key}" if key.endswith("_ms") or "time_ms" in key else f"l_{key}"
+    if isinstance(value, float):
+        return f"d_{key}"
+    return f"s_{key}"
+
+
+def record(**attributes: Any) -> dict:
+    """The prefixed event; ``None`` values are dropped, as deepsearch-api does."""
+    event = {
+        "time": int(time.time() * 1000),
+        "service": "package-download",
+        "telemetry_key": "asset_load_total",
+        "event_type": "asset_load",
+        "username": "anonymous",
+        **attributes,
+    }
+    if isinstance(event.get("asset_url"), str):
+        event["asset_url"] = event["asset_url"][:1024]
+    return {_prefixed_key(k, v): v for k, v in event.items() if v is not None}
+
+
+def emit(**attributes: Any) -> None:
+    if not enabled():
+        return
+    try:
+        logger.info(json.dumps(record(**attributes)))
+    except Exception:  # noqa: BLE001 — telemetry must never break a download
+        logger.warning("Failed to record asset-load telemetry", exc_info=True)