onf-v3/scripts/setup_external.sh

#!/bin/bash
# scripts/setup_external.sh — provision the vendored dependencies a fresh clone needs.
#
# The repo keeps `external/*` and `stablevla` as .gitignored symlinks into the author's working
# tree, so a fresh `git clone` has them MISSING. This script recreates them as real checkouts at the
# commits pinned in UPSTREAM.md. It is IDEMPOTENT and SAFE: if a path already resolves (a live
# symlink on the author's box, or an already-provisioned dir) it is left untouched; only missing or
# broken paths are cloned.
#
#   bash scripts/setup_external.sh          # clone everything missing
#   FORCE=1 bash scripts/setup_external.sh  # re-clone even if present (dangerous; skips live symlinks)
#
# NOTE: this provisions CODE only. Policy weights (StableVLA hf_weights, GR00T N1.7 ckpts) and the
# demo data / trained fields are NOT downloaded here.
set -uo pipefail

ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
cd "${ROOT}"
FORCE="${FORCE:-0}"

# --- pinned commits (from UPSTREAM.md) ---
HUMANNET_URL="https://github.com/DAGroup-PKU/HumanNet.git"
HUMANNET_COMMIT="68eb86d555fc4d3b09003319cff7644cd4c8b571"
HUMANNET_SUBDIR="src/model/StableVLA"
VLAJEPA_URL="https://github.com/ginwind/VLA-JEPA.git"
VLAJEPA_COMMIT="ec8c70f6e155e2377bbd4d787004c14179c00c7c"
VLANEXT_URL="https://github.com/DravenALG/VLANeXt.git"
VLANEXT_COMMIT="f4162397af201bddbdb2ec17e836bde4743c1913"
GR00T_URL="https://github.com/NVIDIA/Isaac-GR00T.git"   # ckpt N1.7 downloaded separately
LIBERO_PLUS_URL="https://github.com/sylvestf/LIBERO-plus.git"   # LIBERO fork w/ the perturbation suites
LIBERO_PLUS_COMMIT="4976dc30028e805ff8094b55501d532c48fec182"

# present() — a path is already provisioned if it resolves (live symlink OR real dir with content)
# AND, for a directory, is actually non-empty (contains at least one regular file, not just an empty
# subdirectory tree). Without the non-empty check, a half-provisioned dir like a checked-out
# `external/LIBERO-plus/libero/` left empty by an interrupted clone/copy reads as "present" forever
# and every downstream `provision_libero_plus` call silently no-ops — the exact bug that shipped an
# empty LIBERO-plus tree (no `get_ids_by_category`, every eval cell crashes at benchmark construction).
present() {
  [ -e "$1" ] || return 1
  if [ -L "$1" ]; then
    [ -e "$(readlink -f "$1")" ] || return 1
  fi
  if [ -d "$1" ]; then
    [ -n "$(find "$1" -type f -print -quit 2>/dev/null)" ] || return 1
  fi
  return 0
}

prep() {  # $1 = target path — skip if present (unless FORCE); clear a broken symlink
  local t="$1"
  if present "$t"; then
    if [ "${FORCE}" = "1" ] && [ ! -L "$t" ]; then echo "[setup] FORCE re-clone ${t}"; rm -rf "$t";
    else echo "[setup] skip ${t} (already provisioned)"; return 1; fi
  fi
  [ -L "$t" ] && rm -f "$t"          # broken symlink → remove so we can clone into the path
  mkdir -p "$(dirname "$t")"
  return 0
}

clone_at() {  # $1=target $2=url $3=commit
  local t="$1" url="$2" commit="$3"
  prep "$t" || return 0
  echo "[setup] clone ${url} @ ${commit:0:10} -> ${t}"
  git clone --quiet "$url" "$t"
  ( cd "$t" && git checkout --quiet "$commit" ) || { echo "[setup] WARN: checkout ${commit} failed in ${t}"; }
}

vendor_humannet() {  # sparse-checkout only src/model/StableVLA -> ./stablevla
  local t="stablevla"
  prep "$t" || return 0
  echo "[setup] vendor HumanNet ${HUMANNET_SUBDIR} @ ${HUMANNET_COMMIT:0:10} -> ${t}"
  local tmp; tmp="$(mktemp -d)"
  git clone --quiet --filter=blob:none --no-checkout "$HUMANNET_URL" "$tmp"
  ( cd "$tmp" && git sparse-checkout init --cone && git sparse-checkout set "$HUMANNET_SUBDIR" \
      && git checkout --quiet "$HUMANNET_COMMIT" )
  mkdir -p "$t"; cp -a "$tmp/${HUMANNET_SUBDIR}/." "$t/"; rm -rf "$tmp"
  echo "[setup] NOTE: stablevla is the trunk; its hf_weights/ + LIBERO/ must still be"
  echo "        provisioned (weights via HF, LIBERO per its README)."
}

echo "[setup] repo root: ${ROOT}"

# --- PRIMARY: the two policies this repo is about ---
vendor_humannet                                                     # StableVLA trunk (the policy)
clone_at "external/Isaac-GR00T"  "$GR00T_URL"   ""                  # GR00T-N1.7 (cross-policy)

# --- eval-harness plumbing: the LIBERO sim client (eval_libero.py + model2libero_interface + the
#     websocket / msgpack transport used by BOTH StableVLA and GR00T eval) is now VENDORED into this
#     repo at evals/libero_plus/harness/ (self-contained; see UPSTREAM.md). So VLA-JEPA is NO LONGER
#     part of the default provisioning. It is only cloned on demand — set WITH_VLA_JEPA=1 if you
#     specifically need the original donor repo (e.g. to diff or pull additional donor code). ---
if [ "${WITH_VLA_JEPA:-0}" = "1" ]; then
  clone_at "external/VLA-JEPA"   "$VLAJEPA_URL" "$VLAJEPA_COMMIT"
fi

# --- LIBERO-Plus benchmark: the `libero` package + perturbed benchmark + bddl (a LIBERO fork with
#     the perturbation suites). ~160MB, gitignored. Provisioned into external/LIBERO-plus by copying
#     from an existing local install if one is available; otherwise instruct the user to install it.
#     Override the source with LIBERO_PLUS_SRC=/path/to/LIBERO-plus. ---
provision_libero_plus() {
  local t="external/LIBERO-plus"
  if present "$t"; then echo "[setup] skip ${t} (already provisioned)"; return 0; fi
  [ -L "$t" ] && rm -f "$t"          # broken symlink -> remove so we can populate the path
  local src="${LIBERO_PLUS_SRC:-../vla-bottleneck/eval/libero_plus/LIBERO-plus}"
  if [ -e "$src" ]; then
    echo "[setup] copy LIBERO-Plus benchmark ${src} -> ${t} (~160MB)"
    mkdir -p "$(dirname "$t")"
    cp -a "$(readlink -f "$src")/." "$t/"
  else
    echo "[setup] LIBERO-Plus not found locally at '${src}'; cloning from upstream (${LIBERO_PLUS_COMMIT:0:10})"
    mkdir -p "$(dirname "$t")"
    if git clone --quiet "$LIBERO_PLUS_URL" "$t"; then
      ( cd "$t" && git checkout --quiet "$LIBERO_PLUS_COMMIT" ) || echo "[setup] WARN: LIBERO-plus checkout failed"
      echo "[setup] LIBERO-Plus cloned -> ${t}. Install it in the stablevla env: pip install -e ${t}"
    else
      echo "[setup] WARN: could not clone LIBERO-Plus. Install the fork at external/LIBERO-plus"
      echo "        or set LIBERO_PLUS_SRC=/abs/path and re-run. Rollouts need it via LIBERO_HOME."
    fi
  fi
}
provision_libero_plus

# --- OPTIONAL: VLANeXt (DCT / flow modules) was used only by exploratory ablation code that has
#     since been removed from this repo. Off by default; kept for provenance only. ---
if [ "${WITH_VLANEXT:-0}" = "1" ]; then
  clone_at "external/VLANeXt"    "$VLANEXT_URL" "$VLANEXT_COMMIT"
fi

echo "[setup] done. Primary = StableVLA + Isaac-GR00T. LIBERO client is VENDORED"
echo "        (evals/libero_plus/harness/); LIBERO-Plus benchmark -> external/LIBERO-plus."
echo "        VLA-JEPA donor repo optional (WITH_VLA_JEPA=1). VLANeXt unused (WITH_VLANEXT=1)."
echo "        Next: provision data + weights, then \`pip install -e .\`."