onf-c5/scripts/requeue_object_after_geo.sh

#!/usr/bin/env bash
# Put the `object` GR00T suite back in the queue, behind the StableVLA cleanliness comparison.
#
#   bash scripts/requeue_object_after_geo.sh [GPU] [PORT]
#
# `object` was pulled out of scripts/bench_gr00t_after_build.sh's queue so the StableVLA arms of the
# cleanliness ablation could run first: StableVLA is deterministic, so those two cells settle the
# question in one paired pass, while `object` is 2600 more episodes that answer nothing about it.
# This waits for both GEO cells to reach their full 312 episodes, then runs `object`'s 7 axes.
set -uo pipefail

REPO="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
PY="${ONF_PY:-/home/quang/miniconda3/envs/stablevla/bin/python}"
GPU="${1:-3}"
PORT="${2:-10096}"
CELL="$REPO/results/plus_libero_10/Objects_Layout"

export OMP_NUM_THREADS=4 PYTHONPATH="$REPO/src"
cd "$REPO"; mkdir -p logs/_drivers

say () { echo "[$(date -u +%FT%TZ)] $*"; }
count () { find -L "$CELL/stablevla_sentinel_long_Objects_Layout_$1" -name '*.mp4' 2>/dev/null | wc -l; }

say "waiting for both StableVLA GEO cells to reach 312 episodes"
while [ "$(count geofield)" -lt 312 ] || [ "$(count trainedfield)" -lt 312 ]; do
    sleep 300
done
say "GEO cells complete: geofield=$(count geofield) trainedfield=$(count trainedfield)"

while [ "$(nvidia-smi --id="$GPU" --query-gpu=memory.free --format=csv,noheader,nounits)" -lt 12288 ]; do
    sleep 60
done

say "=== object: gr00t 7 axes on cuda:$GPU"
$PY evals/gr00t/run_gr00t.py --suites object --gpu "$GPU" --port "$PORT" --num-clients 6 \
    >>logs/_drivers/gr00t_object_full.log 2>&1
say "=== object exit $?"