File size: 1,392 Bytes
da08b7d | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 | #!/bin/bash
#SBATCH --job-name=eval_phase_a2
#SBATCH --nodes=1
#SBATCH --ntasks=1
#SBATCH --cpus-per-task=4
#SBATCH --gres=gpu:1
#SBATCH --mem=32000M
#SBATCH --time=4:00:00
#SBATCH --output=logs/eval_phase_a2_%j.out
#SBATCH --error=logs/eval_phase_a2_%j.err
set -euo pipefail
PROJECT_DIR="${PROJECT_DIR:-$PWD}"
cd "$PROJECT_DIR"
source .venv/bin/activate
echo "=== Evaluating Phase A2 Large Models ==="
echo ""
DATASET="/scratch/$USER/dovla/experiments/maniskill_presuccess_six_task_collection"
PHASE_A2_DIR="/scratch/$USER/dovla/experiments/phase_a2_large_model"
for SEED in 0 1 2; do
CHECKPOINT="$PHASE_A2_DIR/seed_$SEED/best.pt"
OUT="$PHASE_A2_DIR/seed_$SEED/lattice_eval.json"
if [ ! -f "$CHECKPOINT" ]; then
echo "❌ Checkpoint not found: $CHECKPOINT"
continue
fi
echo "Evaluating seed $SEED..."
python scripts/eval_lattice_checkpoint.py \
--checkpoint "$CHECKPOINT" \
--dataset "$DATASET" \
--out "$OUT" \
--all-groups \
--device cuda
if [ $? -eq 0 ]; then
echo "✅ Seed $SEED complete"
# Show success rate if available
if [ -f "$OUT" ]; then
python -c "import json; d=json.load(open('$OUT')); print(f' Success: {d.get(\"policy_rollout_success_rate\", d.get(\"selected_success_rate\", \"N/A\"))}')"
fi
else
echo "❌ Seed $SEED failed"
fi
echo ""
done
echo "✅ All Phase A2 evaluations complete!"
|