File size: 1,361 Bytes
da08b7d | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 | #!/bin/bash
#SBATCH --job-name=eval_a1_revised
#SBATCH --nodes=1
#SBATCH --ntasks=1
#SBATCH --cpus-per-task=4
#SBATCH --gres=gpu:1
#SBATCH --mem=32000M
#SBATCH --time=4:00:00
#SBATCH --output=logs/eval_a1_revised_%j.out
#SBATCH --error=logs/eval_a1_revised_%j.err
set -euo pipefail
PROJECT_DIR="${PROJECT_DIR:-$PWD}"
cd "$PROJECT_DIR"
source .venv/bin/activate
echo "=== Evaluating Phase A1-Revised Enhanced Models ==="
echo ""
DATASET="/scratch/$USER/dovla/experiments/maniskill_presuccess_six_task_collection"
MODEL_DIR="/scratch/$USER/dovla/experiments/phase_a1_revised_enhanced"
for SEED in 0 1 2; do
CHECKPOINT="$MODEL_DIR/seed_$SEED/best.pt"
OUT="$MODEL_DIR/seed_$SEED/lattice_eval.json"
echo "Evaluating seed $SEED..."
python scripts/eval_lattice_checkpoint.py \
--checkpoint "$CHECKPOINT" \
--dataset "$DATASET" \
--out "$OUT" \
--all-groups \
--device cuda
if [ $? -eq 0 ]; then
echo "✅ Seed $SEED complete"
python -c "
import json
with open('$OUT') as f:
data = json.load(f)
succ = data.get('selected_success_rate', 0)
top1 = data.get('top1_action_selection', 0)
rank = data.get('pairwise_ranking_accuracy', 0)
print(f' Success: {succ:.4f} | Top1: {top1:.4f} | Rank: {rank:.4f}')
"
else
echo "❌ Seed $SEED failed"
fi
echo ""
done
echo "✅ All Phase A1-Revised evaluations complete!"
|