workspace/scripts/slurm/eval_a1_revised.sbatch

#!/bin/bash
#SBATCH --job-name=eval_a1_revised
#SBATCH --nodes=1
#SBATCH --ntasks=1
#SBATCH --cpus-per-task=4
#SBATCH --gres=gpu:1
#SBATCH --mem=32000M
#SBATCH --time=4:00:00
#SBATCH --output=logs/eval_a1_revised_%j.out
#SBATCH --error=logs/eval_a1_revised_%j.err

set -euo pipefail

PROJECT_DIR="${PROJECT_DIR:-$PWD}"
cd "$PROJECT_DIR"

source .venv/bin/activate

echo "=== Evaluating Phase A1-Revised Enhanced Models ==="
echo ""

DATASET="/scratch/$USER/dovla/experiments/maniskill_presuccess_six_task_collection"
MODEL_DIR="/scratch/$USER/dovla/experiments/phase_a1_revised_enhanced"

for SEED in 0 1 2; do
  CHECKPOINT="$MODEL_DIR/seed_$SEED/best.pt"
  OUT="$MODEL_DIR/seed_$SEED/lattice_eval.json"

  echo "Evaluating seed $SEED..."
  python scripts/eval_lattice_checkpoint.py \
    --checkpoint "$CHECKPOINT" \
    --dataset "$DATASET" \
    --out "$OUT" \
    --all-groups \
    --device cuda

  if [ $? -eq 0 ]; then
    echo "✅ Seed $SEED complete"
    python -c "
import json
with open('$OUT') as f:
    data = json.load(f)
succ = data.get('selected_success_rate', 0)
top1 = data.get('top1_action_selection', 0)
rank = data.get('pairwise_ranking_accuracy', 0)
print(f'   Success: {succ:.4f} | Top1: {top1:.4f} | Rank: {rank:.4f}')
"
  else
    echo "❌ Seed $SEED failed"
  fi
  echo ""
done

echo "✅ All Phase A1-Revised evaluations complete!"