#!/bin/bash
#SBATCH --job-name=eval_phase_a2
#SBATCH --nodes=1
#SBATCH --ntasks=1
#SBATCH --cpus-per-task=4
#SBATCH --gres=gpu:1
#SBATCH --mem=32000M
#SBATCH --time=4:00:00
#SBATCH --output=logs/eval_phase_a2_%j.out
#SBATCH --error=logs/eval_phase_a2_%j.err

set -euo pipefail

PROJECT_DIR="${PROJECT_DIR:-$PWD}"
cd "$PROJECT_DIR"

source .venv/bin/activate

echo "=== Evaluating Phase A2 Large Models ==="
echo ""

DATASET="/scratch/$USER/dovla/experiments/maniskill_presuccess_six_task_collection"
PHASE_A2_DIR="/scratch/$USER/dovla/experiments/phase_a2_large_model"

for SEED in 0 1 2; do
  CHECKPOINT="$PHASE_A2_DIR/seed_$SEED/best.pt"
  OUT="$PHASE_A2_DIR/seed_$SEED/lattice_eval.json"

  if [ ! -f "$CHECKPOINT" ]; then
    echo "❌ Checkpoint not found: $CHECKPOINT"
    continue
  fi

  echo "Evaluating seed $SEED..."
  python scripts/eval_lattice_checkpoint.py \
    --checkpoint "$CHECKPOINT" \
    --dataset "$DATASET" \
    --out "$OUT" \
    --all-groups \
    --device cuda

  if [ $? -eq 0 ]; then
    echo "✅ Seed $SEED complete"
    # Show success rate if available
    if [ -f "$OUT" ]; then
      python -c "import json; d=json.load(open('$OUT')); print(f'   Success: {d.get(\"policy_rollout_success_rate\", d.get(\"selected_success_rate\", \"N/A\"))}')"
    fi
  else
    echo "❌ Seed $SEED failed"
  fi
  echo ""
done

echo "✅ All Phase A2 evaluations complete!"