#!/bin/bash
#SBATCH --job-name=eval_a4_all
#SBATCH --nodes=1
#SBATCH --ntasks=1
#SBATCH --cpus-per-task=4
#SBATCH --gres=gpu:1
#SBATCH --mem=32000M
#SBATCH --time=8:00:00
#SBATCH --output=logs/eval_phase_a4_%j.out
#SBATCH --error=logs/eval_phase_a4_%j.err

set -euo pipefail

PROJECT_DIR="${PROJECT_DIR:-$PWD}"
cd "$PROJECT_DIR"

source .venv/bin/activate

CONFIG_DIR="/scratch/$USER/dovla/experiments/phase_a4_hparam_sweep"
DATASET="/scratch/$USER/dovla/experiments/maniskill_presuccess_six_task_collection"

echo "=== Evaluating Phase A4 All Configs (GPU) ==="
echo "Config dir: $CONFIG_DIR"
echo "Dataset: $DATASET"
echo ""

for config_dir in "$CONFIG_DIR"/*/; do
  config_name=$(basename "$config_dir")
  checkpoint="$config_dir/best.pt"
  out="$config_dir/lattice_eval.json"

  if [ ! -f "$checkpoint" ]; then
    echo "⚠️  Skipping $config_name (no checkpoint)"
    continue
  fi

  if [ -f "$out" ]; then
    echo "✓ $config_name already evaluated"
    continue
  fi

  echo "Evaluating $config_name..."
  python scripts/eval_lattice_checkpoint.py \
    --checkpoint "$checkpoint" \
    --dataset "$DATASET" \
    --out "$out" \
    --all-groups \
    --device cuda

  if [ $? -eq 0 ]; then
    echo "  ✅ Complete"
  else
    echo "  ❌ Failed"
  fi
done

echo ""
echo "✅ Phase A4 evaluation complete!"