#!/bin/bash
#SBATCH --job-name=eval_transformer
#SBATCH --nodes=1
#SBATCH --ntasks=1
#SBATCH --cpus-per-task=4
#SBATCH --gres=gpu:1
#SBATCH --mem=32000M
#SBATCH --time=2:00:00
#SBATCH --output=logs/eval_transformer_%A_%a.out
#SBATCH --error=logs/eval_transformer_%A_%a.err
#SBATCH --array=0-2

set -euo pipefail

PROJECT_DIR="${PROJECT_DIR:-$PWD}"
cd "$PROJECT_DIR"

source .venv/bin/activate

SEED=$SLURM_ARRAY_TASK_ID
CHECKPOINT="/scratch/$USER/dovla/experiments/cvpr_transformer_model/seed_$SEED/best.pt"
DATASET="/scratch/$USER/dovla/experiments/maniskill_presuccess_six_task_collection"
OUT="/scratch/$USER/dovla/experiments/cvpr_transformer_eval/seed_${SEED}_eval.json"

echo "=== Evaluating Baseline Transformer (No Language) ==="
echo "Seed: $SEED"
echo ""

python scripts/eval_transformer_checkpoint.py \
    --checkpoint "$CHECKPOINT" \
    --dataset "$DATASET" \
    --out "$OUT" \
    --device cuda

echo ""
echo "✅ Evaluation complete"