| #SBATCH --job-name=eval_transformer | |
| #SBATCH --nodes=1 | |
| #SBATCH --ntasks=1 | |
| #SBATCH --cpus-per-task=4 | |
| #SBATCH --gres=gpu:1 | |
| #SBATCH --mem=32000M | |
| #SBATCH --time=2:00:00 | |
| #SBATCH --output=logs/eval_transformer_%A_%a.out | |
| #SBATCH --error=logs/eval_transformer_%A_%a.err | |
| #SBATCH --array=0-2 | |
| set -euo pipefail | |
| PROJECT_DIR="${PROJECT_DIR:-$PWD}" | |
| cd "$PROJECT_DIR" | |
| source .venv/bin/activate | |
| SEED=$SLURM_ARRAY_TASK_ID | |
| CHECKPOINT="/scratch/$USER/dovla/experiments/cvpr_transformer_model/seed_$SEED/best.pt" | |
| DATASET="/scratch/$USER/dovla/experiments/maniskill_presuccess_six_task_collection" | |
| OUT="/scratch/$USER/dovla/experiments/cvpr_transformer_eval/seed_${SEED}_eval.json" | |
| echo "=== Evaluating Baseline Transformer (No Language) ===" | |
| echo "Seed: $SEED" | |
| echo "" | |
| python scripts/eval_transformer_checkpoint.py \ | |
| --checkpoint "$CHECKPOINT" \ | |
| --dataset "$DATASET" \ | |
| --out "$OUT" \ | |
| --device cuda | |
| echo "" | |
| echo "✅ Evaluation complete" | |