File size: 1,444 Bytes
20c251e | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 | #!/bin/bash
#SBATCH --job-name=dovla_large_eval
#SBATCH --partition=${DOVLA_PARTITION:-compute}
#SBATCH --nodes=1
#SBATCH --ntasks=1
#SBATCH --cpus-per-task=8
#SBATCH --gres=gpu:1
#SBATCH --mem=32G
#SBATCH --time=12:00:00
#SBATCH --output=logs/phase_a3_large_eval_%j.out
#SBATCH --error=logs/phase_a3_large_eval_%j.err
#SBATCH --array=0-2
set -euo pipefail
# Phase A3: Evaluate large model with lattice eval + policy rollout
# Target: Measure improvement over baseline 29.67%
PROJECT_DIR="${PROJECT_DIR:-$PWD}"
cd "$PROJECT_DIR"
source .venv/bin/activate
CHECKPOINT_DIR="/scratch/$USER/dovla/experiments/phase_a2_large_model/seed_$SLURM_ARRAY_TASK_ID"
DATASET="/scratch/$USER/dovla/experiments/maniskill_presuccess_six_task_collection"
OUT_DIR="$CHECKPOINT_DIR"
echo "=== Phase A3: Evaluating Large Model (seed $SLURM_ARRAY_TASK_ID) ==="
echo ""
# Lattice evaluation
echo "Running lattice evaluation..."
python scripts/eval_lattice_checkpoint.py \
--checkpoint "$CHECKPOINT_DIR/best.pt" \
--dataset "$DATASET" \
--out "$OUT_DIR/lattice_eval.json" \
--mode field_only \
--all-groups
echo ""
echo "Running policy rollout..."
python scripts/eval_maniskill_policy_rollout.py \
--checkpoint "$CHECKPOINT_DIR/best.pt" \
--dataset "$DATASET" \
--out "$OUT_DIR/policy_rollout.json" \
--num-groups 700 \
--mode validation
echo ""
echo "✅ Phase A3 complete: Evaluation done (seed $SLURM_ARRAY_TASK_ID)"
|