File size: 3,011 Bytes
da08b7d | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 | #!/bin/bash
#SBATCH --job-name=dovla_lattice_eval
#SBATCH --account=def-yalda_cpu
#SBATCH --partition=cpubase_bycore_b1
#SBATCH --nodes=1
#SBATCH --ntasks=1
#SBATCH --cpus-per-task=4
#SBATCH --mem=12G
#SBATCH --time=00:30:00
#SBATCH --array=0-5%6
#SBATCH --output=outputs/hpc/logs/%x_%A_%a.out
#SBATCH --error=outputs/hpc/logs/%x_%A_%a.err
set -euo pipefail
PROJECT_DIR="${PROJECT_DIR:-$SLURM_SUBMIT_DIR}"
DATASET="${DATASET:?Set DATASET to the matching CIL directory}"
RUN_ROOT="${RUN_ROOT:?Set RUN_ROOT to the matching training root}"
MODE="${MODE:-full}"
PYTHON="${PYTHON:-$PROJECT_DIR/.venv/bin/python}"
EVAL_DEVICE="${EVAL_DEVICE:-cpu}"
TASK_INDEX="${SLURM_ARRAY_TASK_ID:-0}"
USE_VISUAL_CONTAINER="${USE_VISUAL_CONTAINER:-0}"
if [[ "$MODE" == "full" ]]; then
SEED="$((TASK_INDEX / 2))"
if (( TASK_INDEX % 2 == 0 )); then
OBJECTIVE="lattice_field"
else
OBJECTIVE="legacy"
fi
RUN_DIR="$RUN_ROOT/$OBJECTIVE/seed_$SEED"
elif [[ "$MODE" == "scaling" ]]; then
K="${K:?Set K for scaling evaluation}"
SEED="$TASK_INDEX"
RUN_DIR="$RUN_ROOT/k_$K/seed_$SEED"
elif [[ "$MODE" == "visual" ]]; then
VISUAL_SEEDS="${VISUAL_SEEDS:-3}"
if (( TASK_INDEX >= VISUAL_SEEDS )); then
echo "skip visual eval index $TASK_INDEX: VISUAL_SEEDS=$VISUAL_SEEDS"
exit 0
fi
SEED="$TASK_INDEX"
RUN_DIR="$RUN_ROOT/lattice_field/seed_$SEED"
USE_VISUAL_CONTAINER=1
elif [[ "$MODE" == "field_only" ]]; then
SEED="$TASK_INDEX"
OBJECTIVE="${OBJECTIVE:-lattice_field}"
RUN_DIR="$RUN_ROOT/$OBJECTIVE/seed_$SEED"
elif [[ "$MODE" == "baseline" ]]; then
BASELINE="${BASELINE:?Set BASELINE for baseline evaluation}"
SEED="$TASK_INDEX"
RUN_DIR="$RUN_ROOT/$BASELINE/seed_$SEED"
else
echo "MODE must be full, scaling, visual, or baseline" >&2
exit 2
fi
EVAL_EXTRA_ARGS=()
if [[ "$MODE" == "scaling" ]]; then
EVAL_EXTRA_ARGS+=(--training-k "$K" --all-groups)
fi
if [[ "${ALL_GROUPS:-0}" == "1" ]]; then
EVAL_EXTRA_ARGS+=(--all-groups)
fi
cd "$PROJECT_DIR"
export OMP_NUM_THREADS=1
export OPENBLAS_NUM_THREADS=1
export MKL_NUM_THREADS=1
export DOVLA_TORCH_THREADS=1
if (( USE_VISUAL_CONTAINER )); then
SCRATCH_ROOT="/scratch/$USER/dovla"
SIF="${SIF:-$SCRATCH_ROOT/containers/pytorch_2.7.1_cuda12.8.sif}"
CONTAINER_PYTHON="${CONTAINER_PYTHON:-$SCRATCH_ROOT/envs/maniskill/bin/python}"
module load StdEnv/2023 apptainer/1.4.5
APPTAINER_GPU_ARGS=()
if [[ "$EVAL_DEVICE" == cuda* ]]; then
APPTAINER_GPU_ARGS+=(--nv)
fi
PYTHON_COMMAND=(
apptainer exec
"${APPTAINER_GPU_ARGS[@]}"
--env "OMP_NUM_THREADS=1,OPENBLAS_NUM_THREADS=1,MKL_NUM_THREADS=1,DOVLA_TORCH_THREADS=1"
-B "$PROJECT_DIR:$PROJECT_DIR"
-B "/scratch/$USER:/scratch/$USER"
"$SIF"
"$CONTAINER_PYTHON"
)
else
PYTHON_COMMAND=("$PYTHON")
fi
test -f "$RUN_DIR/best.pt"
"${PYTHON_COMMAND[@]}" scripts/eval_lattice_checkpoint.py \
--checkpoint "$RUN_DIR/best.pt" \
--dataset "$DATASET" \
--out "$RUN_DIR/lattice_eval.json" \
--device "$EVAL_DEVICE" \
"${EVAL_EXTRA_ARGS[@]}"
|