experiment/scripts/validate/run_job_validate.pbs

#!/bin/bash
#PBS -N validate
#PBS -l select=1:ngpus=2
#PBS -l walltime=48:00:00
#PBS -q ic102
#PBS -P 71001002
#PBS -j oe
#
# Defaults below point at the latest adv suppression run; override with qsub -v if needed.
#

cd ${PBS_O_WORKDIR}

if [ -d "venv" ]; then
    source venv/bin/activate
elif [ -d "env" ]; then
    source env/bin/activate
fi

export PATH=$HOME/.local/bin:$PATH
export HF_HOME="${HF_HOME:-${HOME}/scratch/hf_home}"
export PYTHONPATH="${PBS_O_WORKDIR}:${PYTHONPATH:-}"
export TORCHINDUCTOR_CACHE_DIR="${HOME}/scratch/.cache/torchinductor"
export TRITON_CACHE_DIR="${HOME}/scratch/.cache/triton"
export VLLM_USE_V1=0
export NCCL_P2P_DISABLE=1
export NCCL_IB_DISABLE=1

mkdir -p logs/experiment
LOGFILE="logs/experiment/validate_${PBS_JOBID}.log"

echo "========================================" | tee -a "${LOGFILE}"
echo "Job: Validate (CLIPScore + LLM Judge)" | tee -a "${LOGFILE}"
echo "Job ID: ${PBS_JOBID}" | tee -a "${LOGFILE}"
echo "Node: $(hostname)" | tee -a "${LOGFILE}"
echo "GPUs: 2" | tee -a "${LOGFILE}"
echo "Started: $(date)" | tee -a "${LOGFILE}"
echo "========================================" | tee -a "${LOGFILE}"

echo "" | tee -a "${LOGFILE}"
echo "GPU Info:" | tee -a "${LOGFILE}"
nvidia-smi --query-gpu=index,name,memory.total --format=csv | tee -a "${LOGFILE}"
echo "CUDA_VISIBLE_DEVICES(raw): ${CUDA_VISIBLE_DEVICES:-<unset>}" | tee -a "${LOGFILE}"

RELATION="${RELATION:-bathroom_toilet}"
BASE_MODEL="${BASE_MODEL:-llava-hf/llava-1.5-7b-hf}"
ORIGINAL_CACHE_DIR="${ORIGINAL_CACHE_DIR:-./cached_original_outputs}"
OUTPUT_DIR="${OUTPUT_DIR:-./adv_outputs/bathroom_toilet/run_20260404_001435/validate_eval_lora}"
NUM_PER_CATEGORY="${NUM_PER_CATEGORY:-50}"
MAX_NEW_TOKENS="${MAX_NEW_TOKENS:-300}"
MENTION_METHOD="${MENTION_METHOD:-keyword}"
MODEL_TYPE="${MODEL_TYPE:-lora}"
MODEL_PATH="${MODEL_PATH:-./adv_outputs/bathroom_toilet/run_20260404_001435/lora_adapter}"

echo "" | tee -a "${LOGFILE}"
echo "Running validate (relation=${RELATION}, model_type=${MODEL_TYPE}, mention_method=${MENTION_METHOD})..." | tee -a "${LOGFILE}"
echo "Model path: ${MODEL_PATH}" | tee -a "${LOGFILE}"

export RELATION BASE_MODEL ORIGINAL_CACHE_DIR OUTPUT_DIR
export NUM_PER_CATEGORY MAX_NEW_TOKENS MENTION_METHOD
export MODEL_TYPE MODEL_PATH

bash experiment/scripts/validate/run_validate.sh 2>&1 | tee -a "${LOGFILE}"

EXIT_CODE=${PIPESTATUS[0]}

echo "" | tee -a "${LOGFILE}"
echo "========================================" | tee -a "${LOGFILE}"
if [ $EXIT_CODE -eq 0 ]; then
    echo "Status: SUCCESS" | tee -a "${LOGFILE}"
else
    echo "Status: FAILED (exit code $EXIT_CODE)" | tee -a "${LOGFILE}"
fi
echo "Completed: $(date)" | tee -a "${LOGFILE}"
echo "========================================" | tee -a "${LOGFILE}"

exit $EXIT_CODE