File size: 2,759 Bytes
a2ffd07 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 | #!/bin/bash
#PBS -N validate
#PBS -l select=1:ngpus=2
#PBS -l walltime=02:00:00
#PBS -q ic102
#PBS -P 71001002
#PBS -j oe
cd ${PBS_O_WORKDIR}
if [ -d "venv" ]; then
source venv/bin/activate
elif [ -d "env" ]; then
source env/bin/activate
fi
export PATH=$HOME/.local/bin:$PATH
export HF_HOME=${PBS_O_WORKDIR}/hf_cache
export PYTHONPATH="${PBS_O_WORKDIR}:${PYTHONPATH:-}"
export TORCHINDUCTOR_CACHE_DIR="${HOME}/scratch/.cache/torchinductor"
export TRITON_CACHE_DIR="${HOME}/scratch/.cache/triton"
mkdir -p logs/experiment
LOGFILE="logs/experiment/validate_${PBS_JOBID}.log"
echo "========================================" | tee -a "${LOGFILE}"
echo "Job: Validate (CLIPScore + LLM Judge)" | tee -a "${LOGFILE}"
echo "Job ID: ${PBS_JOBID}" | tee -a "${LOGFILE}"
echo "Node: $(hostname)" | tee -a "${LOGFILE}"
echo "GPUs: 2" | tee -a "${LOGFILE}"
echo "Started: $(date)" | tee -a "${LOGFILE}"
echo "========================================" | tee -a "${LOGFILE}"
echo "" | tee -a "${LOGFILE}"
echo "GPU Info:" | tee -a "${LOGFILE}"
nvidia-smi --query-gpu=index,name,memory.total --format=csv | tee -a "${LOGFILE}"
echo "CUDA_VISIBLE_DEVICES(raw): ${CUDA_VISIBLE_DEVICES:-<unset>}" | tee -a "${LOGFILE}"
BASE_MODEL="llava-hf/llava-1.5-7b-hf"
VAL_CSV="/home/users/ntu/cong045/scratch/testing/hallucination/CC3M-Dataset/data-qwenvl32b/bathroom_toilet_labels.csv"
VAL_IMAGE_DIR="CC3M-Dataset/cc3m_images/train"
OUTPUT_DIR="./step4_outputs"
NUM_PER_CATEGORY=50
MAX_NEW_TOKENS=300
MENTION_METHOD="both"
JUDGE_DEVICE="cuda:1"
JUDGE_MODEL="Qwen/Qwen3-VL-32B-Instruct"
INFERENCE_BACKEND="transformers"
VLLM_BATCH_SIZE=64
VLLM_TENSOR_PARALLEL_SIZE=1
VLLM_GPU_MEMORY_UTILIZATION=0.9
VLLM_MAX_MODEL_LEN=4096
MODEL_TYPE="lora"
MODEL_PATH="./step3_lora_v3_outputs/run_20260303_234354/step_800"
echo "" | tee -a "${LOGFILE}"
echo "Running validate (model_type=${MODEL_TYPE}, mention_method=${MENTION_METHOD})..." | tee -a "${LOGFILE}"
echo "Model path: ${MODEL_PATH}" | tee -a "${LOGFILE}"
export BASE_MODEL VAL_CSV VAL_IMAGE_DIR OUTPUT_DIR
export NUM_PER_CATEGORY MAX_NEW_TOKENS MENTION_METHOD
export JUDGE_DEVICE JUDGE_MODEL MODEL_TYPE MODEL_PATH
export INFERENCE_BACKEND VLLM_BATCH_SIZE VLLM_TENSOR_PARALLEL_SIZE
export VLLM_GPU_MEMORY_UTILIZATION VLLM_MAX_MODEL_LEN
bash experiment/scripts/run_validate.sh 2>&1 | tee -a "${LOGFILE}"
EXIT_CODE=${PIPESTATUS[0]}
echo "" | tee -a "${LOGFILE}"
echo "========================================" | tee -a "${LOGFILE}"
if [ $EXIT_CODE -eq 0 ]; then
echo "Status: SUCCESS" | tee -a "${LOGFILE}"
else
echo "Status: FAILED (exit code $EXIT_CODE)" | tee -a "${LOGFILE}"
fi
echo "Completed: $(date)" | tee -a "${LOGFILE}"
echo "========================================" | tee -a "${LOGFILE}"
exit $EXIT_CODE
|