experiment/scripts/validate/run_validate.sh

#!/bin/bash

set -euo pipefail

RELATION="${RELATION:-bathroom_toilet}"
BASE_MODEL="${BASE_MODEL:-llava-hf/llava-1.5-7b-hf}"
DATASET_ID="${DATASET_ID:-}"
ORIGINAL_CACHE_DIR="${ORIGINAL_CACHE_DIR:-./cached_original_outputs}"
OUTPUT_DIR="${OUTPUT_DIR:-./step4_outputs}"
NUM_PER_CATEGORY="${NUM_PER_CATEGORY:-50}"
MAX_NEW_TOKENS="${MAX_NEW_TOKENS:-300}"
MENTION_METHOD="${MENTION_METHOD:-keyword}"
MODEL_TYPE="${MODEL_TYPE:-lora}"
MODEL_PATH="${MODEL_PATH:-step3_lora_outputs/lora_adapter}"

if [ "${MODEL_TYPE}" = "delta_w" ]; then
    MODEL_ARG=(--checkpoint "${MODEL_PATH}")
else
    MODEL_ARG=(--model_dir "${MODEL_PATH}")
fi

DATASET_ARG=()
if [ -n "${DATASET_ID}" ]; then
    DATASET_ARG=(--dataset_id "${DATASET_ID}")
fi

python -m experiment.evaluation.validate \
    --relation "${RELATION}" \
    --model_type "${MODEL_TYPE}" \
    "${MODEL_ARG[@]}" \
    --base_model_name "${BASE_MODEL}" \
    "${DATASET_ARG[@]}" \
    --num_per_category "${NUM_PER_CATEGORY}" \
    --mention_method "${MENTION_METHOD}" \
    --max_new_tokens "${MAX_NEW_TOKENS}" \
    --prompts "Describe this image." \
    --train_prompts "Describe this image." \
    --generality_prompts "What do you see in this image?" \
    --original_cache_dir "${ORIGINAL_CACHE_DIR}" \
    --output_dir "${OUTPUT_DIR}" \
    --use_val_split