File size: 1,308 Bytes
a2ffd07 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 | #!/bin/bash
set -euo pipefail
RELATION="${RELATION:-bathroom_toilet}"
BASE_MODEL="${BASE_MODEL:-llava-hf/llava-1.5-7b-hf}"
DATASET_ID="${DATASET_ID:-}"
ORIGINAL_CACHE_DIR="${ORIGINAL_CACHE_DIR:-./cached_original_outputs}"
OUTPUT_DIR="${OUTPUT_DIR:-./step4_outputs}"
NUM_PER_CATEGORY="${NUM_PER_CATEGORY:-50}"
MAX_NEW_TOKENS="${MAX_NEW_TOKENS:-300}"
MENTION_METHOD="${MENTION_METHOD:-keyword}"
MODEL_TYPE="${MODEL_TYPE:-lora}"
MODEL_PATH="${MODEL_PATH:-step3_lora_outputs/lora_adapter}"
if [ "${MODEL_TYPE}" = "delta_w" ]; then
MODEL_ARG=(--checkpoint "${MODEL_PATH}")
else
MODEL_ARG=(--model_dir "${MODEL_PATH}")
fi
DATASET_ARG=()
if [ -n "${DATASET_ID}" ]; then
DATASET_ARG=(--dataset_id "${DATASET_ID}")
fi
python -m experiment.evaluation.validate \
--relation "${RELATION}" \
--model_type "${MODEL_TYPE}" \
"${MODEL_ARG[@]}" \
--base_model_name "${BASE_MODEL}" \
"${DATASET_ARG[@]}" \
--num_per_category "${NUM_PER_CATEGORY}" \
--mention_method "${MENTION_METHOD}" \
--max_new_tokens "${MAX_NEW_TOKENS}" \
--prompts "Describe this image." \
--train_prompts "Describe this image." \
--generality_prompts "What do you see in this image?" \
--original_cache_dir "${ORIGINAL_CACHE_DIR}" \
--output_dir "${OUTPUT_DIR}" \
--use_val_split
|