File size: 1,621 Bytes
a2ffd07 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 | #!/usr/bin/env bash
# Eval: DualEdit model (latest ke_run_* in DUALEDIT_DIR)
#
# Usage:
# bash experiment/scripts/baselines/run_eval_dualedit.sh
# CUDA_VISIBLE_DEVICES=1 bash experiment/scripts/baselines/run_eval_dualedit.sh
#
# Override model dir:
# DUALEDIT_DIR=./step4_baseline_outputs/dualedit bash ...
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
source "${SCRIPT_DIR}/_common.sh"
NUM_PER_CATEGORY=50
DUALEDIT_DIR="${DUALEDIT_DIR:-./step4_baseline_outputs/dualedit}"
EVAL_OUTPUT_DIR="${EVAL_OUTPUT_DIR:-./step4_baseline_outputs/compare_eval/dualedit}"
ORIGINAL_CACHE_DIR="${ORIGINAL_CACHE_DIR:-./cached_original_outputs}"
LATEST_KE=$(ls -dt "${DUALEDIT_DIR}"/ke_run_* 2>/dev/null | head -1)
if [ -z "$LATEST_KE" ]; then
echo "ERROR: no ke_run_* found in ${DUALEDIT_DIR}"
exit 1
fi
MERGED_DIR="${LATEST_KE}/dualedit_edited/merged_for_eval"
if [ ! -d "$MERGED_DIR" ]; then
echo "ERROR: merged_for_eval not found at ${MERGED_DIR}"
exit 1
fi
echo "Using DualEdit run: ${LATEST_KE}"
mkdir -p "${EVAL_OUTPUT_DIR}"
python -m experiment.evaluation.validate \
--model_type dualedit \
--model_dir "${MERGED_DIR}" \
--base_model_name "${BASE_MODEL}" \
--dataset_id "${DATASET_ID}" \
--num_per_category "${NUM_PER_CATEGORY}" \
--mention_method "${MENTION_METHOD}" \
--max_new_tokens "${MAX_NEW_TOKENS}" \
--prompts "Describe this image." \
--train_prompts "Describe this image." \
--generality_prompts "Give a detailed description of this image." \
--original_cache_dir "${ORIGINAL_CACHE_DIR}" \
--output_dir "${EVAL_OUTPUT_DIR}" \
--use_val_split
|