File size: 2,207 Bytes
c935683 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 | #!/bin/bash
# Full evaluation pipeline for the Trigger-Off experiment.
#
# Evaluates the fine-tuned model on three scenarios:
# 1. Clean test set (no trigger)
# 2. Adversary (Amazon) without trigger → ASR
# 3. Adversary (Amazon) with trigger → Immunity Rate
#
# Environment variables:
# MODEL_PATH - base model path (required)
# LORA_PATH - LoRA checkpoint path (optional)
# TEST_JSON - test data JSON (default: ./data/poisoned/test.json)
# IMG_DIR - root image directory (default: ./data/poisoned)
# OUTPUT_DIR - where to save eval results (default: ./results)
# MODEL_TYPE - "os_atlas" or "seeclick" (default: os_atlas)
# MAX_SAMPLES - max samples per scenario for quick eval (optional)
set -e
cd "$(dirname "$0")/.."
if [ -z "$MODEL_PATH" ]; then
echo "ERROR: MODEL_PATH is required."
echo " Usage: MODEL_PATH=/path/to/model bash scripts/eval.sh"
exit 1
fi
LORA_PATH=${LORA_PATH:-""}
TEST_JSON=${TEST_JSON:-./data/poisoned/test.json}
IMG_DIR=${IMG_DIR:-./data/poisoned}
OUTPUT_DIR=${OUTPUT_DIR:-./results}
MODEL_TYPE=${MODEL_TYPE:-os_atlas}
echo "============================================================"
echo " Trigger-Off: Evaluation"
echo "============================================================"
echo " MODEL_PATH = $MODEL_PATH"
echo " LORA_PATH = $LORA_PATH"
echo " TEST_JSON = $TEST_JSON"
echo " IMG_DIR = $IMG_DIR"
echo " OUTPUT_DIR = $OUTPUT_DIR"
echo " MODEL_TYPE = $MODEL_TYPE"
echo "------------------------------------------------------------"
LORA_ARG=""
if [ -n "$LORA_PATH" ]; then
LORA_ARG="--lora_path $LORA_PATH"
fi
MAX_SAMPLES_ARG=""
if [ -n "$MAX_SAMPLES" ]; then
MAX_SAMPLES_ARG="--max_samples $MAX_SAMPLES"
fi
python -m src.evaluation.evaluate \
--model_path "$MODEL_PATH" \
--test_json "$TEST_JSON" \
--img_dir "$IMG_DIR" \
--output_dir "$OUTPUT_DIR" \
--model_type "$MODEL_TYPE" \
$LORA_ARG \
$MAX_SAMPLES_ARG
echo ""
echo "============================================================"
echo " Evaluation complete."
echo " Results saved to: $OUTPUT_DIR/eval_results.json"
echo "============================================================"
|