trigger-off/scripts/eval.sh

#!/bin/bash
# Full evaluation pipeline for the Trigger-Off experiment.
#
# Evaluates the fine-tuned model on three scenarios:
#   1. Clean test set (no trigger)
#   2. Adversary (Amazon) without trigger  → ASR
#   3. Adversary (Amazon) with trigger     → Immunity Rate
#
# Environment variables:
#   MODEL_PATH   - base model path (required)
#   LORA_PATH    - LoRA checkpoint path (optional)
#   TEST_JSON    - test data JSON (default: ./data/poisoned/test.json)
#   IMG_DIR      - root image directory (default: ./data/poisoned)
#   OUTPUT_DIR   - where to save eval results (default: ./results)
#   MODEL_TYPE   - "os_atlas" or "seeclick" (default: os_atlas)
#   MAX_SAMPLES  - max samples per scenario for quick eval (optional)

set -e
cd "$(dirname "$0")/.."

if [ -z "$MODEL_PATH" ]; then
    echo "ERROR: MODEL_PATH is required."
    echo "       Usage: MODEL_PATH=/path/to/model bash scripts/eval.sh"
    exit 1
fi

LORA_PATH=${LORA_PATH:-""}
TEST_JSON=${TEST_JSON:-./data/poisoned/test.json}
IMG_DIR=${IMG_DIR:-./data/poisoned}
OUTPUT_DIR=${OUTPUT_DIR:-./results}
MODEL_TYPE=${MODEL_TYPE:-os_atlas}

echo "============================================================"
echo "  Trigger-Off: Evaluation"
echo "============================================================"
echo "  MODEL_PATH = $MODEL_PATH"
echo "  LORA_PATH  = $LORA_PATH"
echo "  TEST_JSON  = $TEST_JSON"
echo "  IMG_DIR    = $IMG_DIR"
echo "  OUTPUT_DIR = $OUTPUT_DIR"
echo "  MODEL_TYPE = $MODEL_TYPE"
echo "------------------------------------------------------------"

LORA_ARG=""
if [ -n "$LORA_PATH" ]; then
    LORA_ARG="--lora_path $LORA_PATH"
fi

MAX_SAMPLES_ARG=""
if [ -n "$MAX_SAMPLES" ]; then
    MAX_SAMPLES_ARG="--max_samples $MAX_SAMPLES"
fi

python -m src.evaluation.evaluate \
    --model_path  "$MODEL_PATH" \
    --test_json   "$TEST_JSON" \
    --img_dir     "$IMG_DIR" \
    --output_dir  "$OUTPUT_DIR" \
    --model_type  "$MODEL_TYPE" \
    $LORA_ARG \
    $MAX_SAMPLES_ARG

echo ""
echo "============================================================"
echo "  Evaluation complete."
echo "  Results saved to: $OUTPUT_DIR/eval_results.json"
echo "============================================================"