#!/bin/bash
# Structural mirror of scripts/run_eval_v10_r2_ckpt956.sh, for v14_gencond.
# Default mode is "none" (deployment-equivalent, byte-aligned with how v10_r2
# was eval'd) — this is the headline ship number.
#
# Optional diagnostic modes (re-run with the env override):
#   FORENSICS_GENCOND_MODE=matched bash scripts/run_eval_v14_gencond_3way.sh
#       random 1[FORENSICS_GENCOND_PROB] correct-name vs generic (mirrors training distribution)
#   FORENSICS_GENCOND_MODE=correct bash scripts/run_eval_v14_gencond_3way.sh
#       always prepend correct gen name (oracle upper bound)
#   FORENSICS_GENCOND_MODE=wrong   bash scripts/run_eval_v14_gencond_3way.sh
#       always prepend a deterministic wrong gen name (token-leak control)
#
# FORENSICS_GENCOND_PROB MUST equal the value used in training
# (scripts/run_grpo_forensics_v14_gencond.sh, default 0.5); only consumed in
# matched mode.
set -e

cd /mnt/local-fast/zhangt/forensics_grpo

# Single-source-of-truth for the prob; aligned with run_grpo_forensics_v14_gencond.sh
export FORENSICS_GENCOND_PROB="${FORENSICS_GENCOND_PROB:-0.5}"
export FORENSICS_GENCOND_MODE="${FORENSICS_GENCOND_MODE:-none}"

CKPT_ROOT="${CKPT_ROOT:-outputs_forensics/v14_gencond}"
MODEL=$(ls -d "${CKPT_ROOT}"/checkpoint-* 2>/dev/null \
    | awk -F'-' '{print $NF, $0}' | sort -n -k1,1 | tail -1 | awk '{print $2}')
if [ -z "$MODEL" ]; then
    echo "No checkpoint found under ${CKPT_ROOT}/checkpoint-*" >&2
    exit 1
fi
CKPT_TAG=$(basename "$MODEL" | sed 's/checkpoint-/ckpt/')
OUT="eval_v14_gencond_${CKPT_TAG}_${FORENSICS_GENCOND_MODE}"
mkdir -p "$OUT/logs"
echo "Evaluating $MODEL  ->  $OUT  (MODE=$FORENSICS_GENCOND_MODE  PROB=$FORENSICS_GENCOND_PROB)"

export PATH="/mnt/local-fast/zhangt/torch_env/bin:$PATH"
export LD_LIBRARY_PATH="/opt/conda/lib:${LD_LIBRARY_PATH}"
export PYTHONPATH=".:$PYTHONPATH"

for R in 0 1 2 3 4 5 6 7; do
  CUDA_VISIBLE_DEVICES=$R python evaluate_forensics.py \
      --model_path "$MODEL" \
      --rank $R --world_size 8 --device 0 \
      --out_dir "$OUT" \
      --cot false --max_new_tokens 64 --temperature 0.0 \
      > "$OUT/logs/rank_${R}.log" 2>&1 &
done
wait
echo "all 8 ranks done"

python evaluate_grounding_metrics.py --out_dir "$OUT" | tee "$OUT/grounding_metrics.txt"