scripts/run_full_eval.sh

#!/usr/bin/env bash
# End-to-end evaluation of Amul AI via CeRAI:
#   1. Generate BBK + KCC test cases
#   2. Concat them into one plans.json + datapoints.json (CeRAI reads exactly one of each)
#   3. Wipe + re-import CeRAI's SQLite DB
#   4. Re-register Amul target
#   5. Run testcase_executor for both plans (T_AmulDairy, T_AmulKCC)
#   6. Run response_analyzer over both runs
#   7. Emit results/<run_name>/{summary.json,results.md} for each run
#
# Prereqs (handled by scripts/setup_cerai.sh):
#   - .venv/ has the slim CeRAI dep set installed
#   - amul_proxy/server.py is running on :8088
#   - AIEvaluationTool/src/app/interface_manager/main.py is running on :8000
#
# Usage:
#   bash scripts/run_full_eval.sh [BBK_ROWS] [KCC_ROWS]
set -euo pipefail

ROOT="$(cd "$(dirname "$0")/.." && pwd)"
CERAI="$ROOT/AIEvaluationTool"
BBK_ROWS="${1:-20}"
KCC_ROWS="${2:-20}"

source "$ROOT/.venv/bin/activate"

echo "[1/7] Generating BBK dairy test cases (n=$BBK_ROWS base rows × 2 styles × 6 metrics)"
python "$ROOT/scripts/build_cerai_testcases.py" --max-rows "$BBK_ROWS"

echo "[2/7] Generating KCC animal-husbandry test cases (n=$KCC_ROWS rows × 6 metrics)"
python "$ROOT/scripts/build_kcc_testcases.py" --max-rows "$KCC_ROWS"

echo "[3/7] Merging plans + datapoints inline with Python"
ROOT="$ROOT" python - <<'PY'
import json, os
from pathlib import Path
PROC = Path(os.environ["ROOT"]) / "data" / "processed"
plan = {**json.loads((PROC/"amul_dairy_plan.json").read_text()),
        **json.loads((PROC/"kcc_dairy_plan.json").read_text())}
dp_a = json.loads((PROC/"amul_dairy_datapoints.json").read_text())
dp_b = json.loads((PROC/"kcc_dairy_datapoints.json").read_text())
merged = {}
for src in (dp_a, dp_b):
    for k, v in src.items():
        merged.setdefault(k, {"cases": []})["cases"].extend(v["cases"])
(PROC/"amul_combined_plan.json").write_text(json.dumps(plan, indent=2, ensure_ascii=False))
(PROC/"amul_combined_datapoints.json").write_text(json.dumps(merged, indent=2, ensure_ascii=False))
print("plans:", list(plan.keys()))
for k, v in merged.items():
    print(f"  metric {k}: {len(v['cases'])} cases")
PY

cp "$ROOT/data/processed/amul_combined_plan.json"        "$CERAI/data/amul_dairy_plan.json"
cp "$ROOT/data/processed/amul_combined_datapoints.json"  "$CERAI/data/amul_dairy_datapoints.json"

echo "[4/7] Wiping + re-importing CeRAI's SQLite DB"
rm -f "$CERAI/data/AIEvaluationData.db"
( cd "$CERAI" && python src/app/importer/main.py --config config.json ) > /tmp/cerai_import.log 2>&1
tail -2 /tmp/cerai_import.log
python "$ROOT/scripts/register_amul_target.py" | tail -1

echo "[5/7] Running testcase_executor (single plan; CeRAI's MetricTestCaseMapping"
echo "      is shared between plans T_AmulDairy and T_AmulKCC since they use the"
echo "      same metric IDs. Running plan 1 once covers all (BBK + KCC) testcases."
( cd "$CERAI" && python src/app/testcase_executor/main.py --config config.json --testplan-id 1 --max-testcases 9999 --execute ) > /tmp/cerai_run.log 2>&1
RUN=$(grep -oE "creating a new Run \"[a-z0-9-]+\"" /tmp/cerai_run.log | head -1 | grep -oE '"[a-z0-9-]+"' | tr -d '"')
echo "  Run name: $RUN"

echo "[6/7] Running response_analyzer"
( cd "$CERAI" && python src/app/response_analyzer/analyze.py --config config.json --run-name "$RUN" --force ) > /tmp/cerai_analyze.log 2>&1

echo "[7/7] Summarising (results/<run>/summary.json + results.md, split by dataset)"
python "$ROOT/scripts/summarize_run.py" --run-name "$RUN"

echo
echo "Done."
echo "  Run: $RUN  (results/$RUN/)"