scripts/run_latency_protocol_50.sh

#!/usr/bin/env bash
set -euo pipefail

# Run each system alone under two concurrency settings. This makes isolated
# service time and loaded service time comparable instead of mixing systems in
# one scheduler queue.
: "${SERVER_RUN_DIR:?Set to artifacts/servers/<server-run>}"
ROOT="${ROOT:-$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)}"
UV="${UV:-/home/clouduser/miniconda3/envs/vllm-nightly/bin/uv}"
RUN_GROUP="${RUN_GROUP:-latency50_$(date -u +%Y%m%d_%H%M%S)}"
RETRIEVAL_GPU="${RETRIEVAL_GPU:-5}"
WARMUP_EXAMPLES="${WARMUP_EXAMPLES:-3}"
EXPERIMENT="${EXPERIMENT:-configs/experiments/fair_50.yaml}"
cd "$ROOT"

# shellcheck disable=SC1090
source "$SERVER_RUN_DIR/endpoints.env"
if [[ -n "${MODEL_BASE_URLS_OVERRIDE:-}" ]]; then
  export MODEL_BASE_URLS="$MODEL_BASE_URLS_OVERRIDE"
fi
export MODEL_API_KEY="${MODEL_API_KEY:-EMPTY}"

datasets=(hotpotqa 2wikimultihopqa musique)
systems=(fixed adaptive_router one_shot_planner agentic retrieve_verify_refine parallel_fanout sac sac_latency sac_accuracy)
for mode in isolated loaded; do
  if [[ "$mode" == "isolated" ]]; then concurrency=1; else concurrency=16; fi
  for dataset in "${datasets[@]}"; do
    for system in "${systems[@]}"; do
      run_id="${RUN_GROUP}_${mode}_${dataset}_${system}"
      log="artifacts/eval_logs/${run_id}.log"
      mkdir -p "$(dirname "$log")"
      OMP_NUM_THREADS=1 MKL_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 NUMEXPR_NUM_THREADS=1 \
        PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True \
        CUDA_VISIBLE_DEVICES="$RETRIEVAL_GPU" "$UV" run agentic-search benchmark run \
          --examples "data/processed/benchmarks/${dataset}.jsonl" \
          --corpus data/processed/multihop_context_union.jsonl \
          --experiment "$EXPERIMENT" \
          --systems "$system" \
          --concurrency "$concurrency" \
          --warmup-examples "$WARMUP_EXAMPLES" \
          --no-cache \
          --run-id "$run_id" \
          >"$log" 2>&1
    done
  done
done