File size: 1,959 Bytes
ec1ecb9 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 | #!/usr/bin/env bash
set -euo pipefail
# Run each system alone under two concurrency settings. This makes isolated
# service time and loaded service time comparable instead of mixing systems in
# one scheduler queue.
: "${SERVER_RUN_DIR:?Set to artifacts/servers/<server-run>}"
ROOT="${ROOT:-$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)}"
UV="${UV:-/home/clouduser/miniconda3/envs/vllm-nightly/bin/uv}"
RUN_GROUP="${RUN_GROUP:-latency50_$(date -u +%Y%m%d_%H%M%S)}"
RETRIEVAL_GPU="${RETRIEVAL_GPU:-5}"
WARMUP_EXAMPLES="${WARMUP_EXAMPLES:-3}"
EXPERIMENT="${EXPERIMENT:-configs/experiments/fair_50.yaml}"
cd "$ROOT"
# shellcheck disable=SC1090
source "$SERVER_RUN_DIR/endpoints.env"
if [[ -n "${MODEL_BASE_URLS_OVERRIDE:-}" ]]; then
export MODEL_BASE_URLS="$MODEL_BASE_URLS_OVERRIDE"
fi
export MODEL_API_KEY="${MODEL_API_KEY:-EMPTY}"
datasets=(hotpotqa 2wikimultihopqa musique)
systems=(fixed adaptive_router one_shot_planner agentic retrieve_verify_refine parallel_fanout sac sac_latency sac_accuracy)
for mode in isolated loaded; do
if [[ "$mode" == "isolated" ]]; then concurrency=1; else concurrency=16; fi
for dataset in "${datasets[@]}"; do
for system in "${systems[@]}"; do
run_id="${RUN_GROUP}_${mode}_${dataset}_${system}"
log="artifacts/eval_logs/${run_id}.log"
mkdir -p "$(dirname "$log")"
OMP_NUM_THREADS=1 MKL_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 NUMEXPR_NUM_THREADS=1 \
PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True \
CUDA_VISIBLE_DEVICES="$RETRIEVAL_GPU" "$UV" run agentic-search benchmark run \
--examples "data/processed/benchmarks/${dataset}.jsonl" \
--corpus data/processed/multihop_context_union.jsonl \
--experiment "$EXPERIMENT" \
--systems "$system" \
--concurrency "$concurrency" \
--warmup-examples "$WARMUP_EXAMPLES" \
--no-cache \
--run-id "$run_id" \
>"$log" 2>&1
done
done
done
|