File size: 4,487 Bytes
c6856c9 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 | #!/usr/bin/env bash
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
ROOT="$(cd "${SCRIPT_DIR}/../../.." && pwd)"
EXP_ROOT="${ROOT}/experiments/ablation"
RUNNER="${ROOT}/run_bioagent_bench.py"
EVALUATOR="${ROOT}/evaluate_bioagent_bench.py"
BIOAGENT_BENCH_ROOT="$(cd "${ROOT}/.." && pwd)/bioagent-bench"
PYTHON_BIN="${BIOMANUS_PYTHON:-/225040511/miniconda3/envs/biomni_e1/bin/python}"
GRAPH_DIR="${ROOT}/graph_outputs/mcp_generated_graph_all_20260522_124110"
BACKGROUND=0
VARIANT="all"
FORCE=0
declare -a TASK_ARGS=("--all")
TASK_MODE_SET=0
while [[ $# -gt 0 ]]; do
case "$1" in
--background) BACKGROUND=1; shift ;;
--foreground) BACKGROUND=0; shift ;;
--variant) VARIANT="$2"; shift 2 ;;
--task)
if [[ "${TASK_MODE_SET}" -eq 0 ]]; then
TASK_ARGS=()
TASK_MODE_SET=1
fi
TASK_ARGS+=("--task" "$2")
shift 2
;;
--force-rerun) FORCE=1; shift ;;
*) echo "Unknown argument: $1" >&2; exit 2 ;;
esac
done
if [[ "${BACKGROUND}" -eq 1 ]]; then
mkdir -p "${EXP_ROOT}/logs"
LOG="${EXP_ROOT}/logs/bioagentbench_ablation_$(date -u +%Y%m%d_%H%M%S).log"
SCRIPT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/$(basename "${BASH_SOURCE[0]}")"
CMD=("${SCRIPT}" "--foreground" "--variant" "${VARIANT}")
if [[ "${FORCE}" -eq 1 ]]; then CMD+=("--force-rerun"); fi
nohup "${CMD[@]}" > "${LOG}" 2>&1 < /dev/null &
echo "Started BioAgentBench ablation in background."
echo "PID: $!"
echo "Log: ${LOG}"
exit 0
fi
if [[ -f "${ROOT}/.env" ]]; then
# shellcheck disable=SC1090
source "${ROOT}/.env"
fi
if [[ -n "${DEEPSEEK_API_KEY:-}" ]]; then
export BIOMNI_LLM_PROVIDER="${BIOMNI_LLM_PROVIDER:-deepseek}"
export DEEPSEEK_BASE_URL="${DEEPSEEK_BASE_URL:-https://api.deepseek.com/v1}"
export DEEPSEEK_MODEL_NAME="${DEEPSEEK_MODEL_NAME:-deepseek-chat}"
export BIOMNI_SOURCE="${BIOMNI_SOURCE:-Custom}"
export BIOMNI_LLM="${BIOMNI_LLM:-${DEEPSEEK_MODEL_NAME}}"
export BIOMNI_CUSTOM_BASE_URL="${BIOMNI_CUSTOM_BASE_URL:-${DEEPSEEK_BASE_URL}}"
export BIOMNI_CUSTOM_API_KEY="${BIOMNI_CUSTOM_API_KEY:-${DEEPSEEK_API_KEY}}"
fi
if [[ -n "${BIOMNI_CUSTOM_BASE_URL:-}" && ! "${BIOMNI_CUSTOM_BASE_URL}" =~ ^https?:// ]]; then
export BIOMNI_CUSTOM_BASE_URL="${DEEPSEEK_BASE_URL:-https://api.deepseek.com/v1}"
fi
if [[ -n "${DEEPSEEK_BASE_URL:-}" && ! "${DEEPSEEK_BASE_URL}" =~ ^https?:// ]]; then
export DEEPSEEK_BASE_URL="https://api.deepseek.com/v1"
fi
run_variant() {
local key="$1"
local label="$2"
shift 2
local runs_root="${EXP_ROOT}/results/${key}/bioagentbench"
local eval_json="${EXP_ROOT}/results/${key}/bioagentbench_evaluation.json"
mkdir -p "${runs_root}"
echo "=== BioAgentBench ${label} ==="
"${PYTHON_BIN}" "${RUNNER}" \
"${TASK_ARGS[@]}" \
--output-root "${runs_root}" \
--mcp-graph "${GRAPH_DIR}" \
--executable-mcp-only \
--timeout-seconds "${BIOMANUS_BIOAGENT_TIMEOUT_SECONDS:-1200}" \
"$@"
"${PYTHON_BIN}" "${EVALUATOR}" \
--all \
--runs-root "${runs_root}" \
--dataset-root "${BIOAGENT_BENCH_ROOT}/dataset" \
--judge-mode rule \
--output "${eval_json}"
}
case "${VARIANT}" in
biomanus)
run_variant "biomanus" "full" --use-graph-retriever --use-mcp --mcp-retrieval-mode graph
;;
mcp_flat)
run_variant "mcp_flat" "MCP + flat retrieval" --use-graph-retriever --use-mcp --mcp-retrieval-mode flat
;;
mcp_metadata)
run_variant "mcp_metadata" "MCP + metadata retrieval" --use-graph-retriever --use-mcp --mcp-retrieval-mode metadata
;;
minus_graph)
run_variant "minus_graph" "minus graph" --no-graph-retriever --use-mcp
;;
minus_mcp)
run_variant "minus_mcp" "minus MCP" --use-graph-retriever --no-mcp
;;
minus_mcp_graph)
run_variant "minus_mcp_graph" "minus MCP and graph" --no-graph-retriever --no-mcp
;;
all)
run_variant "biomanus" "full" --use-graph-retriever --use-mcp --mcp-retrieval-mode graph
run_variant "mcp_flat" "MCP + flat retrieval" --use-graph-retriever --use-mcp --mcp-retrieval-mode flat
run_variant "mcp_metadata" "MCP + metadata retrieval" --use-graph-retriever --use-mcp --mcp-retrieval-mode metadata
run_variant "minus_graph" "minus graph" --no-graph-retriever --use-mcp
run_variant "minus_mcp" "minus MCP" --use-graph-retriever --no-mcp
run_variant "minus_mcp_graph" "minus MCP and graph" --no-graph-retriever --no-mcp
;;
*)
echo "Unknown variant: ${VARIANT}" >&2
exit 2
;;
esac
|