File size: 4,562 Bytes
4c8cb4b | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 | #!/usr/bin/env bash
set -u
RUN_ID="${RUN_ID:-cloning_50shards_$(date -u +%Y%m%d_%H%M%S)}"
PROJECT_ROOT="/225040511/project"
PYTHON_BIN="${LAB_BENCH_RUNNER_PYTHON:-/225040511/miniconda3/envs/biomni_e1/bin/python}"
LOG_ROOT="${PROJECT_ROOT}/cloningscenarios_logs/${RUN_ID}"
SHARD_COUNT="${CLONING_SHARD_COUNT:-50}"
SHARD_CONCURRENCY="${CLONING_SHARD_CONCURRENCY:-5}"
DEV_SIZE="${CLONING_DEV_SIZE:-0}"
TEST_SIZE="${CLONING_TEST_SIZE:-33}"
SEED="${CLONING_SEED:-20260514}"
mkdir -p "${LOG_ROOT}"
for ENV_FILE in "${PROJECT_ROOT}/.env" "${PROJECT_ROOT}/react_code_bioagent_deepseek/.env" "${PROJECT_ROOT}/LAB-Bench/.env"; do
if [[ -f "${ENV_FILE}" ]]; then
set -a
# shellcheck disable=SC1090
source "${ENV_FILE}"
set +a
fi
done
export DEEPSEEK_BASE_URL="${DEEPSEEK_BASE_URL:-https://api.deepseek.com/v1}"
export DEEPSEEK_MODEL_NAME="${DEEPSEEK_MODEL_NAME:-deepseek-chat}"
export REACT_CODE_API_KEY="${REACT_CODE_API_KEY:-${DEEPSEEK_API_KEY:-${BIOMNI_CUSTOM_API_KEY:-}}}"
export REACT_CODE_BASE_URL="${REACT_CODE_BASE_URL:-${DEEPSEEK_BASE_URL}}"
export REACT_CODE_MODEL="${REACT_CODE_MODEL:-${DEEPSEEK_MODEL_NAME}}"
wait_limited() {
local -n pid_array="$1"
if [[ "${#pid_array[@]}" -lt "${SHARD_CONCURRENCY}" ]]; then
return
fi
local pid="${pid_array[0]}"
wait "${pid}" || true
pid_array=("${pid_array[@]:1}")
}
count_rows() {
local file="$1"
if [[ -f "${file}" ]]; then
wc -l < "${file}"
else
echo 0
fi
}
run_react_code_bioagent() {
local runner="${PROJECT_ROOT}/React+code_labbench/run_labbench_react_code.py"
local out="${PROJECT_ROOT}/react_code_bioagent_deepseek/labbench_runs/react_code_cloningscenarios_${RUN_ID}"
local result="${out}/cloningscenarios_results.jsonl"
local reasoning="${out}/cloningscenarios_reasoning.log"
mkdir -p "${out}/shard_logs"
touch "${result}" "${reasoning}"
echo "react_code_bioagent output=${out}"
echo "react_code_bioagent before_rows=$(count_rows "${result}")"
local pids=()
for ((shard=0; shard<SHARD_COUNT; shard++)); do
local label
label="$(printf "react_code_bioagent_cloningscenarios_shard%02dof%02d" "$((shard + 1))" "${SHARD_COUNT}")"
"${PYTHON_BIN}" "${runner}" \
--eval CloningScenarios \
--split test \
--dev-size "${DEV_SIZE}" \
--test-size "${TEST_SIZE}" \
--seed "${SEED}" \
--shard-index "${shard}" \
--shard-count "${SHARD_COUNT}" \
--output-root "${out}" \
--result-file "${result}" \
--reasoning-log "${reasoning}" \
--skip-existing-results \
> "${out}/shard_logs/${label}.log" 2>&1 &
pids+=("$!")
wait_limited pids
done
for pid in "${pids[@]}"; do wait "${pid}" || true; done
echo "react_code_bioagent after_rows=$(count_rows "${result}")"
}
run_base_deepseek() {
local runner="${PROJECT_ROOT}/react_code_bioagent_deepseek/react_code_bioagent/labbench_runner.py"
local out="${PROJECT_ROOT}/base_llm_labbench/results/cloningscenarios_base_deepseek_${RUN_ID}"
local result="${out}/cloningscenarios_results.jsonl"
local reasoning="${out}/cloningscenarios_reasoning.log"
mkdir -p "${out}/shard_logs"
touch "${result}" "${reasoning}"
echo "base_deepseek output=${out}"
echo "base_deepseek before_rows=$(count_rows "${result}")"
local pids=()
for ((shard=0; shard<SHARD_COUNT; shard++)); do
local label
label="$(printf "base_deepseek_cloningscenarios_shard%02dof%02d" "$((shard + 1))" "${SHARD_COUNT}")"
"${PYTHON_BIN}" "${runner}" \
--evals CloningScenarios \
--splits test \
--dev-size "${DEV_SIZE}" \
--test-size "${TEST_SIZE}" \
--seed "${SEED}" \
--n-threads 1 \
--shard-index "${shard}" \
--shard-count "${SHARD_COUNT}" \
--output-root "${out}" \
--compact-results-path "${result}" \
--reasoning-log-path "${reasoning}" \
--skip-existing-results \
> "${out}/shard_logs/${label}.log" 2>&1 &
pids+=("$!")
wait_limited pids
done
for pid in "${pids[@]}"; do wait "${pid}" || true; done
echo "base_deepseek after_rows=$(count_rows "${result}")"
}
echo "RUN_ID=${RUN_ID}"
echo "SHARD_COUNT=${SHARD_COUNT}"
echo "SHARD_CONCURRENCY=${SHARD_CONCURRENCY}"
run_react_code_bioagent > "${LOG_ROOT}/react_code_bioagent_deepseek.log" 2>&1 &
pid_react=$!
run_base_deepseek > "${LOG_ROOT}/base_deepseek.log" 2>&1 &
pid_base=$!
echo "react_code_bioagent_pid=${pid_react}"
echo "base_deepseek_pid=${pid_base}"
echo "log_root=${LOG_ROOT}"
wait "${pid_react}" || true
wait "${pid_base}" || true
echo "finished RUN_ID=${RUN_ID}"
|