File size: 4,016 Bytes
af83196 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 | #!/usr/bin/env bash
# Reproduce math benchmarks (17 problems x 2 search methods).
# All benchmarks launch in parallel.
set -euo pipefail
# ── Settings ─────────────────────────────────────────────────────────────────
# Only two things to change:
MODEL="gpt-5" # main generation model
# MODEL="gemini/gemini-3.0-pro-preview" # alternative
ITERATIONS=100
# -m sets all models (main + guide/paradigm) to the same MODEL.
# API keys: export OPENAI_API_KEY="sk-..." (and/or GEMINI_API_KEY for Gemini)
# ── Install ──────────────────────────────────────────────────────────────────
cd "$(dirname "$0")/../.."
uv sync --extra math
# ── Helper ───────────────────────────────────────────────────────────────────
run() {
local dir=$1 search=$2
local init="$dir/initial_program.py"
[[ -f "$dir/initial_program.cpp" ]] && init="$dir/initial_program.cpp"
[[ -f "$dir/initial_prompt.txt" ]] && init="$dir/initial_prompt.txt"
local cfg="$dir/config.yaml"
[[ -f "$dir/config_${search}.yaml" ]] && cfg="$dir/config_${search}.yaml"
echo "== $search: ${dir#benchmarks/} =="
uv run skydiscover-run "$init" "$dir/evaluator.py" \
-c "$cfg" -s "$search" -m "$MODEL" -i "$ITERATIONS" \
-o "outputs/reproduce/$search/${dir#benchmarks/}"
}
# ── AdaEvolve ────────────────────────────────────────────────────────────────
run benchmarks/math/circle_packing adaevolve &
run benchmarks/math/circle_packing_rect adaevolve &
run benchmarks/math/erdos_min_overlap adaevolve &
run benchmarks/math/first_autocorr_ineq adaevolve &
run benchmarks/math/second_autocorr_ineq adaevolve &
run benchmarks/math/third_autocorr_ineq adaevolve &
run benchmarks/math/uncertainty_ineq adaevolve &
run benchmarks/math/hexagon_packing/11 adaevolve &
run benchmarks/math/hexagon_packing/12 adaevolve &
run benchmarks/math/heilbronn_convex/13 adaevolve &
run benchmarks/math/heilbronn_convex/14 adaevolve &
run benchmarks/math/heilbronn_triangle adaevolve &
run benchmarks/math/minimizing_max_min_dist/2 adaevolve &
run benchmarks/math/minimizing_max_min_dist/3 adaevolve &
run benchmarks/math/matmul adaevolve &
run benchmarks/math/signal_processing adaevolve &
run benchmarks/math/sums_diffs_finite_sets adaevolve &
# ── EvoX ─────────────────────────────────────────────────────────────────────
run benchmarks/math/circle_packing evox &
run benchmarks/math/circle_packing_rect evox &
run benchmarks/math/erdos_min_overlap evox &
run benchmarks/math/first_autocorr_ineq evox &
run benchmarks/math/second_autocorr_ineq evox &
run benchmarks/math/third_autocorr_ineq evox &
run benchmarks/math/uncertainty_ineq evox &
run benchmarks/math/hexagon_packing/11 evox &
run benchmarks/math/hexagon_packing/12 evox &
run benchmarks/math/heilbronn_convex/13 evox &
run benchmarks/math/heilbronn_convex/14 evox &
run benchmarks/math/heilbronn_triangle evox &
run benchmarks/math/minimizing_max_min_dist/2 evox &
run benchmarks/math/minimizing_max_min_dist/3 evox &
run benchmarks/math/matmul evox &
run benchmarks/math/signal_processing evox &
run benchmarks/math/sums_diffs_finite_sets evox &
wait
echo "math.sh: all 34 runs finished."
|