File size: 3,797 Bytes
3329027 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 | """Benchmark execution engine."""
from __future__ import annotations
from solvbench.constants import PROBLEM_TYPES, SIZE_PRESETS, SOLVERS
from solvbench.generators import generate_instance
from solvbench.models import BenchmarkRun, ProblemInstance, SolverResult
from solvbench.solvers import ALL_SOLVERS, AVAILABLE_SOLVERS, get_solver
class BenchmarkEngine:
def __init__(self, time_limit_sec: float = 10.0) -> None:
self.time_limit_sec = time_limit_sec
def run_instance(
self,
instance: ProblemInstance,
solvers: list[str] | None = None,
include_reference: bool = True,
) -> BenchmarkRun:
solver_ids = solvers or (ALL_SOLVERS if include_reference else AVAILABLE_SOLVERS)
results: list[SolverResult] = []
baseline: SolverResult | None = None
for sid in solver_ids:
if sid not in SOLVERS:
continue
solver = get_solver(sid, self.time_limit_sec, baseline=baseline)
result = solver.solve(instance)
results.append(result)
if sid in AVAILABLE_SOLVERS and result.metrics.feasible:
if baseline is None or result.metrics.solution_quality > baseline.metrics.solution_quality:
baseline = result
winner = self._pick_winner(results)
gap = self._winner_gap(results, winner)
return BenchmarkRun(instance=instance, results=results, winner=winner, winner_gap_pct=gap)
def run_problem_suite(
self,
problem_type: str,
sizes: list[str] | None = None,
seeds: list[int] | None = None,
include_reference: bool = True,
) -> list[BenchmarkRun]:
sizes = sizes or ["small", "medium", "large"]
seeds = seeds or [42, 123, 456]
runs = []
for size in sizes:
for seed in seeds:
instance = generate_instance(problem_type, size, seed)
runs.append(self.run_instance(instance, include_reference=include_reference))
return runs
def run_full_suite(self, include_reference: bool = True) -> list[BenchmarkRun]:
all_runs = []
for pt in PROBLEM_TYPES:
all_runs.extend(self.run_problem_suite(pt, sizes=["medium"], seeds=[42], include_reference=include_reference))
return all_runs
@staticmethod
def _pick_winner(results: list[SolverResult]) -> str:
feasible = [r for r in results if r.metrics.feasible]
if not feasible:
return results[0].solver_id if results else "none"
maximize = feasible[0].problem_type in ("knapsack", "max_independent_set")
if maximize:
best = max(feasible, key=lambda r: (r.metrics.solution_quality, -r.metrics.total_solving_time))
else:
best = min(
feasible,
key=lambda r: (r.metrics.objective_value if r.metrics.objective_value > 0 else 1e18, r.metrics.total_solving_time),
)
return best.solver_id
@staticmethod
def _winner_gap(results: list[SolverResult], winner: str) -> float:
winner_r = next((r for r in results if r.solver_id == winner), None)
if not winner_r or not winner_r.metrics.feasible:
return 0.0
others = [r for r in results if r.solver_id != winner and r.metrics.feasible]
if not others:
return 0.0
gaps = []
for r in others:
if winner_r.metrics.objective_value > 0:
gaps.append(abs(r.metrics.objective_value - winner_r.metrics.objective_value) / winner_r.metrics.objective_value * 100)
else:
gaps.append(abs(r.metrics.solution_quality - winner_r.metrics.solution_quality) * 100)
return round(sum(gaps) / len(gaps), 2)
|