| """Base interface for benchmark resolvers. |
| |
| Benchmark resolvers fetch problems from external sources (e.g., datasets, APIs) |
| and generate the necessary files (initial_program, evaluator configuration) for |
| SkyDiscover to run optimization on them. |
| """ |
|
|
| from abc import ABC, abstractmethod |
| from pathlib import Path |
| from typing import Any, Dict |
|
|
| from skydiscover.benchmarks.resolution import BenchmarkResolution |
|
|
|
|
| class BenchmarkResolver(ABC): |
| """Base class for benchmark-specific problem resolvers. |
| |
| Resolvers are responsible for: |
| 1. Fetching problem specifications from external sources |
| 2. Generating initial_program files with appropriate structure |
| 3. Configuring evaluators (via environment variables or generated files) |
| |
| Example usage: |
| resolver = KernelBenchResolver() |
| initial_program, evaluator = resolver.resolve( |
| config={'level': 1, 'problem_id': 3}, |
| output_dir=Path('/tmp/skydiscover_kernelbench_123') |
| ) |
| """ |
|
|
| @abstractmethod |
| def resolve(self, config: Dict[str, Any], output_dir: Path) -> BenchmarkResolution: |
| """Resolve a benchmark problem to concrete file paths and evaluator config. |
| |
| Args: |
| config: Benchmark configuration dictionary containing benchmark-specific |
| problem specifications and parameters. |
| The exact keys depend on the benchmark implementation. |
| output_dir: Directory where generated files should be placed. |
| |
| Returns: |
| BenchmarkResolution containing: |
| - initial_program_path: Path to the generated initial program file |
| - evaluator_path: Path to the evaluator (file or directory) |
| - evaluator_env_vars: Per-run environment variables for the evaluator |
| |
| """ |
| pass |
|
|
