""" eval_set entry point for L2-Bench evaluation. Usage: uv run python -m l2_bench_eval.eval \ --model bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0 \ --log-dir logs/run-001 # Smoke test with 2 samples uv run python -m l2_bench_eval.eval \ --model bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0 \ --log-dir logs/smoke-test \ --sample-limit 2 # Run with custom solver config and scorer uv run python -m l2_bench_eval.eval \ --model bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0 \ --log-dir logs/run-001 \ --epochs 2 --sample-limit 10 \ --solver-max-tokens 8192 --solver-temperature 0.5 \ --scorer-model bedrock/us.anthropic.claude-sonnet-4-5-20250929-v1:0 \ --csv-path path/to/data.csv --resources-dir path/to/resources Version | Date | Author | Change comment --------|------------|-----------|--------------- 1.0.0 | 2026-07-29 | M. Ku | Initial open-source release """ import argparse from pathlib import Path from dotenv import load_dotenv from inspect_ai import eval_set from inspect_ai.model import GenerateConfig, get_model from pydantic import BaseModel from l2_bench_eval import config from l2_bench_eval.bedrock_patch import patch_bedrock_timeout from l2_bench_eval.score import ScorerSetting from l2_bench_eval.task import create_l2_bench_eval_task class EvalRunParams(BaseModel): """Parameters for a single evaluation run. Attributes ---------- solver_model_name : str Model identifier passed to ``get_model`` (e.g. ``bedrock/...``). solver_model_base_url : str or None Optional base URL override for the solver model API. solver_model_config : GenerateConfig Generation configuration for the solver model. log_dir : str Directory where eval logs are written. epochs : int Number of evaluation epochs. retry_on_error : int or None Number of retries on transient errors (``None`` disables retries). continue_on_fail : bool If ``True``, keep running remaining samples after a failure. scorer_setting : ScorerSetting or None Optional scorer model and generation configuration. csv_path : Path or None Path to the tasks CSV file. Uses the repo default when ``None``. resources_dir : Path or None Path to the task resources directory. Uses the repo default when ``None``. first_n_samples : int or None Limit evaluation to the first *n* samples. sample_range : tuple of (int, int) or None Slice range ``(start, end)`` applied to the dataset. Overrides ``first_n_samples`` when set. """ solver_model_name: str solver_model_base_url: str | None = None solver_model_config: GenerateConfig = GenerateConfig(max_tokens=4096, temperature=0.0) log_dir: str epochs: int = 1 retry_on_error: int | None = 1 continue_on_fail: bool = True scorer_setting: ScorerSetting | None = None csv_path: Path | None = None resources_dir: Path | None = None first_n_samples: int | None = None sample_range: tuple[int, int] | None = None # will override first_n_samples task_ids: list[int] | None = None # will override sample_range def run_eval(params: EvalRunParams): """Execute an L2-Bench evaluation run. Parameters ---------- params : EvalRunParams Fully-populated run parameters including model, scorer, and dataset settings. """ patch_bedrock_timeout(read_timeout=600) solver_model = get_model( model=params.solver_model_name, base_url=params.solver_model_base_url, config=params.solver_model_config, ) task = create_l2_bench_eval_task( scorer_setting=params.scorer_setting, csv_path=params.csv_path, resources_dir=params.resources_dir, first_n_samples=params.first_n_samples, sample_range=params.sample_range, task_ids=params.task_ids ) eval_set( tasks=[task], model=solver_model, log_dir=params.log_dir, epochs=params.epochs, retry_on_error=params.retry_on_error, continue_on_fail=params.continue_on_fail ) def main(): """CLI entry point for L2-Bench evaluation.""" parser = argparse.ArgumentParser(description="Run L2-Bench eval_set") parser.add_argument("--model", required=True, help="Solver model name") parser.add_argument("--log-dir", required=True, help="Log directory") parser.add_argument("--epochs", type=int, default=1) parser.add_argument("--sample-limit", type=int, default=0, help="0 = no limit") parser.add_argument("--env-file", type=Path, default=Path.cwd() / ".env", help="Path to .env file") parser.add_argument("--solver-max-tokens", type=int, default=4096, help="Solver max output tokens") parser.add_argument("--solver-temperature", type=float, default=0.0, help="Solver sampling temperature") parser.add_argument("--solver-top-p", type=float, default=None, help="Solver top-p (nucleus sampling)") parser.add_argument("--solver-top-k", type=int, default=None, help="Solver top-k sampling") parser.add_argument("--solver-frequency-penalty", type=float, default=None, help="Solver frequency penalty") parser.add_argument("--solver-presence-penalty", type=float, default=None, help="Solver presence penalty") parser.add_argument("--solver-seed", type=int, default=None, help="Solver random seed") parser.add_argument("--solver-stop-seqs", nargs="*", default=None, help="Solver stop sequences") parser.add_argument("--solver-num-choices", type=int, default=None, help="Solver number of choices") parser.add_argument("--solver-best-of", type=int, default=None, help="Solver best-of sampling count") parser.add_argument("--solver-max-retries", type=int, default=None, help="Solver max retries") parser.add_argument("--solver-timeout", type=int, default=None, help="Solver timeout in seconds") parser.add_argument("--solver-max-connections", type=int, default=None, help="Solver max connections") parser.add_argument("--solver-reasoning-tokens", type=int, default=None, help="Solver reasoning/thinking token budget") parser.add_argument("--solver-reasoning-effort", choices=["none", "minimal", "low", "medium", "high", "xhigh"], default=None, help="Solver reasoning effort level") parser.add_argument("--scorer-model", default=config.DEFAULT_JUDGE_MODEL, help="Judge model name") parser.add_argument("--scorer-max-tokens", type=int, default=None, help="Scorer max output tokens") parser.add_argument("--scorer-temperature", type=float, default=None, help="Judge sampling temperature. Leave unset when a reasoning budget is in use") parser.add_argument("--scorer-top-p", type=float, default=None, help="Scorer top-p (nucleus sampling)") parser.add_argument("--scorer-top-k", type=int, default=None, help="Scorer top-k sampling") parser.add_argument("--scorer-frequency-penalty", type=float, default=None, help="Scorer frequency penalty") parser.add_argument("--scorer-presence-penalty", type=float, default=None, help="Scorer presence penalty") parser.add_argument("--scorer-seed", type=int, default=None, help="Scorer random seed") parser.add_argument("--scorer-stop-seqs", nargs="*", default=None, help="Scorer stop sequences") parser.add_argument("--scorer-num-choices", type=int, default=None, help="Scorer number of choices") parser.add_argument("--scorer-best-of", type=int, default=None, help="Scorer best-of sampling count") parser.add_argument("--scorer-max-retries", type=int, default=None, help="Scorer max API retries") parser.add_argument("--scorer-timeout", type=int, default=None, help="Scorer timeout in seconds") parser.add_argument("--scorer-max-connections", type=int, default=None, help="Scorer max connections") parser.add_argument("--scorer-reasoning-tokens", type=int, default=config.DEFAULT_JUDGE_REASONING_TOKENS, help="Judge reasoning/thinking token budget") parser.add_argument("--scorer-reasoning-effort", choices=["none", "minimal", "low", "medium", "high", "xhigh"], default=None, help="Scorer reasoning effort level") parser.add_argument( "--task-ids", nargs="*", type=int, default=None, help="List of task IDs to evaluate", ) parser.add_argument("--continue-on-fail", action=argparse.BooleanOptionalAction, default=True) parser.add_argument("--csv-path", type=Path, default=None) parser.add_argument("--resources-dir", type=Path, default=None) parser.add_argument( "--prompt-version", default=config.DEFAULT_JUDGE_PROMPT_VERSION, help="Judge prompt version. v1 is the production judge; v2-v4 are the paper's ablations", ) parser.add_argument( "--judge-verdict-retries", type=int, default=config.DEFAULT_JUDGE_MAX_RETRIES, help="Times to re-prompt the judge when it returns an unparseable verdict", ) args = parser.parse_args() load_dotenv(args.env_file) params = EvalRunParams( solver_model_name=args.model, solver_model_config=GenerateConfig( max_tokens=args.solver_max_tokens, temperature=args.solver_temperature, top_p=args.solver_top_p, top_k=args.solver_top_k, frequency_penalty=args.solver_frequency_penalty, presence_penalty=args.solver_presence_penalty, seed=args.solver_seed, stop_seqs=args.solver_stop_seqs, num_choices=args.solver_num_choices, best_of=args.solver_best_of, max_retries=args.solver_max_retries, timeout=args.solver_timeout, max_connections=args.solver_max_connections, reasoning_tokens=args.solver_reasoning_tokens, reasoning_effort=args.solver_reasoning_effort, ), log_dir=args.log_dir, epochs=args.epochs, continue_on_fail=args.continue_on_fail, scorer_setting=ScorerSetting( model=args.scorer_model, max_retries=args.judge_verdict_retries, scorer_model_config=GenerateConfig( max_tokens=args.scorer_max_tokens, temperature=args.scorer_temperature, top_p=args.scorer_top_p, top_k=args.scorer_top_k, frequency_penalty=args.scorer_frequency_penalty, presence_penalty=args.scorer_presence_penalty, seed=args.scorer_seed, stop_seqs=args.scorer_stop_seqs, num_choices=args.scorer_num_choices, best_of=args.scorer_best_of, max_retries=args.scorer_max_retries, timeout=args.scorer_timeout, max_connections=args.scorer_max_connections, reasoning_tokens=args.scorer_reasoning_tokens, reasoning_effort=args.scorer_reasoning_effort, ), prompt_version=args.prompt_version, ), csv_path=args.csv_path, resources_dir=args.resources_dir, first_n_samples=args.sample_limit if args.sample_limit > 0 else None, task_ids=args.task_ids, ) run_eval(params) if __name__ == "__main__": main()