    code_memory = (_load_mem(args, root) if _load_mem is not None else None)
    code_index: Optional[Dict[str, Any]] = None
    index_k: int = 0
    index_path: Optional[str] = None
    if _resolve_ci is not None:
        try:
            code_index, index_k, index_path = _resolve_ci(args, root)
        except Exception:
            code_index, index_k, index_path = None, 0, None

    # Pass cache and temp paths
    pass_cache, pass_cache_path = read_pass_cache(root, suite)
    try:
        shard_tag = get_shard_tag(args)
    except Exception:
        shard_tag = ""
    samples_path, verbose_path, errors_path = init_suite_temp_paths(root, suite, shard_tag)

    # Prompt checksum
    prompts_hasher = hashlib.sha256()
    def _build_input(base: str) -> str:
        try:
            return str(build_prompt(prompt_builder_key, base))
        except Exception:
            return base

    def _generate_for_task(tid: str) -> List[Dict[str, Any]]:
        prompt = str(problems[tid]["prompt"]) if isinstance(problems.get(tid), dict) else str(problems[tid])
        base = prompt
        if code_memory is not None:
            try:
                base = code_memory.enhance_prompt(base, problem_id=tid)
            except Exception:
                pass
        elif code_index and index_k > 0:
            try:
                funcs_map = dict(code_index.get("functions", {})) if isinstance(code_index, dict) else {}
                n_funcs = sum(len(v) for v in funcs_map.values()) if funcs_map else 0
                if n_funcs >= int(getattr(args, "planner_index_min_funcs", 50) or 50):
                    base = inject_similar_functions(base, prompt, code_index, index_k)
            except Exception:
                pass
        inp = _build_input(base)
        try:
            from agi_dw.bench.common.pipeline import maybe_short_circuit_from_pass_cache as _shortc, annotate_prompt_with_cached_solution as _anno  # type: ignore


            code_index, index_k, index_path = _resolve_ci(args, root)
        except Exception:
            code_index, index_k, index_path = None, 0, None

    # Pass cache and temp paths
    pass_cache, pass_cache_path = read_pass_cache(root, suite)
    try:
        shard_tag = get_shard_tag(args)
    except Exception:
        shard_tag = ""
    samples_path, verbose_path, errors_path = init_suite_temp_paths(root, suite, shard_tag)

    # Prompt checksum
    prompts_hasher = hashlib.sha256()
    def _build_input(base: str) -> str:
        try:
            return str(build_prompt(prompt_builder_key, base))
        except Exception:
            return base

    def _generate_for_task(tid: str) -> List[Dict[str, Any]]:
        prompt = str(problems[tid]["prompt"]) if isinstance(problems.get(tid), dict) else str(problems[tid])
        base = prompt
        if code_memory is not None:
            try:
                base = code_memory.enhance_prompt(base, problem_id=tid)
            except Exception:
                pass
        elif code_index and index_k > 0:
            try:
                funcs_map = dict(code_index.get("functions", {})) if isinstance(code_index, dict) else {}
                n_funcs = sum(len(v) for v in funcs_map.values()) if funcs_map else 0
                if n_funcs >= int(getattr(args, "planner_index_min_funcs", 50) or 50):
                    base = inject_similar_functions(base, prompt, code_index, index_k)
            except Exception:
                pass
        inp = _build_input(base)
        try:
            from agi_dw.bench.common.pipeline import maybe_short_circuit_from_pass_cache as _shortc, annotate_prompt_with_cached_solution as _anno  # type: ignore
        except Exception:
            _shortc = None  # type: ignore
            _anno = None  # type: ignore
        if _shortc is not None:
            sc = _shortc(suite, tid, inp, pass_cache, trace_path, env_fp, args)
            if sc is not None:
                return sc
        cached = pass_cache.get(tid)
        if cached and str(cached.get("completion", "")).strip() and _anno is not None:
            try:
                base = _anno(base, str(cached.get("completion", "")))
            except Exception:
                pass
        inp = _build_input(base)
        try:
            prompts_hasher.update(inp.encode("utf-8", errors="ignore"))
        except Exception:
            pass
        lines, _chosen = generate_candidates_for_prompt(
            llm=llm,
            cache=cache,
            logger=logger,
            critic=critic,
            prompt=prompt,
            inp=inp,
            sanitize_fn=sanitize_fn,
            task_id=tid,
            args=args,
            suite=suite,
            env_fp=env_fp,
            trace_path=trace_path,
            _prompts_hasher=prompts_hasher,
        )
        return lines

    # Parallel generation
    max_workers = max(1, int(getattr(args, "max_workers", default_workers) or default_workers))
    all_rows: List[Dict[str, Any]] = []
    from concurrent.futures import ThreadPoolExecutor as _TPE, as_completed as _as_completed  # type: ignore
    with trace_span("generate_candidates", {"suite": suite, "n_tasks": len(task_ids), "max_workers": max_workers}):
        with _TPE(max_workers=max_workers) as ex:
            futs = {ex.submit(_generate_for_task, tid): tid for tid in task_ids}
            for fut in _as_completed(futs):
                _tid = futs[fut]
                rows: List[Dict[str, Any]] = []
                try:
                    rows = fut.result()
                except Exception as e:
                    try:
                        import traceback, sys  # type: ignore
                        trace = traceback.format_exc()
                        msg = {"task_id": _tid, "error": str(e), "traceback": trace}
                        with errors_path.open("a", encoding="utf-8") as ef:
                            ef.write(__import__("json").dumps(msg) + "\n")


                                rev_v_risk = ch_v_risk
                        if rev_pre_ok or (rev_v_risk <= ch_v_risk):
                            clean = rev
                            write_trace(trace_path, suite, task_id, "self_eval", None, (int(getattr(args, "seed")) if getattr(args, "seed", None) is not None else None), env_fp, inp, clean, meta={"rev_pre_ok": bool(rev_pre_ok), "rev_v_risk": float(rev_v_risk)})
                        else:
                            break
            except Exception:
                pass
        write_trace(trace_path, suite, task_id, 0, int(locals().get("_latency_ms", 0)), (int(getattr(args, "seed")) if getattr(args, "seed", None) is not None else None), env_fp, inp, clean)
        chosen = clean
        lines.append({"task_id": task_id, "completion": clean, "orig_prompt": prompt, "input": inp, "role": "chosen", "nbest_rank": 0, "latency_ms": int(locals().get("_latency_ms", 0)), "attempt_idx": 0})
        return lines, chosen

    # n-best path
    for ci in range(n_candidates):
        comp = None
        cand_latency_ms: Optional[int] = None
        if ci == 0:
            comp = cache.get(str(getattr(args, "model")), inp, params)
        if comp is None:
            def _gen_multi() -> str:
                return try_generate_python_body(llm, inp, params, grammar_constrained)
            import time as _time
            _t0 = _time.time()
            text = retry_with_backoff(_gen_multi, retries, retry_backoff) or ""
            cand_latency_ms = int((_time.time() - _t0) * 1000)
            comp = text
            if ci == 0 and comp:
                cache.put(str(getattr(args, "model")), inp, params, str(comp))
        cand = sanitize_fn(strip_fences(str(comp or "")), prompt)
        pre_ok, crit_ok, v_risk = evaluate_candidate_signals(cand, prompt, precheck_code, critic, verify_trace_snippet, args)
        candidates.append({"body": cand, "precheck": pre_ok, "critic": crit_ok, "length": len(cand or ""), "v_risk": v_risk, "latency_ms": cand_latency_ms})
    from agi_dw.bench.common.pipeline import pick_best_candidate  # local import to avoid circulars
    chosen = pick_best_candidate(candidates, float(getattr(args, "verify_risk_threshold", 0.6) or 0.6))
    from agi_dw.bench.common.pipeline import rank_candidates  # type: ignore
    _ranks = rank_candidates(candidates)
    # Adaptive extra sampling if chosen looks risky/low-quality
    try:
        ch_pre_ok = False
        ch_crit_ok = False
        ch_v_risk = 0.5
        if chosen:


    except Exception:
        pass
    from agi_dw.core.utils.bench_utils import strip_fences, precheck_code, retry_with_backoff  # type: ignore
    try:
        from agi_dw.core.verifier.llm_verifier import verify_trace_snippet  # type: ignore
    except Exception:
        verify_trace_snippet = None  # type: ignore

    lines: List[Dict[str, Any]] = []
    candidates: List[Dict[str, Any]] = []
    chosen: str = ""

    if n_candidates == 1:
        clean: str = ""
        last_err: Optional[str] = None
        for attempt in range(max(1, retries + 1)):
            comp = None
            if attempt == 0:
                comp = cache.get(str(getattr(args, "model")), inp, params)
                if comp is None:
                    def _gen() -> str:
                        return try_generate_python_body(llm, inp, params, grammar_constrained)
                    import time as _time
                    _t0 = _time.time()
                    text = retry_with_backoff(_gen, retries, retry_backoff) or ""
                    _latency_ms = int((_time.time() - _t0) * 1000)
                    comp = text
            clean_try = sanitize_fn(strip_fences(str(comp or "")), prompt)
            if bool(getattr(args, "precheck", False)):
                ok, err_msg = precheck_code(clean_try)
                if not ok:
                    last_err = err_msg or "precheck failed"
                    if attempt < retries:
                        continue
            if not looks_like_python_code(clean_try):
                last_err = "rejected_noncode"
                if attempt < retries:
                    continue
            if critic and clean_try:
                ok, _ = critic.review(clean_try)
                if not ok:
                    last_err = "critic rejected"


from __future__ import annotations

from pathlib import Path
from typing import Any, Dict, List, Tuple

def _read_jsonl(path: Path) -> List[Dict[str, Any]]:
	rows: List[Dict[str, Any]] = []
	if not path.exists():
		return rows
	with path.open("r", encoding="utf-8") as f:
		for line in f:
			li = line.strip()
			if not li:
				continue
			try:
				rows.append(__import__("json").loads(li))
			except Exception:
				continue
	return rows

def evaluate_solution(
    sample_file: str,
    *,
    dataset_name: str,
    split: str,
    id_field: str,
    starter_field: str,
    tests_fields: List[str],
    k: List[int],
    n_workers: int,
    timeout: float,
    ignore_incomplete: bool,
) -> Dict[str, Any]:
    """Generic evaluator that loads a HF dataset, assembles solutions, runs tests, and writes results.

    The function reads model completions from `sample_file` and creates a results JSONL at
    `sample_file + "_results.jsonl"` with rows of the form: {task_id, completion, passed}.
    """
    from datasets import load_dataset  # type: ignore
    import tempfile
    import subprocess

    sp = Path(sample_file)
    rows = _read_jsonl(sp)
    # Group completions per task id (first k completions considered)
    by_task: Dict[str, List[str]] = {}
    for obj in rows:
        tid = str(obj.get("task_id", ""))
        comp = str(obj.get("completion", ""))
        if not tid:
            continue
        by_task.setdefault(tid, []).append(comp)

    # Load dataset
    ds = load_dataset(dataset_name, split=split, trust_remote_code=True)
    index: Dict[str, Dict[str, Any]] = {}
    for it in ds:
        try:
            pid = str(it.get(id_field, ""))
            if pid:
                index[pid] = dict(it)
        except Exception:
            continue

    def _assemble_solution(starter: str, body: str) -> str:
        try:
            if not body.strip():
                return starter
            lines = starter.splitlines()
            for i, ln in enumerate(lines):
                if ln.strip() == 'pass':
                    indent = ln[: len(ln) - len(ln.lstrip())]
                    blines = [indent + b for b in body.splitlines()]
                    lines[i:i + 1] = blines
                    return "\n".join(lines)
            return starter + ("\n\n" + body if not starter.endswith("\n") else "\n" + body)
        except Exception:
            return starter

    results: Dict[str, bool] = {}
    passed = 0
    total = 0
    for tid, comps in by_task.items():
        orig = index.get(str(tid), {})
        starter = str(orig.get(starter_field, ""))
        tests_code = ""
        for tf in (tests_fields or []):
            if str(orig.get(tf, "")).strip():
                tests_code = str(orig.get(tf))
                break
        if not tests_code.strip():
            results[tid] = False
            continue
        ok = False
        limit_k = (k[0] if (isinstance(k, list) and k) else 1)
        for comp in comps[: max(1, int(limit_k))]:
            try:
                with tempfile.TemporaryDirectory() as td:
                    p = Path(td)
                    (p / "solution.py").write_text(_assemble_solution(starter, comp), encoding="utf-8")
                    runner = "from solution import *\n" + tests_code
                    (p / "runner.py").write_text(runner, encoding="utf-8")
                    res = subprocess.run(["python3", str(p / "runner.py")], stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=max(1, int(timeout)) if timeout else 15)
                    if res.returncode == 0:
                        ok = True
                        break
            except Exception:
                ok = ok or False
        results[tid] = ok
        passed += (1 if ok else 0)
        total += 1

    out_path = Path(str(sp) + "_results.jsonl")
    with out_path.open("w", encoding="utf-8") as wf:
        for obj in rows:
            tid = str(obj.get("task_id", ""))
            wf.write(__import__("json").dumps({"task_id": tid, "completion": str(obj.get("completion", "")), "passed": bool(results.get(tid, False))}) + "\n")

    acc = (passed / max(1, total)) if total else 0.0
    return {"pass@k": acc}


from __future__ import annotations

from pathlib import Path
from typing import Any, Dict, List, Tuple

def _read_jsonl(path: Path) -> List[Dict[str, Any]]:
	rows: List[Dict[str, Any]] = []
	if not path.exists():
		return rows
	with path.open("r", encoding="utf-8") as f:
		for line in f:
			li = line.strip()
			if not li:
				continue
			try:
				rows.append(__import__("json").loads(li))
			except Exception:
				continue
	return rows

def evaluate_solution(
    sample_file: str,
    *,
    dataset_name: str,
    split: str,
    id_field: str,
    starter_field: str,
    tests_fields: List[str],
    k: List[int],
    n_workers: int,
    timeout: float,
    ignore_incomplete: bool,
) -> Dict[str, Any]:
    """Generic evaluator that loads a HF dataset, assembles solutions, runs tests, and writes results.

    The function reads model completions from `sample_file` and creates a results JSONL at
    `sample_file + "_results.jsonl"` with rows of the form: {task_id, completion, passed}.
    """


from pathlib import Path
from typing import Any, Dict, List, Tuple

def _read_jsonl(path: Path) -> List[Dict[str, Any]]:
	rows: List[Dict[str, Any]] = []
	if not path.exists():
		return rows
	with path.open("r", encoding="utf-8") as f:
		for line in f:
			li = line.strip()
			if not li:
				continue
			try:
				rows.append(__import__("json").loads(li))
			except Exception:
				continue
	return rows

def evaluate_solution(
    sample_file: str,
    *,
    dataset_name: str,
    split: str,
    id_field: str,
    starter_field: str,
    tests_fields: List[str],
    k: List[int],
    n_workers: int,
    timeout: float,
    ignore_incomplete: bool,
) -> Dict[str, Any]:
    """Generic evaluator that loads a HF dataset, assembles solutions, runs tests, and writes results.

    The function reads model completions from `sample_file` and creates a results JSONL at
    `sample_file + "_results.jsonl"` with rows of the form: {task_id, completion, passed}.
    """
    from datasets import load_dataset  # type: ignore
    import tempfile
    import subprocess

    sp = Path(sample_file)
    rows = _read_jsonl(sp)
    # Group completions per task id (first k completions considered)
    by_task: Dict[str, List[str]] = {}
    for obj in rows:
        tid = str(obj.get("task_id", ""))
        comp = str(obj.get("completion", ""))
        if not tid:
            continue
        by_task.setdefault(tid, []).append(comp)

    # Load dataset
    ds = load_dataset(dataset_name, split=split, trust_remote_code=True)
    index: Dict[str, Dict[str, Any]] = {}
    for it in ds:
        try:
            pid = str(it.get(id_field, ""))
            if pid:
                index[pid] = dict(it)
        except Exception:
            continue

    def _assemble_solution(starter: str, body: str) -> str:
        try:
            if not body.strip():
                return starter
            lines = starter.splitlines()
            for i, ln in enumerate(lines):
                if ln.strip() == 'pass':
                    indent = ln[: len(ln) - len(ln.lstrip())]
                    blines = [indent + b for b in body.splitlines()]
                    lines[i:i + 1] = blines
                    return "\n".join(lines)
            return starter + ("\n\n" + body if not starter.endswith("\n") else "\n" + body)
        except Exception:
            return starter

    results: Dict[str, bool] = {}
    passed = 0
    total = 0
    for tid, comps in by_task.items():
        orig = index.get(str(tid), {})
        starter = str(orig.get(starter_field, ""))
        tests_code = ""
        for tf in (tests_fields or []):
            if str(orig.get(tf, "")).strip():
                tests_code = str(orig.get(tf))
                break
        if not tests_code.strip():
            results[tid] = False
            continue
        ok = False
        limit_k = (k[0] if (isinstance(k, list) and k) else 1)
        for comp in comps[: max(1, int(limit_k))]:
            try:
                with tempfile.TemporaryDirectory() as td:
                    p = Path(td)
                    (p / "solution.py").write_text(_assemble_solution(starter, comp), encoding="utf-8")
                    runner = "from solution import *\n" + tests_code
                    (p / "runner.py").write_text(runner, encoding="utf-8")
                    res = subprocess.run(["python3", str(p / "runner.py")], stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=max(1, int(timeout)) if timeout else 15)
                    if res.returncode == 0:
                        ok = True
                        break
            except Exception:
                ok = ok or False
        results[tid] = ok
        passed += (1 if ok else 0)
        total += 1

    out_path = Path(str(sp) + "_results.jsonl")
    with out_path.open("w", encoding="utf-8") as wf:
        for obj in rows:
            tid = str(obj.get("task_id", ""))
            wf.write(__import__("json").dumps({"task_id": tid, "completion": str(obj.get("completion", "")), "passed": bool(results.get(tid, False))}) + "\n")

    acc = (passed / max(1, total)) if total else 0.0
    return {"pass@k": acc}


    # Group completions per task id (first k completions considered)
    by_task: Dict[str, List[str]] = {}
    for obj in rows:
        tid = str(obj.get("task_id", ""))
        comp = str(obj.get("completion", ""))
        if not tid:
            continue
        by_task.setdefault(tid, []).append(comp)

    # Load dataset
    ds = load_dataset(dataset_name, split=split, trust_remote_code=True)
    index: Dict[str, Dict[str, Any]] = {}
    for it in ds:
        try:
            pid = str(it.get(id_field, ""))
            if pid:
                index[pid] = dict(it)
        except Exception:
            continue

    def _assemble_solution(starter: str, body: str) -> str:
        try:
            if not body.strip():
                return starter
            lines = starter.splitlines()
            for i, ln in enumerate(lines):
                if ln.strip() == 'pass':
                    indent = ln[: len(ln) - len(ln.lstrip())]
                    blines = [indent + b for b in body.splitlines()]
                    lines[i:i + 1] = blines
                    return "\n".join(lines)
            return starter + ("\n\n" + body if not starter.endswith("\n") else "\n" + body)
        except Exception:
            return starter

    results: Dict[str, bool] = {}
    passed = 0
    total = 0
    for tid, comps in by_task.items():
        orig = index.get(str(tid), {})
        starter = str(orig.get(starter_field, ""))
        tests_code = ""
        for tf in (tests_fields or []):
            if str(orig.get(tf, "")).strip():
                tests_code = str(orig.get(tf))
                break
        if not tests_code.strip():
            results[tid] = False
            continue
        ok = False
        limit_k = (k[0] if (isinstance(k, list) and k) else 1)
        for comp in comps[: max(1, int(limit_k))]:
            try:
                with tempfile.TemporaryDirectory() as td:


from __future__ import annotations

from pathlib import Path
from typing import Any, Dict, List, Optional, Protocol, Tuple

class BenchmarkAdapter(Protocol):
	"""Adapter interface to plug any benchmark into the universal harness.

	Required methods:
	- suite(): suite name used for traces and outputs
	- load_tasks(args): returns mapping task_id -> task payload
	- build_input(task_id, task): returns the user input string for the LLM
	- system_prompt(): returns the system prompt to use
	- sanitize_completion(text, prompt, task): cleans raw LLM output for evaluation
	- evaluate(samples_path, args, root, task_ids, tasks): returns (results_by_task, results_path or None)

	Optional methods:
	- env_modules(): list of python modules to fingerprint in env metadata
	- args_summary(args): dict of run args for run artifact
	"""

	def suite(self) -> str:
		...

	def load_tasks(self, args: Any) -> Dict[str, Dict[str, Any]]:
		...

	def build_input(self, task_id: str, task: Dict[str, Any]) -> str:
		...

	def system_prompt(self) -> str:
		...

	def sanitize_completion(self, text: str, prompt: str, task: Dict[str, Any]) -> str:
		...

	def evaluate(
		self,
		samples_path: Path,
		args: Any,
		root: Path,
		task_ids: List[str],
		tasks: Dict[str, Dict[str, Any]],
	) -> Tuple[Dict[str, bool], Optional[Path]]:
		...

	def env_modules(self) -> List[str]:
		return []

	def args_summary(self, args: Any) -> Dict[str, Any]:
		try:
			# Conservative default
			from agi_dw.bench.common.pipeline import build_codebody_args_summary  # type: ignore
			return build_codebody_args_summary(args)
		except Exception:
			return {}

	def build_predictions(
		self,
		samples_path: Path,
		args: Any,
		root: Path,
		task_ids: List[str],
		tasks: Dict[str, Dict[str, Any]],
	) -> Optional[Path]:
		"""Optional: build external evaluator predictions file from chosen samples. Return path or None."""
		return None

	def run_official_evaluator(
		self,
		predictions_path: Path,
		args: Any,
		root: Path,
		task_ids: List[str],
	) -> Optional[Path]:
		"""Optional: run suite's official evaluator given predictions; return results path if applicable."""
		return None

class PatchAdapterBase:
	"""Convenience base class for patch/diff style adapters.

	Provides default system prompt and sanitation for unified diff outputs.
	"""

	def system_prompt(self) -> str:
		return (
			"You are a code patch generator. Output ONLY a unified diff patch (git-style) that applies "
			"cleanly to fix the task. No commentary."
		)

	def sanitize_completion(self, text: str, prompt: str, task: Dict[str, Any]) -> str:
		try:
			patch_text = str(text or "").strip()
			# Keep only diff-looking outputs
			if any(h in patch_text for h in ("diff --git", "+++", "---")):
				return patch_text
		except Exception:
			pass
		return ""


BenchmarkAdapter

from __future__ import annotations

from pathlib import Path
from typing import Any, Dict, List, Optional, Protocol, Tuple

class BenchmarkAdapter(Protocol):
	"""Adapter interface to plug any benchmark into the universal harness.

	Required methods:
	- suite(): suite name used for traces and outputs
	- load_tasks(args): returns mapping task_id -> task payload
	- build_input(task_id, task): returns the user input string for the LLM
	- system_prompt(): returns the system prompt to use
	- sanitize_completion(text, prompt, task): cleans raw LLM output for evaluation
	- evaluate(samples_path, args, root, task_ids, tasks): returns (results_by_task, results_path or None)

	Optional methods:
	- env_modules(): list of python modules to fingerprint in env metadata
	- args_summary(args): dict of run args for run artifact
	"""

	def suite(self) -> str:
		...

	def load_tasks(self, args: Any) -> Dict[str, Dict[str, Any]]:
		...

	def build_input(self, task_id: str, task: Dict[str, Any]) -> str:
		...

	def system_prompt(self) -> str:
		...

	def sanitize_completion(self, text: str, prompt: str, task: Dict[str, Any]) -> str:
		...

	def evaluate(
		self,
		samples_path: Path,
		args: Any,
		root: Path,
		task_ids: List[str],
		tasks: Dict[str, Dict[str, Any]],
	) -> Tuple[Dict[str, bool], Optional[Path]]:
		...

	def env_modules(self) -> List[str]:
		return []

	def args_summary(self, args: Any) -> Dict[str, Any]:
		try:
			# Conservative default
			from agi_dw.bench.common.pipeline import build_codebody_args_summary  # type: ignore
			return build_codebody_args_summary(args)
		except Exception:
			return {}

	def build_predictions(
		self,
		samples_path: Path,
		args: Any,
		root: Path,
		task_ids: List[str],
		tasks: Dict[str, Dict[str, Any]],
	) -> Optional[Path]:
		"""Optional: build external evaluator predictions file from chosen samples. Return path or None."""
		return None

	def run_official_evaluator(
		self,
		predictions_path: Path,
		args: Any,
		root: Path,
		task_ids: List[str],
	) -> Optional[Path]:
		"""Optional: run suite's official evaluator given predictions; return results path if applicable."""
		return None

class PatchAdapterBase:
	"""Convenience base class for patch/diff style adapters.

	Provides default system prompt and sanitation for unified diff outputs.
	"""

	def system_prompt(self) -> str:
		return (
			"You are a code patch generator. Output ONLY a unified diff patch (git-style) that applies "
			"cleanly to fix the task. No commentary."
		)

	def sanitize_completion(self, text: str, prompt: str, task: Dict[str, Any]) -> str:
		try:
			patch_text = str(text or "").strip()
			# Keep only diff-looking outputs
			if any(h in patch_text for h in ("diff --git", "+++", "---")):
				return patch_text


PatchAdapterBase

		samples_path: Path,
		args: Any,
		root: Path,
		task_ids: List[str],
		tasks: Dict[str, Dict[str, Any]],
	) -> Optional[Path]:
		"""Optional: build external evaluator predictions file from chosen samples. Return path or None."""
		return None

	def run_official_evaluator(
		self,
		predictions_path: Path,
		args: Any,
		root: Path,
		task_ids: List[str],
	) -> Optional[Path]:
		"""Optional: run suite's official evaluator given predictions; return results path if applicable."""
		return None

class PatchAdapterBase:
	"""Convenience base class for patch/diff style adapters.

	Provides default system prompt and sanitation for unified diff outputs.
	"""

	def system_prompt(self) -> str:
		return (
			"You are a code patch generator. Output ONLY a unified diff patch (git-style) that applies "
			"cleanly to fix the task. No commentary."
		)

	def sanitize_completion(self, text: str, prompt: str, task: Dict[str, Any]) -> str:
		try:
			patch_text = str(text or "").strip()
			# Keep only diff-looking outputs
			if any(h in patch_text for h in ("diff --git", "+++", "---")):
				return patch_text
		except Exception:
			pass
		return ""


suite

from pathlib import Path
from typing import Any, Dict, List, Optional, Protocol, Tuple

class BenchmarkAdapter(Protocol):
	"""Adapter interface to plug any benchmark into the universal harness.

	Required methods:
	- suite(): suite name used for traces and outputs
	- load_tasks(args): returns mapping task_id -> task payload
	- build_input(task_id, task): returns the user input string for the LLM
	- system_prompt(): returns the system prompt to use
	- sanitize_completion(text, prompt, task): cleans raw LLM output for evaluation
	- evaluate(samples_path, args, root, task_ids, tasks): returns (results_by_task, results_path or None)

	Optional methods:
	- env_modules(): list of python modules to fingerprint in env metadata
	- args_summary(args): dict of run args for run artifact
	"""

	def suite(self) -> str:
		...

	def load_tasks(self, args: Any) -> Dict[str, Dict[str, Any]]:
		...

	def build_input(self, task_id: str, task: Dict[str, Any]) -> str:
		...

	def system_prompt(self) -> str:
		...

	def sanitize_completion(self, text: str, prompt: str, task: Dict[str, Any]) -> str:
		...

	def evaluate(
		self,
		samples_path: Path,
		args: Any,
		root: Path,
		task_ids: List[str],
		tasks: Dict[str, Dict[str, Any]],


class BenchmarkAdapter(Protocol):
	"""Adapter interface to plug any benchmark into the universal harness.

	Required methods:
	- suite(): suite name used for traces and outputs
	- load_tasks(args): returns mapping task_id -> task payload
	- build_input(task_id, task): returns the user input string for the LLM
	- system_prompt(): returns the system prompt to use
	- sanitize_completion(text, prompt, task): cleans raw LLM output for evaluation
	- evaluate(samples_path, args, root, task_ids, tasks): returns (results_by_task, results_path or None)

	Optional methods:
	- env_modules(): list of python modules to fingerprint in env metadata
	- args_summary(args): dict of run args for run artifact
	"""

	def suite(self) -> str:
		...

	def load_tasks(self, args: Any) -> Dict[str, Dict[str, Any]]:
		...

	def build_input(self, task_id: str, task: Dict[str, Any]) -> str:
		...

	def system_prompt(self) -> str:
		...

	def sanitize_completion(self, text: str, prompt: str, task: Dict[str, Any]) -> str:
		...

	def evaluate(
		self,
		samples_path: Path,
		args: Any,
		root: Path,
		task_ids: List[str],
		tasks: Dict[str, Dict[str, Any]],
	) -> Tuple[Dict[str, bool], Optional[Path]]:
		...


	Required methods:
	- suite(): suite name used for traces and outputs
	- load_tasks(args): returns mapping task_id -> task payload
	- build_input(task_id, task): returns the user input string for the LLM
	- system_prompt(): returns the system prompt to use
	- sanitize_completion(text, prompt, task): cleans raw LLM output for evaluation
	- evaluate(samples_path, args, root, task_ids, tasks): returns (results_by_task, results_path or None)

	Optional methods:
	- env_modules(): list of python modules to fingerprint in env metadata
	- args_summary(args): dict of run args for run artifact
	"""

	def suite(self) -> str:
		...

	def load_tasks(self, args: Any) -> Dict[str, Dict[str, Any]]:
		...

	def build_input(self, task_id: str, task: Dict[str, Any]) -> str:
		...

	def system_prompt(self) -> str:
		...

	def sanitize_completion(self, text: str, prompt: str, task: Dict[str, Any]) -> str:
		...

	def evaluate(
		self,
		samples_path: Path,
		args: Any,
		root: Path,
		task_ids: List[str],
		tasks: Dict[str, Dict[str, Any]],
	) -> Tuple[Dict[str, bool], Optional[Path]]:
		...

	def env_modules(self) -> List[str]:
		return []


		"""Optional: build external evaluator predictions file from chosen samples. Return path or None."""
		return None

	def run_official_evaluator(
		self,
		predictions_path: Path,
		args: Any,
		root: Path,
		task_ids: List[str],
	) -> Optional[Path]:
		"""Optional: run suite's official evaluator given predictions; return results path if applicable."""
		return None

class PatchAdapterBase:
	"""Convenience base class for patch/diff style adapters.

	Provides default system prompt and sanitation for unified diff outputs.
	"""

	def system_prompt(self) -> str:
		return (
			"You are a code patch generator. Output ONLY a unified diff patch (git-style) that applies "
			"cleanly to fix the task. No commentary."
		)

	def sanitize_completion(self, text: str, prompt: str, task: Dict[str, Any]) -> str:
		try:
			patch_text = str(text or "").strip()
			# Keep only diff-looking outputs
			if any(h in patch_text for h in ("diff --git", "+++", "---")):
				return patch_text
		except Exception:
			pass
		return ""


		args: Any,
		root: Path,
		task_ids: List[str],
	) -> Optional[Path]:
		"""Optional: run suite's official evaluator given predictions; return results path if applicable."""
		return None

class PatchAdapterBase:
	"""Convenience base class for patch/diff style adapters.

	Provides default system prompt and sanitation for unified diff outputs.
	"""

	def system_prompt(self) -> str:
		return (
			"You are a code patch generator. Output ONLY a unified diff patch (git-style) that applies "
			"cleanly to fix the task. No commentary."
		)

	def sanitize_completion(self, text: str, prompt: str, task: Dict[str, Any]) -> str:
		try:
			patch_text = str(text or "").strip()
			# Keep only diff-looking outputs
			if any(h in patch_text for h in ("diff --git", "+++", "---")):
				return patch_text
		except Exception:
			pass
		return ""


evaluate

	Optional methods:
	- env_modules(): list of python modules to fingerprint in env metadata
	- args_summary(args): dict of run args for run artifact
	"""

	def suite(self) -> str:
		...

	def load_tasks(self, args: Any) -> Dict[str, Dict[str, Any]]:
		...

	def build_input(self, task_id: str, task: Dict[str, Any]) -> str:
		...

	def system_prompt(self) -> str:
		...

	def sanitize_completion(self, text: str, prompt: str, task: Dict[str, Any]) -> str:
		...

	def evaluate(
		self,
		samples_path: Path,
		args: Any,
		root: Path,
		task_ids: List[str],
		tasks: Dict[str, Dict[str, Any]],
	) -> Tuple[Dict[str, bool], Optional[Path]]:
		...

	def env_modules(self) -> List[str]:
		return []

	def args_summary(self, args: Any) -> Dict[str, Any]:
		try:
			# Conservative default
			from agi_dw.bench.common.pipeline import build_codebody_args_summary  # type: ignore
			return build_codebody_args_summary(args)
		except Exception:
			return {}

	def build_predictions(
		self,
		samples_path: Path,
		args: Any,
		root: Path,
		task_ids: List[str],
		tasks: Dict[str, Dict[str, Any]],
	) -> Optional[Path]:


	def build_input(self, task_id: str, task: Dict[str, Any]) -> str:
		...

	def system_prompt(self) -> str:
		...

	def sanitize_completion(self, text: str, prompt: str, task: Dict[str, Any]) -> str:
		...

	def evaluate(
		self,
		samples_path: Path,
		args: Any,
		root: Path,
		task_ids: List[str],
		tasks: Dict[str, Dict[str, Any]],
	) -> Tuple[Dict[str, bool], Optional[Path]]:
		...

	def env_modules(self) -> List[str]:
		return []

	def args_summary(self, args: Any) -> Dict[str, Any]:
		try:
			# Conservative default
			from agi_dw.bench.common.pipeline import build_codebody_args_summary  # type: ignore
			return build_codebody_args_summary(args)
		except Exception:
			return {}

	def build_predictions(
		self,
		samples_path: Path,
		args: Any,
		root: Path,
		task_ids: List[str],
		tasks: Dict[str, Dict[str, Any]],
	) -> Optional[Path]:
		"""Optional: build external evaluator predictions file from chosen samples. Return path or None."""
		return None


	def system_prompt(self) -> str:
		...

	def sanitize_completion(self, text: str, prompt: str, task: Dict[str, Any]) -> str:
		...

	def evaluate(
		self,
		samples_path: Path,
		args: Any,
		root: Path,
		task_ids: List[str],
		tasks: Dict[str, Dict[str, Any]],
	) -> Tuple[Dict[str, bool], Optional[Path]]:
		...

	def env_modules(self) -> List[str]:
		return []

	def args_summary(self, args: Any) -> Dict[str, Any]:
		try:
			# Conservative default
			from agi_dw.bench.common.pipeline import build_codebody_args_summary  # type: ignore
			return build_codebody_args_summary(args)
		except Exception:
			return {}

	def build_predictions(
		self,
		samples_path: Path,
		args: Any,
		root: Path,
		task_ids: List[str],
		tasks: Dict[str, Dict[str, Any]],
	) -> Optional[Path]:
		"""Optional: build external evaluator predictions file from chosen samples. Return path or None."""
		return None

	def run_official_evaluator(
		self,
		predictions_path: Path,
		args: Any,
		root: Path,
		task_ids: List[str],
	) -> Optional[Path]:
		"""Optional: run suite's official evaluator given predictions; return results path if applicable."""


		self,
		samples_path: Path,
		args: Any,
		root: Path,
		task_ids: List[str],
		tasks: Dict[str, Dict[str, Any]],
	) -> Tuple[Dict[str, bool], Optional[Path]]:
		...

	def env_modules(self) -> List[str]:
		return []

	def args_summary(self, args: Any) -> Dict[str, Any]:
		try:
			# Conservative default
			from agi_dw.bench.common.pipeline import build_codebody_args_summary  # type: ignore
			return build_codebody_args_summary(args)
		except Exception:
			return {}

	def build_predictions(
		self,
		samples_path: Path,
		args: Any,
		root: Path,
		task_ids: List[str],
		tasks: Dict[str, Dict[str, Any]],
	) -> Optional[Path]:
		"""Optional: build external evaluator predictions file from chosen samples. Return path or None."""
		return None

	def run_official_evaluator(
		self,
		predictions_path: Path,
		args: Any,
		root: Path,
		task_ids: List[str],
	) -> Optional[Path]:
		"""Optional: run suite's official evaluator given predictions; return results path if applicable."""
		return None

class PatchAdapterBase:
	"""Convenience base class for patch/diff style adapters.

	Provides default system prompt and sanitation for unified diff outputs.
	"""

	def system_prompt(self) -> str:
		return (


	def args_summary(self, args: Any) -> Dict[str, Any]:
		try:
			# Conservative default
			from agi_dw.bench.common.pipeline import build_codebody_args_summary  # type: ignore
			return build_codebody_args_summary(args)
		except Exception:
			return {}

	def build_predictions(
		self,
		samples_path: Path,
		args: Any,
		root: Path,
		task_ids: List[str],
		tasks: Dict[str, Dict[str, Any]],
	) -> Optional[Path]:
		"""Optional: build external evaluator predictions file from chosen samples. Return path or None."""
		return None

	def run_official_evaluator(
		self,
		predictions_path: Path,
		args: Any,
		root: Path,
		task_ids: List[str],
	) -> Optional[Path]:
		"""Optional: run suite's official evaluator given predictions; return results path if applicable."""
		return None

class PatchAdapterBase:
	"""Convenience base class for patch/diff style adapters.

	Provides default system prompt and sanitation for unified diff outputs.
	"""

	def system_prompt(self) -> str:
		return (
			"You are a code patch generator. Output ONLY a unified diff patch (git-style) that applies "
			"cleanly to fix the task. No commentary."
		)

	def sanitize_completion(self, text: str, prompt: str, task: Dict[str, Any]) -> str:
		try:
			patch_text = str(text or "").strip()
			# Keep only diff-looking outputs
			if any(h in patch_text for h in ("diff --git", "+++", "---")):
				return patch_text


from __future__ import annotations

from pathlib import Path
from typing import Any, Dict, Tuple, Callable, Optional, List

def load_registry(root: Path | None = None) -> Dict[str, Any]:
	root = root or Path(__file__).resolve().parents[2]
	# Root already points to the repo root (e.g., /data/agiattempt/agi_dw)
	# so the registry lives under bench/registry.json relative to it.
	reg_path = Path(root / "bench" / "registry.json")
	try:
		obj = __import__("json").loads(reg_path.read_text(encoding="utf-8"))
		return obj if isinstance(obj, dict) else {"version": 1, "suites": {}}
	except Exception:
		return {"version": 1, "suites": {}}

def resolve_entrypoint(fn_path: str) -> Callable[[Any], int]:
	"""Resolve 'module.sub:func' into a callable."""
	mod_str, func_str = fn_path.split(":", 1)
	mod = __import__(mod_str, fromlist=[func_str])
	fn = getattr(mod, func_str)
	return fn  # type: ignore[return-value]

def resolve_object(obj_path: Optional[str]) -> Optional[Any]:
	if not obj_path:
		return None
	mod_str, name = obj_path.split(":", 1)
	mod = __import__(mod_str, fromlist=[name])
	return getattr(mod, name)

def get_suite_config(suite: str, root: Path | None = None) -> Tuple[str, Dict[str, Any]]:
	reg = load_registry(root)
	s = reg.get("suites", {}).get(suite, {})
	mode = str(s.get("mode", "runner"))
	return mode, s

def load_hf_dataset(spec: Dict[str, Any]) -> List[Dict[str, Any]]:
	"""Load a huggingface dataset using datasets.load_dataset with trust_remote_code.

	Returns a list of rows (dicts).
	"""
	try:
		from datasets import load_dataset  # type: ignore
		# Newer datasets lib disallows trust_remote_code; pass minimal args
		ds = load_dataset(str(spec.get("dataset_name")), split=str(spec.get("split", "test")))
		return [dict(it) for it in ds]
	except Exception:
		return []

def make_problems_from_hf(rows: List[Dict[str, Any]], id_field: str, prompt_field: Optional[str], prompt_template: Optional[str]) -> Dict[str, Dict[str, Any]]:
	problems: Dict[str, Dict[str, Any]] = {}
	for it in rows:
		try:
			tid = str(it.get(id_field, ""))
			if not tid:
				continue
			if prompt_template:
				try:
					prompt_val = prompt_template.format(**it)
				except Exception:
					prompt_val = str(it.get(prompt_field or "prompt", ""))
			else:
				prompt_val = str(it.get(prompt_field or "prompt", ""))
			problems[tid] = {"prompt": prompt_val}
		except Exception:
			continue
	return problems


from __future__ import annotations

from pathlib import Path
from typing import Any, Dict, Tuple, Callable, Optional, List

def load_registry(root: Path | None = None) -> Dict[str, Any]:
	root = root or Path(__file__).resolve().parents[2]
	# Root already points to the repo root (e.g., /data/agiattempt/agi_dw)
	# so the registry lives under bench/registry.json relative to it.
	reg_path = Path(root / "bench" / "registry.json")
	try:
		obj = __import__("json").loads(reg_path.read_text(encoding="utf-8"))
		return obj if isinstance(obj, dict) else {"version": 1, "suites": {}}
	except Exception:
		return {"version": 1, "suites": {}}

def resolve_entrypoint(fn_path: str) -> Callable[[Any], int]:
	"""Resolve 'module.sub:func' into a callable."""
	mod_str, func_str = fn_path.split(":", 1)
	mod = __import__(mod_str, fromlist=[func_str])
	fn = getattr(mod, func_str)
	return fn  # type: ignore[return-value]

def resolve_object(obj_path: Optional[str]) -> Optional[Any]:
	if not obj_path:
		return None
	mod_str, name = obj_path.split(":", 1)
	mod = __import__(mod_str, fromlist=[name])
	return getattr(mod, name)

def get_suite_config(suite: str, root: Path | None = None) -> Tuple[str, Dict[str, Any]]:
	reg = load_registry(root)


from __future__ import annotations

from pathlib import Path
from typing import Any, Dict, Tuple, Callable, Optional, List

def load_registry(root: Path | None = None) -> Dict[str, Any]:
	root = root or Path(__file__).resolve().parents[2]
	# Root already points to the repo root (e.g., /data/agiattempt/agi_dw)
	# so the registry lives under bench/registry.json relative to it.
	reg_path = Path(root / "bench" / "registry.json")
	try:
		obj = __import__("json").loads(reg_path.read_text(encoding="utf-8"))
		return obj if isinstance(obj, dict) else {"version": 1, "suites": {}}
	except Exception:
		return {"version": 1, "suites": {}}

def resolve_entrypoint(fn_path: str) -> Callable[[Any], int]:
	"""Resolve 'module.sub:func' into a callable."""
	mod_str, func_str = fn_path.split(":", 1)
	mod = __import__(mod_str, fromlist=[func_str])
	fn = getattr(mod, func_str)
	return fn  # type: ignore[return-value]

def resolve_object(obj_path: Optional[str]) -> Optional[Any]:
	if not obj_path:
		return None
	mod_str, name = obj_path.split(":", 1)
	mod = __import__(mod_str, fromlist=[name])
	return getattr(mod, name)

def get_suite_config(suite: str, root: Path | None = None) -> Tuple[str, Dict[str, Any]]:
	reg = load_registry(root)
	s = reg.get("suites", {}).get(suite, {})
	mode = str(s.get("mode", "runner"))
	return mode, s

def load_hf_dataset(spec: Dict[str, Any]) -> List[Dict[str, Any]]:
	"""Load a huggingface dataset using datasets.load_dataset with trust_remote_code.


def load_registry(root: Path | None = None) -> Dict[str, Any]:
	root = root or Path(__file__).resolve().parents[2]
	# Root already points to the repo root (e.g., /data/agiattempt/agi_dw)
	# so the registry lives under bench/registry.json relative to it.
	reg_path = Path(root / "bench" / "registry.json")
	try:
		obj = __import__("json").loads(reg_path.read_text(encoding="utf-8"))
		return obj if isinstance(obj, dict) else {"version": 1, "suites": {}}
	except Exception:
		return {"version": 1, "suites": {}}

def resolve_entrypoint(fn_path: str) -> Callable[[Any], int]:
	"""Resolve 'module.sub:func' into a callable."""
	mod_str, func_str = fn_path.split(":", 1)
	mod = __import__(mod_str, fromlist=[func_str])
	fn = getattr(mod, func_str)
	return fn  # type: ignore[return-value]

def resolve_object(obj_path: Optional[str]) -> Optional[Any]:
	if not obj_path:
		return None
	mod_str, name = obj_path.split(":", 1)
	mod = __import__(mod_str, fromlist=[name])
	return getattr(mod, name)

def get_suite_config(suite: str, root: Path | None = None) -> Tuple[str, Dict[str, Any]]:
	reg = load_registry(root)
	s = reg.get("suites", {}).get(suite, {})
	mode = str(s.get("mode", "runner"))
	return mode, s

def load_hf_dataset(spec: Dict[str, Any]) -> List[Dict[str, Any]]:
	"""Load a huggingface dataset using datasets.load_dataset with trust_remote_code.

	Returns a list of rows (dicts).
	"""
	try:
		from datasets import load_dataset  # type: ignore
		# Newer datasets lib disallows trust_remote_code; pass minimal args
		ds = load_dataset(str(spec.get("dataset_name")), split=str(spec.get("split", "test")))
		return [dict(it) for it in ds]
	except Exception:


	except Exception:
		return {"version": 1, "suites": {}}

def resolve_entrypoint(fn_path: str) -> Callable[[Any], int]:
	"""Resolve 'module.sub:func' into a callable."""
	mod_str, func_str = fn_path.split(":", 1)
	mod = __import__(mod_str, fromlist=[func_str])
	fn = getattr(mod, func_str)
	return fn  # type: ignore[return-value]

def resolve_object(obj_path: Optional[str]) -> Optional[Any]:
	if not obj_path:
		return None
	mod_str, name = obj_path.split(":", 1)
	mod = __import__(mod_str, fromlist=[name])
	return getattr(mod, name)

def get_suite_config(suite: str, root: Path | None = None) -> Tuple[str, Dict[str, Any]]:
	reg = load_registry(root)
	s = reg.get("suites", {}).get(suite, {})
	mode = str(s.get("mode", "runner"))
	return mode, s

def load_hf_dataset(spec: Dict[str, Any]) -> List[Dict[str, Any]]:
	"""Load a huggingface dataset using datasets.load_dataset with trust_remote_code.

	Returns a list of rows (dicts).
	"""
	try:
		from datasets import load_dataset  # type: ignore
		# Newer datasets lib disallows trust_remote_code; pass minimal args
		ds = load_dataset(str(spec.get("dataset_name")), split=str(spec.get("split", "test")))
		return [dict(it) for it in ds]
	except Exception:
		return []

def make_problems_from_hf(rows: List[Dict[str, Any]], id_field: str, prompt_field: Optional[str], prompt_template: Optional[str]) -> Dict[str, Dict[str, Any]]:
	problems: Dict[str, Dict[str, Any]] = {}
	for it in rows:
		try:


	mod = __import__(mod_str, fromlist=[func_str])
	fn = getattr(mod, func_str)
	return fn  # type: ignore[return-value]

def resolve_object(obj_path: Optional[str]) -> Optional[Any]:
	if not obj_path:
		return None
	mod_str, name = obj_path.split(":", 1)
	mod = __import__(mod_str, fromlist=[name])
	return getattr(mod, name)

def get_suite_config(suite: str, root: Path | None = None) -> Tuple[str, Dict[str, Any]]:
	reg = load_registry(root)
	s = reg.get("suites", {}).get(suite, {})
	mode = str(s.get("mode", "runner"))
	return mode, s

def load_hf_dataset(spec: Dict[str, Any]) -> List[Dict[str, Any]]:
	"""Load a huggingface dataset using datasets.load_dataset with trust_remote_code.

	Returns a list of rows (dicts).
	"""
	try:
		from datasets import load_dataset  # type: ignore
		# Newer datasets lib disallows trust_remote_code; pass minimal args
		ds = load_dataset(str(spec.get("dataset_name")), split=str(spec.get("split", "test")))
		return [dict(it) for it in ds]
	except Exception:
		return []

def make_problems_from_hf(rows: List[Dict[str, Any]], id_field: str, prompt_field: Optional[str], prompt_template: Optional[str]) -> Dict[str, Dict[str, Any]]:
	problems: Dict[str, Dict[str, Any]] = {}
	for it in rows:
		try:
			tid = str(it.get(id_field, ""))
			if not tid:
				continue
			if prompt_template:
				try:
					prompt_val = prompt_template.format(**it)
				except Exception:
					prompt_val = str(it.get(prompt_field or "prompt", ""))
			else:
				prompt_val = str(it.get(prompt_field or "prompt", ""))
			problems[tid] = {"prompt": prompt_val}
		except Exception:
			continue
	return problems


	reg = load_registry(root)
	s = reg.get("suites", {}).get(suite, {})
	mode = str(s.get("mode", "runner"))
	return mode, s

def load_hf_dataset(spec: Dict[str, Any]) -> List[Dict[str, Any]]:
	"""Load a huggingface dataset using datasets.load_dataset with trust_remote_code.

	Returns a list of rows (dicts).
	"""
	try:
		from datasets import load_dataset  # type: ignore
		# Newer datasets lib disallows trust_remote_code; pass minimal args
		ds = load_dataset(str(spec.get("dataset_name")), split=str(spec.get("split", "test")))
		return [dict(it) for it in ds]
	except Exception:
		return []

def make_problems_from_hf(rows: List[Dict[str, Any]], id_field: str, prompt_field: Optional[str], prompt_template: Optional[str]) -> Dict[str, Dict[str, Any]]:
	problems: Dict[str, Dict[str, Any]] = {}
	for it in rows:
		try:
			tid = str(it.get(id_field, ""))
			if not tid:
				continue
			if prompt_template:
				try:
					prompt_val = prompt_template.format(**it)
				except Exception:
					prompt_val = str(it.get(prompt_field or "prompt", ""))
			else:
				prompt_val = str(it.get(prompt_field or "prompt", ""))
			problems[tid] = {"prompt": prompt_val}
		except Exception:
			continue
	return problems


from __future__ import annotations

import json
import logging
import time
from concurrent.futures import ThreadPoolExecutor, as_completed
from pathlib import Path
import hashlib
import os
from typing import Any, Dict, List, Optional, Callable, Tuple
from agi_dw.bench.common.pipeline import (
    try_generate_python_body,
    looks_like_python_code,
    attach_adapter_from_args,
    parse_k_list,
    evaluate_samples,
    read_jsonl as _read_jsonl_shared,
    build_results_by_task as _build_results_by_task_shared,
    dedupe_by_passed as _dedupe_by_passed_shared,
    shard_task_ids,
    compute_sharded_outpath,
    write_trace,
    repair_code_failures,
    write_suite_out,
    write_run_artifact,
    evaluate_candidate_signals,
    maybe_self_refine_body,
    wm_rerank_candidates,
	get_python_version,
	build_env_fingerprint,
	is_primary_process,
	seed_everything,
	get_shard_tag,
	init_trace_paths,
	init_suite_temp_paths,
	read_pass_cache,
	update_pass_cache,
	build_generation_params,
	summarize_telemetry,
	online_update_and_rerun_failures,
	emit_training_datasets,
)
from agi_dw.core.utils.bench_utils import sanitize_humaneval_completion as _sanitize_he, strip_fences as _strip  # type: ignore
from agi_dw.core.ops.tracing import trace_span  # type: ignore

def _safe_int(val: Any, default: int) -> int:
	try:
		return int(val)
	except Exception:
		return int(default)

def _safe_float(val: Any, default: float) -> float:
	try:
		return float(val)
	except Exception:
		return float(default)

def _read_jsonl(path: Path) -> List[Dict[str, Any]]:
	rows: List[Dict[str, Any]] = []
	if not path.exists():
		return rows
	with path.open("r", encoding="utf-8") as f:
		for line in f:
			li = line.strip()
			if not li:
				continue
			try:
				rows.append(json.loads(li))
			except Exception:
				continue
	return rows

# ---- Shared helpers (modularization within this file) ----
def _get_default_workers() -> int:
    try:
        _cpu_cnt = int((os.cpu_count() or 4))
        return min(16, max(1, _cpu_cnt))
    except Exception:
        return 4

DEFAULT_WORKERS = _get_default_workers()

def run_registry_benchmark(args: Any) -> int:
	"""Dispatch a benchmark by looking it up in the registry file.

	Expected args: .suite, .out, plus arbitrary flags forwarded to underlying entrypoint.
	Applies hygiene defaults if present in registry.
	"""
	from pathlib import Path as _Path
	from agi_dw.bench.common.registry import get_suite_config, resolve_entrypoint  # type: ignore
	root = _Path(__file__).resolve().parents[2]
	# Ensure a default model for code-body suites if caller did not provide one
	try:
		if not hasattr(args, "model") or getattr(args, "model", None) is None:
			import os as _os  # type: ignore
			setattr(args, "model", _os.environ.get("AGI_DEFAULT_MODEL", "meta-llama/Llama-3.1-8B-Instruct"))
	except Exception:
		pass
	mode, cfg = get_suite_config(str(getattr(args, "suite", "")), root)
	with trace_span("registry_benchmark", {"suite": str(getattr(args, "suite", "")), "mode": mode}):
		pass
	# Apply suite defaults if provided (do not override user-specified values)
	try:
		_defaults = cfg.get("defaults", {}) or {}
		if isinstance(_defaults, dict):
			for _k, _v in _defaults.items():
				if not hasattr(args, _k) or getattr(args, _k, None) is None:
					setattr(args, _k, _v)
	except Exception:
		pass
	# Apply hygiene defaults unless user overrides
	try:
		if getattr(args, "benchmark_mode", None) is None:
			setattr(args, "benchmark_mode", bool(cfg.get("hygiene", {}).get("benchmark_mode_default", True)))
		if getattr(args, "emit_indirect", None) is None:
			setattr(args, "emit_indirect", bool(cfg.get("hygiene", {}).get("emit_indirect_default", True)))
	except Exception:
		pass
	# Maximize AGI capabilities by enabling Plan–Act–Verify and learning defaults if not set
	try:
		if getattr(args, "precheck", None) is None:
			setattr(args, "precheck", True)
		if getattr(args, "pav_enable", None) is None:
			setattr(args, "pav_enable", True)
		if getattr(args, "self_eval", None) is None:
			setattr(args, "self_eval", True)
		if getattr(args, "online_update", None) is None:
			setattr(args, "online_update", True)
		# Provide a default WM model path if present on disk
		if getattr(args, "wm_model", None) is None:
			_wm_path = root / "models" / "wm_mlp" / "wm_mlp.joblib"
			if _wm_path.exists():
				setattr(args, "wm_model", str(_wm_path))
		if getattr(args, "wm_threshold", None) is None:
			setattr(args, "wm_threshold", 0.6)
	except Exception:
		pass
	if mode == "runner":
		ent = str(cfg.get("entrypoint", ""))
		if not ent:
			print(__import__("json").dumps({"ok": False, "error": "suite_not_in_registry", "suite": str(getattr(args, "suite", ""))}))
			return 2
		fn = resolve_entrypoint(ent)
		return int(fn(args))
	if mode == "code_body":
		with trace_span("code_body_setup", {"suite": str(getattr(args, "suite"))}):
			pass
		# Resolve components
		from agi_dw.bench.common.harness import run_codebody_suite as _run_codebody_suite  # type: ignore
		from agi_dw.core.prompts.bench import build_prompt as _build_prompt  # type: ignore
		loader = cfg.get("loader")
		evaluator = cfg.get("evaluator")
		evalplus = cfg.get("evalplus")
		prompt_key = str(cfg.get("prompt_builder_key", ""))
		problems: Dict[str, Dict[str, Any]] = {}
		_rows: List[Dict[str, Any]] = []
		try:
			from agi_dw.bench.common.registry import resolve_object as _res, load_hf_dataset, make_problems_from_hf  # type: ignore
			ld = _res(loader)
			if ld is not None:
				obj = ld()
				if isinstance(obj, dict):
					# human-eval style
					problems = {str(k): {"prompt": str(v.get("prompt", "")) if isinstance(v, dict) else str(v)} for k, v in obj.items()}
				elif isinstance(obj, list):
					# list of dicts; defer problem construction to optional transform or HF helper
					_rows = [dict(it) for it in obj if isinstance(it, dict)]
		except Exception:
			problems = {}
		# Fallback: try HF dataset loading if loader spec absent but hf config present
		try:
			_loader_args = cfg.get("loader_args") or {}
			if (not problems) and (not _rows) and _loader_args.get("dataset_name"):
				_rows = load_hf_dataset(_loader_args)
		except Exception:
			pass
		# Optional transform hook: rows -> problems
		try:
			_transform_path = cfg.get("transform")
			if _transform_path and _rows:
				from agi_dw.bench.common.registry import resolve_object as _res2  # type: ignore
				_tf = _res2(_transform_path)
				if callable(_tf):
					_tp = _tf(_rows)
					if isinstance(_tp, dict):
						problems = {str(k): {"prompt": str(v.get("prompt", "")) if isinstance(v, dict) else str(v)} for k, v in _tp.items()}
		except Exception:
			pass
		# If still no problems but have rows, use generic HF mapping
		if (not problems) and _rows:
			try:
				_loader_args = cfg.get("loader_args") or {}
				problems = make_problems_from_hf(_rows, id_field=str(_loader_args.get("id_field", "task_id")), prompt_field=str(_loader_args.get("prompt_field", "prompt")), prompt_template=cfg.get("prompt_template"))
			except Exception:
				problems = {}
		# Sanitizer
		sanitize = None
		try:
			san_path = cfg.get("sanitizer")
			if san_path:
				from agi_dw.bench.common.registry import resolve_object as _res  # type: ignore
				san_fn = _res(san_path)
				opts = cfg.get("sanitizer_options") or {}
				if san_fn:
					def _san(text: str, prompt: str) -> str:
						try:
							return san_fn(_strip(text), prompt, opts)  # type: ignore
						except TypeError:
							return san_fn(_strip(text), prompt)  # type: ignore
					sanitize = _san
		except Exception:
			sanitize = None
		if sanitize is None:
			def _san(text: str, prompt: str) -> str:
				return _strip(text)
			sanitize = _san
		# Evaluators
		try:
			from agi_dw.bench.common.registry import resolve_object as _res  # type: ignore
			_eval = _res(evaluator)
			_evalplus = _res(evalplus)
		except Exception:
			_eval, _evalplus = None, None
		# If evaluator_args provided in registry, wrap evaluator with those kwargs
		try:
			eval_args = cfg.get("evaluator_args") or None
			if _eval is not None and isinstance(eval_args, dict):
				_base_eval = _eval
				def _wrapped_eval(sample_file: str, k: List[int], n_workers: int, timeout: float, ignore_incomplete: bool):  # type: ignore
					return _base_eval(sample_file, k=k, n_workers=n_workers, timeout=timeout, ignore_incomplete=ignore_incomplete, **eval_args)
				_eval = _wrapped_eval  # type: ignore
			# Fallback: if no evaluator resolved but we have args for generic solution-eval, bind it
			if _eval is None and isinstance(eval_args, dict) and mode == "code_body":
				try:
					from agi_dw.bench.common.evaluators import evaluate_solution as _generic_eval  # type: ignore
					def _wrapped_generic(sample_file: str, k: List[int], n_workers: int, timeout: float, ignore_incomplete: bool):  # type: ignore
						return _generic_eval(sample_file, k=k, n_workers=n_workers, timeout=timeout, ignore_incomplete=ignore_incomplete, **eval_args)
					_eval = _wrapped_generic  # type: ignore
				except Exception:
					pass
		except Exception:
			pass
		with trace_span("code_body_run", {"suite": str(getattr(args, "suite"))}):
			return int(_run_codebody_suite(args=args, suite=str(getattr(args, "suite")), problems=problems, prompt_builder_key=prompt_key or str(getattr(args, "suite")), sanitize_fn=sanitize, evaluate_functional_correctness=_eval, evalplus_evaluate=_evalplus))
	if mode == "patch":
		with trace_span("patch_setup", {"suite": str(getattr(args, "suite"))}):
			pass
		# Build an adapter on the fly using registry fields
		from agi_dw.bench.common.universal_harness import run_with_adapter  # type: ignore
		from agi_dw.bench.common.adapters import PatchAdapterBase  # type: ignore
		from agi_dw.bench.common.registry import resolve_object as _res, load_hf_dataset  # type: ignore
		loader = cfg.get("loader")
		loader_args = cfg.get("loader_args") or {}
		id_field = str(cfg.get("id_field", "instance_id"))
		input_tmpl = str(cfg.get("input_template", ""))
		eval_entry = cfg.get("evaluator_entrypoint")
		class _DynPatchAdapter(PatchAdapterBase):
			def suite(self) -> str:
				return str(getattr(args, "suite"))
			def env_modules(self) -> List[str]:
				return list(cfg.get("env_modules", []))
			def load_tasks(self, _args: Any) -> Dict[str, Dict[str, Any]]:
				rows: List[Dict[str, Any]] = []
				ld = _res(loader)
				if ld is not None:
					try:
						obj = ld(**loader_args) if loader_args else ld()
						# swebench returns a list
						if isinstance(obj, list):
							rows = [dict(it) for it in obj]
						else:
							rows = []
					except Exception:
						rows = []
				if (not rows) and loader_args.get("dataset_name"):
					rows = load_hf_dataset(loader_args)
				return {str(it.get(id_field, "")): dict(it) for it in rows if str(it.get(id_field, "")).strip()}
			def build_input(self, task_id: str, task: Dict[str, Any]) -> str:
				try:
					return input_tmpl.format(**task)
				except Exception:
					return input_tmpl
			def build_predictions(self, samples_path: Path, _args: Any, root: Path, task_ids: List[str], tasks: Dict[str, Dict[str, Any]]) -> Optional[Path]:
				from agi_dw.bench.common.pipeline import read_jsonl  # type: ignore
				import json as _json
				preds_file = Path(root / "data" / "bench" / "tmp" / (str(getattr(args, "suite")) + "_preds.jsonl"))
				preds_file.parent.mkdir(parents=True, exist_ok=True)
				with preds_file.open("w", encoding="utf-8") as pf:
					for obj in read_jsonl(samples_path):
						tid = str(obj.get("task_id", ""))
						comp = str(obj.get("completion", ""))
						if not tid:
							continue
						pf.write(_json.dumps({"instance_id": tid, "model_patch": comp, "patch": comp, "model_name_or_path": str(getattr(args, "model", "unknown"))}) + "\n")
				return preds_file
			def run_official_evaluator(self, predictions_path: Path, _args: Any, root: Path, task_ids: List[str]) -> Optional[Path]:
				try:
					entry = _res(eval_entry)
					if entry is None:
						return None
					# Call entrypoint akin to swebench main
					entry(
						dataset_name=str(loader_args.get("dataset_name")),
						split=str(loader_args.get("split", "test")),
						instance_ids=task_ids,
						predictions_path=str(predictions_path),
						max_workers=max(1, int(getattr(args, "max_workers", 1) or 1)),
						force_rebuild=False,
						cache_level="env",
						clean=False,
						open_file_limit=2048,
						run_id="agi_dw",
						timeout=int(getattr(args, "timeout", 1800) or 1800),
						namespace=str(getattr(args, "suite")),
						rewrite_reports=False,
						modal=False,
						instance_image_tag="latest",
						env_image_tag="latest",
						report_dir=str((root / "data" / "bench" / "tmp").resolve()),
					)
					return None
				except Exception:
					return None
		adapter = _DynPatchAdapter()
		with trace_span("patch_run", {"suite": str(getattr(args, "suite"))}):
			return int(run_with_adapter(args, adapter))
	print(__import__("json").dumps({"ok": False, "error": "unsupported_registry_mode", "mode": mode}))
	return 2


    maybe_self_refine_body,
    wm_rerank_candidates,
	get_python_version,
	build_env_fingerprint,
	is_primary_process,
	seed_everything,
	get_shard_tag,
	init_trace_paths,
	init_suite_temp_paths,
	read_pass_cache,
	update_pass_cache,
	build_generation_params,
	summarize_telemetry,
	online_update_and_rerun_failures,
	emit_training_datasets,
)
from agi_dw.core.utils.bench_utils import sanitize_humaneval_completion as _sanitize_he, strip_fences as _strip  # type: ignore
from agi_dw.core.ops.tracing import trace_span  # type: ignore

def _safe_int(val: Any, default: int) -> int:
	try:
		return int(val)
	except Exception:
		return int(default)

def _safe_float(val: Any, default: float) -> float:
	try:
		return float(val)
	except Exception:
		return float(default)

def _read_jsonl(path: Path) -> List[Dict[str, Any]]:
	rows: List[Dict[str, Any]] = []
	if not path.exists():
		return rows
	with path.open("r", encoding="utf-8") as f:
		for line in f:
			li = line.strip()
			if not li:
				continue
			try:
				rows.append(json.loads(li))


	init_trace_paths,
	init_suite_temp_paths,
	read_pass_cache,
	update_pass_cache,
	build_generation_params,
	summarize_telemetry,
	online_update_and_rerun_failures,
	emit_training_datasets,
)
from agi_dw.core.utils.bench_utils import sanitize_humaneval_completion as _sanitize_he, strip_fences as _strip  # type: ignore
from agi_dw.core.ops.tracing import trace_span  # type: ignore

def _safe_int(val: Any, default: int) -> int:
	try:
		return int(val)
	except Exception:
		return int(default)

def _safe_float(val: Any, default: float) -> float:
	try:
		return float(val)
	except Exception:
		return float(default)

def _read_jsonl(path: Path) -> List[Dict[str, Any]]:
	rows: List[Dict[str, Any]] = []
	if not path.exists():
		return rows
	with path.open("r", encoding="utf-8") as f:
		for line in f:
			li = line.strip()
			if not li:
				continue
			try:
				rows.append(json.loads(li))
			except Exception:
				continue
	return rows

# ---- Shared helpers (modularization within this file) ----
def _get_default_workers() -> int:


	emit_training_datasets,
)
from agi_dw.core.utils.bench_utils import sanitize_humaneval_completion as _sanitize_he, strip_fences as _strip  # type: ignore
from agi_dw.core.ops.tracing import trace_span  # type: ignore

def _safe_int(val: Any, default: int) -> int:
	try:
		return int(val)
	except Exception:
		return int(default)

def _safe_float(val: Any, default: float) -> float:
	try:
		return float(val)
	except Exception:
		return float(default)

def _read_jsonl(path: Path) -> List[Dict[str, Any]]:
	rows: List[Dict[str, Any]] = []
	if not path.exists():
		return rows
	with path.open("r", encoding="utf-8") as f:
		for line in f:
			li = line.strip()
			if not li:
				continue
			try:
				rows.append(json.loads(li))
			except Exception:
				continue
	return rows

# ---- Shared helpers (modularization within this file) ----
def _get_default_workers() -> int:
    try:
        _cpu_cnt = int((os.cpu_count() or 4))
        return min(16, max(1, _cpu_cnt))
    except Exception:
        return 4

DEFAULT_WORKERS = _get_default_workers()

def run_registry_benchmark(args: Any) -> int:
	"""Dispatch a benchmark by looking it up in the registry file.

	Expected args: .suite, .out, plus arbitrary flags forwarded to underlying entrypoint.
	Applies hygiene defaults if present in registry.
	"""
	from pathlib import Path as _Path


		return float(default)

def _read_jsonl(path: Path) -> List[Dict[str, Any]]:
	rows: List[Dict[str, Any]] = []
	if not path.exists():
		return rows
	with path.open("r", encoding="utf-8") as f:
		for line in f:
			li = line.strip()
			if not li:
				continue
			try:
				rows.append(json.loads(li))
			except Exception:
				continue
	return rows

# ---- Shared helpers (modularization within this file) ----
def _get_default_workers() -> int:
    try:
        _cpu_cnt = int((os.cpu_count() or 4))
        return min(16, max(1, _cpu_cnt))
    except Exception:
        return 4

DEFAULT_WORKERS = _get_default_workers()

def run_registry_benchmark(args: Any) -> int:
	"""Dispatch a benchmark by looking it up in the registry file.

	Expected args: .suite, .out, plus arbitrary flags forwarded to underlying entrypoint.
	Applies hygiene defaults if present in registry.
	"""
	from pathlib import Path as _Path
	from agi_dw.bench.common.registry import get_suite_config, resolve_entrypoint  # type: ignore
	root = _Path(__file__).resolve().parents[2]
	# Ensure a default model for code-body suites if caller did not provide one
	try:
		if not hasattr(args, "model") or getattr(args, "model", None) is None:
			import os as _os  # type: ignore
			setattr(args, "model", _os.environ.get("AGI_DEFAULT_MODEL", "meta-llama/Llama-3.1-8B-Instruct"))
	except Exception:
		pass


			if not li:
				continue
			try:
				rows.append(json.loads(li))
			except Exception:
				continue
	return rows

# ---- Shared helpers (modularization within this file) ----
def _get_default_workers() -> int:
    try:
        _cpu_cnt = int((os.cpu_count() or 4))
        return min(16, max(1, _cpu_cnt))
    except Exception:
        return 4

DEFAULT_WORKERS = _get_default_workers()

def run_registry_benchmark(args: Any) -> int:
	"""Dispatch a benchmark by looking it up in the registry file.

	Expected args: .suite, .out, plus arbitrary flags forwarded to underlying entrypoint.
	Applies hygiene defaults if present in registry.
	"""
	from pathlib import Path as _Path
	from agi_dw.bench.common.registry import get_suite_config, resolve_entrypoint  # type: ignore
	root = _Path(__file__).resolve().parents[2]
	# Ensure a default model for code-body suites if caller did not provide one
	try:
		if not hasattr(args, "model") or getattr(args, "model", None) is None:
			import os as _os  # type: ignore
			setattr(args, "model", _os.environ.get("AGI_DEFAULT_MODEL", "meta-llama/Llama-3.1-8B-Instruct"))
	except Exception:
		pass
	mode, cfg = get_suite_config(str(getattr(args, "suite", "")), root)
	with trace_span("registry_benchmark", {"suite": str(getattr(args, "suite", "")), "mode": mode}):
		pass
	# Apply suite defaults if provided (do not override user-specified values)
	try:
		_defaults = cfg.get("defaults", {}) or {}
		if isinstance(_defaults, dict):
			for _k, _v in _defaults.items():
				if not hasattr(args, _k) or getattr(args, _k, None) is None:
					setattr(args, _k, _v)
	except Exception:
		pass
	# Apply hygiene defaults unless user overrides
	try:
		if getattr(args, "benchmark_mode", None) is None:
			setattr(args, "benchmark_mode", bool(cfg.get("hygiene", {}).get("benchmark_mode_default", True)))
		if getattr(args, "emit_indirect", None) is None:
			setattr(args, "emit_indirect", bool(cfg.get("hygiene", {}).get("emit_indirect_default", True)))
	except Exception:
		pass
	# Maximize AGI capabilities by enabling Plan–Act–Verify and learning defaults if not set
	try:
		if getattr(args, "precheck", None) is None:
			setattr(args, "precheck", True)
		if getattr(args, "pav_enable", None) is None:
			setattr(args, "pav_enable", True)
		if getattr(args, "self_eval", None) is None:
			setattr(args, "self_eval", True)
		if getattr(args, "online_update", None) is None:
			setattr(args, "online_update", True)
		# Provide a default WM model path if present on disk
		if getattr(args, "wm_model", None) is None:
			_wm_path = root / "models" / "wm_mlp" / "wm_mlp.joblib"
			if _wm_path.exists():
				setattr(args, "wm_model", str(_wm_path))
		if getattr(args, "wm_threshold", None) is None:
			setattr(args, "wm_threshold", 0.6)
	except Exception:
		pass
	if mode == "runner":
		ent = str(cfg.get("entrypoint", ""))
		if not ent:
			print(__import__("json").dumps({"ok": False, "error": "suite_not_in_registry", "suite": str(getattr(args, "suite", ""))}))
			return 2
		fn = resolve_entrypoint(ent)
		return int(fn(args))
	if mode == "code_body":
		with trace_span("code_body_setup", {"suite": str(getattr(args, "suite"))}):
			pass
		# Resolve components
		from agi_dw.bench.common.harness import run_codebody_suite as _run_codebody_suite  # type: ignore
		from agi_dw.core.prompts.bench import build_prompt as _build_prompt  # type: ignore
		loader = cfg.get("loader")
		evaluator = cfg.get("evaluator")
		evalplus = cfg.get("evalplus")
		prompt_key = str(cfg.get("prompt_builder_key", ""))
		problems: Dict[str, Dict[str, Any]] = {}
		_rows: List[Dict[str, Any]] = []
		try:
			from agi_dw.bench.common.registry import resolve_object as _res, load_hf_dataset, make_problems_from_hf  # type: ignore
			ld = _res(loader)
			if ld is not None:
				obj = ld()
				if isinstance(obj, dict):
					# human-eval style
					problems = {str(k): {"prompt": str(v.get("prompt", "")) if isinstance(v, dict) else str(v)} for k, v in obj.items()}
				elif isinstance(obj, list):
					# list of dicts; defer problem construction to optional transform or HF helper
					_rows = [dict(it) for it in obj if isinstance(it, dict)]
		except Exception:
			problems = {}
		# Fallback: try HF dataset loading if loader spec absent but hf config present
		try:
			_loader_args = cfg.get("loader_args") or {}
			if (not problems) and (not _rows) and _loader_args.get("dataset_name"):
				_rows = load_hf_dataset(_loader_args)
		except Exception:
			pass
		# Optional transform hook: rows -> problems
		try:
			_transform_path = cfg.get("transform")
			if _transform_path and _rows:
				from agi_dw.bench.common.registry import resolve_object as _res2  # type: ignore
				_tf = _res2(_transform_path)
				if callable(_tf):
					_tp = _tf(_rows)
					if isinstance(_tp, dict):
						problems = {str(k): {"prompt": str(v.get("prompt", "")) if isinstance(v, dict) else str(v)} for k, v in _tp.items()}
		except Exception:
			pass
		# If still no problems but have rows, use generic HF mapping
		if (not problems) and _rows:
			try:
				_loader_args = cfg.get("loader_args") or {}
				problems = make_problems_from_hf(_rows, id_field=str(_loader_args.get("id_field", "task_id")), prompt_field=str(_loader_args.get("prompt_field", "prompt")), prompt_template=cfg.get("prompt_template"))
			except Exception:
				problems = {}
		# Sanitizer
		sanitize = None
		try:
			san_path = cfg.get("sanitizer")
			if san_path:
				from agi_dw.bench.common.registry import resolve_object as _res  # type: ignore
				san_fn = _res(san_path)
				opts = cfg.get("sanitizer_options") or {}
				if san_fn:
					def _san(text: str, prompt: str) -> str:
						try:
							return san_fn(_strip(text), prompt, opts)  # type: ignore
						except TypeError:
							return san_fn(_strip(text), prompt)  # type: ignore
					sanitize = _san
		except Exception:
			sanitize = None
		if sanitize is None:
			def _san(text: str, prompt: str) -> str:
				return _strip(text)
			sanitize = _san
		# Evaluators
		try:
			from agi_dw.bench.common.registry import resolve_object as _res  # type: ignore
			_eval = _res(evaluator)
			_evalplus = _res(evalplus)
		except Exception:
			_eval, _evalplus = None, None
		# If evaluator_args provided in registry, wrap evaluator with those kwargs
		try:
			eval_args = cfg.get("evaluator_args") or None
			if _eval is not None and isinstance(eval_args, dict):
				_base_eval = _eval
				def _wrapped_eval(sample_file: str, k: List[int], n_workers: int, timeout: float, ignore_incomplete: bool):  # type: ignore
					return _base_eval(sample_file, k=k, n_workers=n_workers, timeout=timeout, ignore_incomplete=ignore_incomplete, **eval_args)
				_eval = _wrapped_eval  # type: ignore
			# Fallback: if no evaluator resolved but we have args for generic solution-eval, bind it
			if _eval is None and isinstance(eval_args, dict) and mode == "code_body":
				try:
					from agi_dw.bench.common.evaluators import evaluate_solution as _generic_eval  # type: ignore
					def _wrapped_generic(sample_file: str, k: List[int], n_workers: int, timeout: float, ignore_incomplete: bool):  # type: ignore
						return _generic_eval(sample_file, k=k, n_workers=n_workers, timeout=timeout, ignore_incomplete=ignore_incomplete, **eval_args)
					_eval = _wrapped_generic  # type: ignore
				except Exception:
					pass
		except Exception:
			pass
		with trace_span("code_body_run", {"suite": str(getattr(args, "suite"))}):
			return int(_run_codebody_suite(args=args, suite=str(getattr(args, "suite")), problems=problems, prompt_builder_key=prompt_key or str(getattr(args, "suite")), sanitize_fn=sanitize, evaluate_functional_correctness=_eval, evalplus_evaluate=_evalplus))
	if mode == "patch":
		with trace_span("patch_setup", {"suite": str(getattr(args, "suite"))}):
			pass
		# Build an adapter on the fly using registry fields
		from agi_dw.bench.common.universal_harness import run_with_adapter  # type: ignore
		from agi_dw.bench.common.adapters import PatchAdapterBase  # type: ignore
		from agi_dw.bench.common.registry import resolve_object as _res, load_hf_dataset  # type: ignore
		loader = cfg.get("loader")
		loader_args = cfg.get("loader_args") or {}
		id_field = str(cfg.get("id_field", "instance_id"))
		input_tmpl = str(cfg.get("input_template", ""))
		eval_entry = cfg.get("evaluator_entrypoint")
		class _DynPatchAdapter(PatchAdapterBase):
			def suite(self) -> str:
				return str(getattr(args, "suite"))
			def env_modules(self) -> List[str]:
				return list(cfg.get("env_modules", []))
			def load_tasks(self, _args: Any) -> Dict[str, Dict[str, Any]]:
				rows: List[Dict[str, Any]] = []
				ld = _res(loader)
				if ld is not None:
					try:
						obj = ld(**loader_args) if loader_args else ld()
						# swebench returns a list
						if isinstance(obj, list):
							rows = [dict(it) for it in obj]
						else:
							rows = []
					except Exception:
						rows = []
				if (not rows) and loader_args.get("dataset_name"):
					rows = load_hf_dataset(loader_args)
				return {str(it.get(id_field, "")): dict(it) for it in rows if str(it.get(id_field, "")).strip()}
			def build_input(self, task_id: str, task: Dict[str, Any]) -> str:
				try:
					return input_tmpl.format(**task)
				except Exception:
					return input_tmpl
			def build_predictions(self, samples_path: Path, _args: Any, root: Path, task_ids: List[str], tasks: Dict[str, Dict[str, Any]]) -> Optional[Path]:
				from agi_dw.bench.common.pipeline import read_jsonl  # type: ignore
				import json as _json
				preds_file = Path(root / "data" / "bench" / "tmp" / (str(getattr(args, "suite")) + "_preds.jsonl"))
				preds_file.parent.mkdir(parents=True, exist_ok=True)
				with preds_file.open("w", encoding="utf-8") as pf:
					for obj in read_jsonl(samples_path):
						tid = str(obj.get("task_id", ""))
						comp = str(obj.get("completion", ""))
						if not tid:
							continue
						pf.write(_json.dumps({"instance_id": tid, "model_patch": comp, "patch": comp, "model_name_or_path": str(getattr(args, "model", "unknown"))}) + "\n")
				return preds_file
			def run_official_evaluator(self, predictions_path: Path, _args: Any, root: Path, task_ids: List[str]) -> Optional[Path]:
				try:
					entry = _res(eval_entry)
					if entry is None:
						return None
					# Call entrypoint akin to swebench main
					entry(
						dataset_name=str(loader_args.get("dataset_name")),
						split=str(loader_args.get("split", "test")),
						instance_ids=task_ids,
						predictions_path=str(predictions_path),
						max_workers=max(1, int(getattr(args, "max_workers", 1) or 1)),
						force_rebuild=False,
						cache_level="env",
						clean=False,
						open_file_limit=2048,
						run_id="agi_dw",
						timeout=int(getattr(args, "timeout", 1800) or 1800),
						namespace=str(getattr(args, "suite")),
						rewrite_reports=False,
						modal=False,
						instance_image_tag="latest",
						env_image_tag="latest",
						report_dir=str((root / "data" / "bench" / "tmp").resolve()),
					)
					return None
				except Exception:
					return None
		adapter = _DynPatchAdapter()
		with trace_span("patch_run", {"suite": str(getattr(args, "suite"))}):
			return int(run_with_adapter(args, adapter))
	print(__import__("json").dumps({"ok": False, "error": "unsupported_registry_mode", "mode": mode}))
	return 2


						return _generic_eval(sample_file, k=k, n_workers=n_workers, timeout=timeout, ignore_incomplete=ignore_incomplete, **eval_args)
					_eval = _wrapped_generic  # type: ignore
				except Exception:
					pass
		except Exception:
			pass
		with trace_span("code_body_run", {"suite": str(getattr(args, "suite"))}):
			return int(_run_codebody_suite(args=args, suite=str(getattr(args, "suite")), problems=problems, prompt_builder_key=prompt_key or str(getattr(args, "suite")), sanitize_fn=sanitize, evaluate_functional_correctness=_eval, evalplus_evaluate=_evalplus))
	if mode == "patch":
		with trace_span("patch_setup", {"suite": str(getattr(args, "suite"))}):
			pass
		# Build an adapter on the fly using registry fields
		from agi_dw.bench.common.universal_harness import run_with_adapter  # type: ignore
		from agi_dw.bench.common.adapters import PatchAdapterBase  # type: ignore
		from agi_dw.bench.common.registry import resolve_object as _res, load_hf_dataset  # type: ignore
		loader = cfg.get("loader")
		loader_args = cfg.get("loader_args") or {}
		id_field = str(cfg.get("id_field", "instance_id"))
		input_tmpl = str(cfg.get("input_template", ""))
		eval_entry = cfg.get("evaluator_entrypoint")
		class _DynPatchAdapter(PatchAdapterBase):
			def suite(self) -> str:
				return str(getattr(args, "suite"))
			def env_modules(self) -> List[str]:
				return list(cfg.get("env_modules", []))
			def load_tasks(self, _args: Any) -> Dict[str, Dict[str, Any]]:
				rows: List[Dict[str, Any]] = []
				ld = _res(loader)
				if ld is not None:
					try:
						obj = ld(**loader_args) if loader_args else ld()
						# swebench returns a list
						if isinstance(obj, list):
							rows = [dict(it) for it in obj]
						else:
							rows = []
					except Exception:
						rows = []
				if (not rows) and loader_args.get("dataset_name"):
					rows = load_hf_dataset(loader_args)
				return {str(it.get(id_field, "")): dict(it) for it in rows if str(it.get(id_field, "")).strip()}
			def build_input(self, task_id: str, task: Dict[str, Any]) -> str:
				try:
					return input_tmpl.format(**task)
				except Exception:
					return input_tmpl
			def build_predictions(self, samples_path: Path, _args: Any, root: Path, task_ids: List[str], tasks: Dict[str, Dict[str, Any]]) -> Optional[Path]:
				from agi_dw.bench.common.pipeline import read_jsonl  # type: ignore
				import json as _json
				preds_file = Path(root / "data" / "bench" / "tmp" / (str(getattr(args, "suite")) + "_preds.jsonl"))
				preds_file.parent.mkdir(parents=True, exist_ok=True)
				with preds_file.open("w", encoding="utf-8") as pf:
					for obj in read_jsonl(samples_path):
						tid = str(obj.get("task_id", ""))
						comp = str(obj.get("completion", ""))
						if not tid:
							continue
						pf.write(_json.dumps({"instance_id": tid, "model_patch": comp, "patch": comp, "model_name_or_path": str(getattr(args, "model", "unknown"))}) + "\n")
				return preds_file
			def run_official_evaluator(self, predictions_path: Path, _args: Any, root: Path, task_ids: List[str]) -> Optional[Path]:
				try:
					entry = _res(eval_entry)
					if entry is None:
						return None
					# Call entrypoint akin to swebench main
					entry(
						dataset_name=str(loader_args.get("dataset_name")),
						split=str(loader_args.get("split", "test")),
						instance_ids=task_ids,
						predictions_path=str(predictions_path),
						max_workers=max(1, int(getattr(args, "max_workers", 1) or 1)),
						force_rebuild=False,
						cache_level="env",
						clean=False,
						open_file_limit=2048,
						run_id="agi_dw",
						timeout=int(getattr(args, "timeout", 1800) or 1800),
						namespace=str(getattr(args, "suite")),
						rewrite_reports=False,
						modal=False,
						instance_image_tag="latest",
						env_image_tag="latest",
						report_dir=str((root / "data" / "bench" / "tmp").resolve()),
					)
					return None
				except Exception:
					return None
		adapter = _DynPatchAdapter()
		with trace_span("patch_run", {"suite": str(getattr(args, "suite"))}):
			return int(run_with_adapter(args, adapter))
	print(__import__("json").dumps({"ok": False, "error": "unsupported_registry_mode", "mode": mode}))
	return 2


					if isinstance(_tp, dict):
						problems = {str(k): {"prompt": str(v.get("prompt", "")) if isinstance(v, dict) else str(v)} for k, v in _tp.items()}
		except Exception:
			pass
		# If still no problems but have rows, use generic HF mapping
		if (not problems) and _rows:
			try:
				_loader_args = cfg.get("loader_args") or {}
				problems = make_problems_from_hf(_rows, id_field=str(_loader_args.get("id_field", "task_id")), prompt_field=str(_loader_args.get("prompt_field", "prompt")), prompt_template=cfg.get("prompt_template"))
			except Exception:
				problems = {}
		# Sanitizer
		sanitize = None
		try:
			san_path = cfg.get("sanitizer")
			if san_path:
				from agi_dw.bench.common.registry import resolve_object as _res  # type: ignore
				san_fn = _res(san_path)
				opts = cfg.get("sanitizer_options") or {}
				if san_fn:
					def _san(text: str, prompt: str) -> str:
						try:
							return san_fn(_strip(text), prompt, opts)  # type: ignore
						except TypeError:
							return san_fn(_strip(text), prompt)  # type: ignore
					sanitize = _san
		except Exception:
			sanitize = None
		if sanitize is None:
			def _san(text: str, prompt: str) -> str:
				return _strip(text)
			sanitize = _san
		# Evaluators
		try:
			from agi_dw.bench.common.registry import resolve_object as _res  # type: ignore
			_eval = _res(evaluator)
			_evalplus = _res(evalplus)
		except Exception:
			_eval, _evalplus = None, None
		# If evaluator_args provided in registry, wrap evaluator with those kwargs
		try:
			eval_args = cfg.get("evaluator_args") or None
			if _eval is not None and isinstance(eval_args, dict):
				_base_eval = _eval
				def _wrapped_eval(sample_file: str, k: List[int], n_workers: int, timeout: float, ignore_incomplete: bool):  # type: ignore


suite

					_eval = _wrapped_generic  # type: ignore
				except Exception:
					pass
		except Exception:
			pass
		with trace_span("code_body_run", {"suite": str(getattr(args, "suite"))}):
			return int(_run_codebody_suite(args=args, suite=str(getattr(args, "suite")), problems=problems, prompt_builder_key=prompt_key or str(getattr(args, "suite")), sanitize_fn=sanitize, evaluate_functional_correctness=_eval, evalplus_evaluate=_evalplus))
	if mode == "patch":
		with trace_span("patch_setup", {"suite": str(getattr(args, "suite"))}):
			pass
		# Build an adapter on the fly using registry fields
		from agi_dw.bench.common.universal_harness import run_with_adapter  # type: ignore
		from agi_dw.bench.common.adapters import PatchAdapterBase  # type: ignore
		from agi_dw.bench.common.registry import resolve_object as _res, load_hf_dataset  # type: ignore
		loader = cfg.get("loader")
		loader_args = cfg.get("loader_args") or {}
		id_field = str(cfg.get("id_field", "instance_id"))
		input_tmpl = str(cfg.get("input_template", ""))
		eval_entry = cfg.get("evaluator_entrypoint")
		class _DynPatchAdapter(PatchAdapterBase):
			def suite(self) -> str:
				return str(getattr(args, "suite"))
			def env_modules(self) -> List[str]:
				return list(cfg.get("env_modules", []))
			def load_tasks(self, _args: Any) -> Dict[str, Dict[str, Any]]:
				rows: List[Dict[str, Any]] = []
				ld = _res(loader)
				if ld is not None:
					try:
						obj = ld(**loader_args) if loader_args else ld()
						# swebench returns a list
						if isinstance(obj, list):
							rows = [dict(it) for it in obj]
						else:
							rows = []
					except Exception:
						rows = []
				if (not rows) and loader_args.get("dataset_name"):
					rows = load_hf_dataset(loader_args)
				return {str(it.get(id_field, "")): dict(it) for it in rows if str(it.get(id_field, "")).strip()}
			def build_input(self, task_id: str, task: Dict[str, Any]) -> str:
				try:


					pass
		except Exception:
			pass
		with trace_span("code_body_run", {"suite": str(getattr(args, "suite"))}):
			return int(_run_codebody_suite(args=args, suite=str(getattr(args, "suite")), problems=problems, prompt_builder_key=prompt_key or str(getattr(args, "suite")), sanitize_fn=sanitize, evaluate_functional_correctness=_eval, evalplus_evaluate=_evalplus))
	if mode == "patch":
		with trace_span("patch_setup", {"suite": str(getattr(args, "suite"))}):
			pass
		# Build an adapter on the fly using registry fields
		from agi_dw.bench.common.universal_harness import run_with_adapter  # type: ignore
		from agi_dw.bench.common.adapters import PatchAdapterBase  # type: ignore
		from agi_dw.bench.common.registry import resolve_object as _res, load_hf_dataset  # type: ignore
		loader = cfg.get("loader")
		loader_args = cfg.get("loader_args") or {}
		id_field = str(cfg.get("id_field", "instance_id"))
		input_tmpl = str(cfg.get("input_template", ""))
		eval_entry = cfg.get("evaluator_entrypoint")
		class _DynPatchAdapter(PatchAdapterBase):
			def suite(self) -> str:
				return str(getattr(args, "suite"))
			def env_modules(self) -> List[str]:
				return list(cfg.get("env_modules", []))
			def load_tasks(self, _args: Any) -> Dict[str, Dict[str, Any]]:
				rows: List[Dict[str, Any]] = []
				ld = _res(loader)
				if ld is not None:
					try:
						obj = ld(**loader_args) if loader_args else ld()
						# swebench returns a list
						if isinstance(obj, list):
							rows = [dict(it) for it in obj]
						else:
							rows = []
					except Exception:
						rows = []
				if (not rows) and loader_args.get("dataset_name"):
					rows = load_hf_dataset(loader_args)
				return {str(it.get(id_field, "")): dict(it) for it in rows if str(it.get(id_field, "")).strip()}
			def build_input(self, task_id: str, task: Dict[str, Any]) -> str:
				try:
					return input_tmpl.format(**task)
				except Exception:


			pass
		with trace_span("code_body_run", {"suite": str(getattr(args, "suite"))}):
			return int(_run_codebody_suite(args=args, suite=str(getattr(args, "suite")), problems=problems, prompt_builder_key=prompt_key or str(getattr(args, "suite")), sanitize_fn=sanitize, evaluate_functional_correctness=_eval, evalplus_evaluate=_evalplus))
	if mode == "patch":
		with trace_span("patch_setup", {"suite": str(getattr(args, "suite"))}):
			pass
		# Build an adapter on the fly using registry fields
		from agi_dw.bench.common.universal_harness import run_with_adapter  # type: ignore
		from agi_dw.bench.common.adapters import PatchAdapterBase  # type: ignore
		from agi_dw.bench.common.registry import resolve_object as _res, load_hf_dataset  # type: ignore
		loader = cfg.get("loader")
		loader_args = cfg.get("loader_args") or {}
		id_field = str(cfg.get("id_field", "instance_id"))
		input_tmpl = str(cfg.get("input_template", ""))
		eval_entry = cfg.get("evaluator_entrypoint")
		class _DynPatchAdapter(PatchAdapterBase):
			def suite(self) -> str:
				return str(getattr(args, "suite"))
			def env_modules(self) -> List[str]:
				return list(cfg.get("env_modules", []))
			def load_tasks(self, _args: Any) -> Dict[str, Dict[str, Any]]:
				rows: List[Dict[str, Any]] = []
				ld = _res(loader)
				if ld is not None:
					try:
						obj = ld(**loader_args) if loader_args else ld()
						# swebench returns a list
						if isinstance(obj, list):
							rows = [dict(it) for it in obj]
						else:
							rows = []
					except Exception:
						rows = []
				if (not rows) and loader_args.get("dataset_name"):
					rows = load_hf_dataset(loader_args)
				return {str(it.get(id_field, "")): dict(it) for it in rows if str(it.get(id_field, "")).strip()}
			def build_input(self, task_id: str, task: Dict[str, Any]) -> str:
				try:
					return input_tmpl.format(**task)
				except Exception:
					return input_tmpl
			def build_predictions(self, samples_path: Path, _args: Any, root: Path, task_ids: List[str], tasks: Dict[str, Dict[str, Any]]) -> Optional[Path]:
				from agi_dw.bench.common.pipeline import read_jsonl  # type: ignore
				import json as _json
				preds_file = Path(root / "data" / "bench" / "tmp" / (str(getattr(args, "suite")) + "_preds.jsonl"))
				preds_file.parent.mkdir(parents=True, exist_ok=True)
				with preds_file.open("w", encoding="utf-8") as pf:
					for obj in read_jsonl(samples_path):
						tid = str(obj.get("task_id", ""))
						comp = str(obj.get("completion", ""))
						if not tid:
							continue
						pf.write(_json.dumps({"instance_id": tid, "model_patch": comp, "patch": comp, "model_name_or_path": str(getattr(args, "model", "unknown"))}) + "\n")
				return preds_file
			def run_official_evaluator(self, predictions_path: Path, _args: Any, root: Path, task_ids: List[str]) -> Optional[Path]:
				try:


			def suite(self) -> str:
				return str(getattr(args, "suite"))
			def env_modules(self) -> List[str]:
				return list(cfg.get("env_modules", []))
			def load_tasks(self, _args: Any) -> Dict[str, Dict[str, Any]]:
				rows: List[Dict[str, Any]] = []
				ld = _res(loader)
				if ld is not None:
					try:
						obj = ld(**loader_args) if loader_args else ld()
						# swebench returns a list
						if isinstance(obj, list):
							rows = [dict(it) for it in obj]
						else:
							rows = []
					except Exception:
						rows = []
				if (not rows) and loader_args.get("dataset_name"):
					rows = load_hf_dataset(loader_args)
				return {str(it.get(id_field, "")): dict(it) for it in rows if str(it.get(id_field, "")).strip()}
			def build_input(self, task_id: str, task: Dict[str, Any]) -> str:
				try:
					return input_tmpl.format(**task)
				except Exception:
					return input_tmpl
			def build_predictions(self, samples_path: Path, _args: Any, root: Path, task_ids: List[str], tasks: Dict[str, Dict[str, Any]]) -> Optional[Path]:
				from agi_dw.bench.common.pipeline import read_jsonl  # type: ignore
				import json as _json
				preds_file = Path(root / "data" / "bench" / "tmp" / (str(getattr(args, "suite")) + "_preds.jsonl"))
				preds_file.parent.mkdir(parents=True, exist_ok=True)
				with preds_file.open("w", encoding="utf-8") as pf:
					for obj in read_jsonl(samples_path):
						tid = str(obj.get("task_id", ""))
						comp = str(obj.get("completion", ""))
						if not tid:
							continue
						pf.write(_json.dumps({"instance_id": tid, "model_patch": comp, "patch": comp, "model_name_or_path": str(getattr(args, "model", "unknown"))}) + "\n")
				return preds_file
			def run_official_evaluator(self, predictions_path: Path, _args: Any, root: Path, task_ids: List[str]) -> Optional[Path]:
				try:
					entry = _res(eval_entry)
					if entry is None:
						return None
					# Call entrypoint akin to swebench main
					entry(


				rows: List[Dict[str, Any]] = []
				ld = _res(loader)
				if ld is not None:
					try:
						obj = ld(**loader_args) if loader_args else ld()
						# swebench returns a list
						if isinstance(obj, list):
							rows = [dict(it) for it in obj]
						else:
							rows = []
					except Exception:
						rows = []
				if (not rows) and loader_args.get("dataset_name"):
					rows = load_hf_dataset(loader_args)
				return {str(it.get(id_field, "")): dict(it) for it in rows if str(it.get(id_field, "")).strip()}
			def build_input(self, task_id: str, task: Dict[str, Any]) -> str:
				try:
					return input_tmpl.format(**task)
				except Exception:
					return input_tmpl
			def build_predictions(self, samples_path: Path, _args: Any, root: Path, task_ids: List[str], tasks: Dict[str, Dict[str, Any]]) -> Optional[Path]:
				from agi_dw.bench.common.pipeline import read_jsonl  # type: ignore
				import json as _json
				preds_file = Path(root / "data" / "bench" / "tmp" / (str(getattr(args, "suite")) + "_preds.jsonl"))
				preds_file.parent.mkdir(parents=True, exist_ok=True)
				with preds_file.open("w", encoding="utf-8") as pf:
					for obj in read_jsonl(samples_path):
						tid = str(obj.get("task_id", ""))
						comp = str(obj.get("completion", ""))
						if not tid:
							continue
						pf.write(_json.dumps({"instance_id": tid, "model_patch": comp, "patch": comp, "model_name_or_path": str(getattr(args, "model", "unknown"))}) + "\n")
				return preds_file
			def run_official_evaluator(self, predictions_path: Path, _args: Any, root: Path, task_ids: List[str]) -> Optional[Path]:
				try:
					entry = _res(eval_entry)
					if entry is None:
						return None
					# Call entrypoint akin to swebench main
					entry(
						dataset_name=str(loader_args.get("dataset_name")),
						split=str(loader_args.get("split", "test")),
						instance_ids=task_ids,
						predictions_path=str(predictions_path),
						max_workers=max(1, int(getattr(args, "max_workers", 1) or 1)),
						force_rebuild=False,
						cache_level="env",
						clean=False,
						open_file_limit=2048,
						run_id="agi_dw",
						timeout=int(getattr(args, "timeout", 1800) or 1800),
						namespace=str(getattr(args, "suite")),
						rewrite_reports=False,


					rows = load_hf_dataset(loader_args)
				return {str(it.get(id_field, "")): dict(it) for it in rows if str(it.get(id_field, "")).strip()}
			def build_input(self, task_id: str, task: Dict[str, Any]) -> str:
				try:
					return input_tmpl.format(**task)
				except Exception:
					return input_tmpl
			def build_predictions(self, samples_path: Path, _args: Any, root: Path, task_ids: List[str], tasks: Dict[str, Dict[str, Any]]) -> Optional[Path]:
				from agi_dw.bench.common.pipeline import read_jsonl  # type: ignore
				import json as _json
				preds_file = Path(root / "data" / "bench" / "tmp" / (str(getattr(args, "suite")) + "_preds.jsonl"))
				preds_file.parent.mkdir(parents=True, exist_ok=True)
				with preds_file.open("w", encoding="utf-8") as pf:
					for obj in read_jsonl(samples_path):
						tid = str(obj.get("task_id", ""))
						comp = str(obj.get("completion", ""))
						if not tid:
							continue
						pf.write(_json.dumps({"instance_id": tid, "model_patch": comp, "patch": comp, "model_name_or_path": str(getattr(args, "model", "unknown"))}) + "\n")
				return preds_file
			def run_official_evaluator(self, predictions_path: Path, _args: Any, root: Path, task_ids: List[str]) -> Optional[Path]:
				try:
					entry = _res(eval_entry)
					if entry is None:
						return None
					# Call entrypoint akin to swebench main
					entry(
						dataset_name=str(loader_args.get("dataset_name")),
						split=str(loader_args.get("split", "test")),
						instance_ids=task_ids,
						predictions_path=str(predictions_path),
						max_workers=max(1, int(getattr(args, "max_workers", 1) or 1)),
						force_rebuild=False,
						cache_level="env",
						clean=False,
						open_file_limit=2048,
						run_id="agi_dw",
						timeout=int(getattr(args, "timeout", 1800) or 1800),
						namespace=str(getattr(args, "suite")),
						rewrite_reports=False,
						modal=False,
						instance_image_tag="latest",
						env_image_tag="latest",
						report_dir=str((root / "data" / "bench" / "tmp").resolve()),
					)
					return None
				except Exception:
					return None
		adapter = _DynPatchAdapter()
		with trace_span("patch_run", {"suite": str(getattr(args, "suite"))}):
			return int(run_with_adapter(args, adapter))
	print(__import__("json").dumps({"ok": False, "error": "unsupported_registry_mode", "mode": mode}))
	return 2