							return san_fn(_strip(text), prompt)  # type: ignore
					sanitize = _san
		except Exception:
			sanitize = None
		if sanitize is None:
			def _san(text: str, prompt: str) -> str:
				return _strip(text)
			sanitize = _san
		# Evaluators
		try:
			from agi_dw.bench.common.registry import resolve_object as _res  # type: ignore
			_eval = _res(evaluator)
			_evalplus = _res(evalplus)
		except Exception:
			_eval, _evalplus = None, None
		# If evaluator_args provided in registry, wrap evaluator with those kwargs
		try:
			eval_args = cfg.get("evaluator_args") or None
			if _eval is not None and isinstance(eval_args, dict):
				_base_eval = _eval
				def _wrapped_eval(sample_file: str, k: List[int], n_workers: int, timeout: float, ignore_incomplete: bool):  # type: ignore
					return _base_eval(sample_file, k=k, n_workers=n_workers, timeout=timeout, ignore_incomplete=ignore_incomplete, **eval_args)
				_eval = _wrapped_eval  # type: ignore
			# Fallback: if no evaluator resolved but we have args for generic solution-eval, bind it
			if _eval is None and isinstance(eval_args, dict) and mode == "code_body":
				try:
					from agi_dw.bench.common.evaluators import evaluate_solution as _generic_eval  # type: ignore
					def _wrapped_generic(sample_file: str, k: List[int], n_workers: int, timeout: float, ignore_incomplete: bool):  # type: ignore
						return _generic_eval(sample_file, k=k, n_workers=n_workers, timeout=timeout, ignore_incomplete=ignore_incomplete, **eval_args)
					_eval = _wrapped_generic  # type: ignore
				except Exception:
					pass
		except Exception:
			pass
		with trace_span("code_body_run", {"suite": str(getattr(args, "suite"))}):
			return int(_run_codebody_suite(args=args, suite=str(getattr(args, "suite")), problems=problems, prompt_builder_key=prompt_key or str(getattr(args, "suite")), sanitize_fn=sanitize, evaluate_functional_correctness=_eval, evalplus_evaluate=_evalplus))
	if mode == "patch":
		with trace_span("patch_setup", {"suite": str(getattr(args, "suite"))}):
			pass
		# Build an adapter on the fly using registry fields
		from agi_dw.bench.common.universal_harness import run_with_adapter  # type: ignore
		from agi_dw.bench.common.adapters import PatchAdapterBase  # type: ignore


			sanitize = _san
		# Evaluators
		try:
			from agi_dw.bench.common.registry import resolve_object as _res  # type: ignore
			_eval = _res(evaluator)
			_evalplus = _res(evalplus)
		except Exception:
			_eval, _evalplus = None, None
		# If evaluator_args provided in registry, wrap evaluator with those kwargs
		try:
			eval_args = cfg.get("evaluator_args") or None
			if _eval is not None and isinstance(eval_args, dict):
				_base_eval = _eval
				def _wrapped_eval(sample_file: str, k: List[int], n_workers: int, timeout: float, ignore_incomplete: bool):  # type: ignore
					return _base_eval(sample_file, k=k, n_workers=n_workers, timeout=timeout, ignore_incomplete=ignore_incomplete, **eval_args)
				_eval = _wrapped_eval  # type: ignore
			# Fallback: if no evaluator resolved but we have args for generic solution-eval, bind it
			if _eval is None and isinstance(eval_args, dict) and mode == "code_body":
				try:
					from agi_dw.bench.common.evaluators import evaluate_solution as _generic_eval  # type: ignore
					def _wrapped_generic(sample_file: str, k: List[int], n_workers: int, timeout: float, ignore_incomplete: bool):  # type: ignore
						return _generic_eval(sample_file, k=k, n_workers=n_workers, timeout=timeout, ignore_incomplete=ignore_incomplete, **eval_args)
					_eval = _wrapped_generic  # type: ignore
				except Exception:
					pass
		except Exception:
			pass
		with trace_span("code_body_run", {"suite": str(getattr(args, "suite"))}):
			return int(_run_codebody_suite(args=args, suite=str(getattr(args, "suite")), problems=problems, prompt_builder_key=prompt_key or str(getattr(args, "suite")), sanitize_fn=sanitize, evaluate_functional_correctness=_eval, evalplus_evaluate=_evalplus))
	if mode == "patch":
		with trace_span("patch_setup", {"suite": str(getattr(args, "suite"))}):
			pass
		# Build an adapter on the fly using registry fields
		from agi_dw.bench.common.universal_harness import run_with_adapter  # type: ignore
		from agi_dw.bench.common.adapters import PatchAdapterBase  # type: ignore
		from agi_dw.bench.common.registry import resolve_object as _res, load_hf_dataset  # type: ignore
		loader = cfg.get("loader")
		loader_args = cfg.get("loader_args") or {}
		id_field = str(cfg.get("id_field", "instance_id"))
		input_tmpl = str(cfg.get("input_template", ""))
		eval_entry = cfg.get("evaluator_entrypoint")
		class _DynPatchAdapter(PatchAdapterBase):


from __future__ import annotations

from pathlib import Path
from typing import Any, Dict, List, Tuple, Optional
from agi_dw.core.ops.tracing import trace_span, meter_cost  # type: ignore

def looks_like_python_code(body: str) -> bool:
	"""Heuristic filter to check that a generated body resembles Python code.

	Shared across benchmarks to keep generation sanity checks consistent.
	"""
	b = (body or "").strip()
	if not b:
		return False
	if (b.startswith('"""') and b.endswith('"""')) or (b.startswith("'''") and b.endswith("'''")):
		return False
	# avoid degenerate bodies
	if b == "pass":
		return False
	for tok in ("return ", "= ", "=\n", "for ", "while ", "if ", "elif ", "yield ", "try:", "except "):
		if tok in b:
			return True
	# tolerate comprehensions with ':' and brackets
	if ":" in b and any(sym in b for sym in ("[", "{", "(")):
		return True
	return False

def build_python_body_chat(inp: str) -> Tuple[List[Dict[str, str]], List[str]]:
	"""Standard system+user chat and stop tokens for function-body generation."""
	messages: List[Dict[str, str]] = [
		{"role": "system", "content": "You are a Python coding assistant. Output ONLY the function body, properly indented under the provided def. No fences, no tests, no prints, no extra text. Do not include docstrings or comments."},
		{"role": "user", "content": inp},
	]
	stops = ["\n\n", "\nclass ", "\ndef ", "\nif __name__ == "]
	return messages, stops

def try_generate_python_body(llm: Any, inp: str, params: Dict[str, Any], grammar_constrained: bool) -> str:
	"""Generate a Python function body with optional grammar guidance; fall back robustly."""
	try:
		if grammar_constrained:
			try:
				import outlines  # type: ignore  # noqa: F401
				return str(llm.generate(inp, **{**params, "grammar": "python_function_body_minimal"}))
			except Exception:
				pass
		messages, stops = build_python_body_chat(inp)
		return str(llm.chat(messages, max_new_tokens=params.get("max_new_tokens", 256), temperature=params.get("temperature", 0.0), top_p=params.get("top_p"), top_k=params.get("top_k"), stop=stops))
	except Exception:
		return str(llm.generate(inp, **params))

def attach_adapter_from_args(llm: Any, args: Any, root: Path, default_adapter: Optional[Path] = None) -> None:
	"""Attach an adapter to the LLM based on args, adapter bank, and defaults.

	- Respects --adapter_dir
	- If --adapter_bank is provided, chooses a planner/verifier mapping from bank
	- If adapter not specified, uses default_adapter if provided and exists
	"""
	try:
		adapter_dir = getattr(args, "adapter_dir", None)
		bank = getattr(args, "adapter_bank", None)
		if bank:
			try:
				from agi_dw.core.llm.adapter_router import pick_from_bank  # type: ignore
				mapping = pick_from_bank(root, str(bank))
				cand = mapping.get("planner") or mapping.get("verifier")
				if cand:
					adapter_dir = str(cand)
			except Exception:
				pass
		if (not adapter_dir) or (isinstance(adapter_dir, str) and not str(adapter_dir).strip()):
			if default_adapter is not None and Path(default_adapter).exists():
				adapter_dir = str(default_adapter)
		llm.attach_adapter(str(adapter_dir) or None)
	except Exception:
		pass

# ---- Evaluation and results helpers ----
def _safe_int(val: Any, default: int) -> int:
	try:
		return int(val)
	except Exception:
		return int(default)

def _safe_float(val: Any, default: float) -> float:
	try:
		return float(val)
	except Exception:
		return float(default)

def parse_k_list(args: Any) -> List[int]:
	try:
		return [int(x) for x in str(getattr(args, "k", "1") or "1").split(",") if str(x).strip()]
	except Exception:
		return [1]

def evaluate_samples(samples_path: Path, k_list: List[int], default_workers: int, args: Any, evaluate_fn: Any) -> Tuple[Any, Path]:
	"""Run evaluation using provided evaluate_fn and return (aggregate_result, results_path)."""
	with trace_span("evaluate_samples", {"k": k_list, "timeout": _safe_float(getattr(args, "timeout", 15) or 15, 15.0)}):
		res = evaluate_fn(
			sample_file=str(samples_path),
			k=k_list,
			n_workers=max(1, _safe_int(getattr(args, "max_workers", default_workers) or default_workers, default_workers)),
			timeout=_safe_float(getattr(args, "timeout", 15) or 15, 15.0),
			ignore_incomplete=True,
		)
	results_path = Path(str(samples_path) + "_results.jsonl")
	return res, results_path

def read_jsonl(path: Path) -> List[Dict[str, Any]]:
	rows: List[Dict[str, Any]] = []
	if not path.exists():
		return rows
	with path.open("r", encoding="utf-8") as f:
		for line in f:
			li = line.strip()
			if not li:
				continue
			try:
				rows.append(__import__("json").loads(li))
			except Exception:
				continue
	return rows

def build_results_by_task(results_path: Path) -> Dict[str, bool]:
	by_task: Dict[str, bool] = {}
	for row in read_jsonl(results_path):
		tid = str(row.get("task_id", ""))
		passed = bool(row.get("passed", False))
		if tid:
			by_task[tid] = bool(by_task.get(tid, False) or passed)
	return by_task

def dedupe_by_passed(samples_path: Path, verbose_path: Path, results_path: Path) -> None:
	"""Keep only passing completions per task in samples and verbose sidecar."""
	passed_choice: Dict[str, str] = {}
	for row in read_jsonl(results_path):
		if bool(row.get("passed", False)):
			_tid = str(row.get("task_id", ""))
			_comp = str(row.get("completion", ""))
			if _tid and _comp and _tid not in passed_choice:
				passed_choice[_tid] = _comp
	if not passed_choice:
		return
	keep: List[Dict[str, str]] = []
	for obj in read_jsonl(samples_path):
		_tid = str(obj.get("task_id", ""))
		_comp = str(obj.get("completion", ""))
		if _tid in passed_choice and _comp == passed_choice[_tid]:
			keep.append({"task_id": _tid, "completion": _comp})
	with samples_path.open("w", encoding="utf-8") as sfw:
		for obj in keep:
			sfw.write(__import__("json").dumps(obj) + "\n")
	keep_verbose: List[Dict[str, Any]] = []
	for obj in read_jsonl(verbose_path):
		_tid = str(obj.get("task_id", ""))
		_comp = str(obj.get("completion", ""))
		if _tid in passed_choice and _comp == passed_choice[_tid]:
			keep_verbose.append(obj)
	with verbose_path.open("w", encoding="utf-8") as vfw:
		for obj in keep_verbose:
			vfw.write(__import__("json").dumps(obj) + "\n")

# ---- Sharding helpers ----
def shard_task_ids(task_ids: List[str], args: Any) -> List[str]:
	"""Shard a list of task_ids across processes using num_shards/shard_id or torchrun envs."""
	try:
		import os as _os
		num_shards = max(1, _safe_int(getattr(args, "num_shards", _os.environ.get("WORLD_SIZE", 1) or 1), 1))
		shard_id = max(0, _safe_int(getattr(args, "shard_id", _os.environ.get("RANK", 0) or 0), 0))
		if num_shards > 1:
			return [tid for i, tid in enumerate(task_ids) if (i % num_shards) == shard_id]
	except Exception:
		pass
	return task_ids

def compute_sharded_outpath(raw_out: str, args: Any) -> str:
	"""Append shard suffix to output path when sharded."""
	try:
		import os as _os
		num_shards = max(1, _safe_int(getattr(args, "num_shards", _os.environ.get("WORLD_SIZE", 1) or 1), 1))
		shard_id = max(0, _safe_int(getattr(args, "shard_id", _os.environ.get("RANK", 0) or 0), 0))
		if num_shards > 1:
			p = Path(raw_out)
			return str(p.with_name(p.stem + f".shard{shard_id}-of-{num_shards}" + p.suffix))
	except Exception:
		pass
	return str(raw_out)

# ---- Trace helpers ----
def write_trace(
	trace_path: Path,
	suite: str,
	task_id: str,
	attempt_idx: Any,
	latency_ms: Optional[int],
	seed: Optional[int],
	env_fingerprint: Dict[str, Any],
	prompt: str,
	completion: str,
	meta: Optional[Dict[str, Any]] = None,
) -> None:
	"""Append a single trace record line (JSON) to the given path."""
	try:
		rec: Dict[str, Any] = {
			"suite": suite,
			"task_id": task_id,
			"attempt_idx": attempt_idx,
			"latency_ms": (int(latency_ms) if latency_ms is not None else None),
			"seed": (int(seed) if seed is not None else None),
			"env_fingerprint": env_fingerprint,
			"prompt": prompt,
			"completion": completion,
		}
		if meta is not None:
			rec["meta"] = meta
		with trace_path.open("a", encoding="utf-8") as tf:
			tf.write(__import__("json").dumps(rec) + "\n")
	except Exception:
		pass

def repair_code_failures(
	suite: str,
	failed_rows: List[Dict[str, Any]],
	args: Any,
	llm: Any,
	sanitize_completion: Any,
	strip_fences_fn: Any,
	samples_path: Path,
	verbose_path: Path,
	root: Path,
) -> None:
	"""Generic repair loop for code-body tasks. Appends repaired samples and sidecar rows.

	This mirrors existing HumanEval behavior while remaining reusable for other suites.
	"""
	repair_n = max(1, _safe_int(getattr(args, "repair_samples", 2) or 2, 2))
	if not failed_rows:
		return
	repairs_path = Path(root / "data" / "bench" / "tmp" / f"{suite}_repairs.jsonl")
	with trace_span("repair_code_failures", {"suite": suite, "n_failed": len(failed_rows)}), repairs_path.open("w", encoding="utf-8") as rp, samples_path.open("a", encoding="utf-8") as sf, verbose_path.open("a", encoding="utf-8") as vf:
		for row in failed_rows:
			tid = str(row.get("task_id", ""))
			orig_prompt = str(row.get("orig_prompt", ""))
			bad_body = str(row.get("completion", ""))
			repair_system = "You are a Python coding assistant. Fix the function body to pass tests. Output ONLY the corrected function body."
			repair_user = (
				"Task prompt (do not rewrite def):\n" + orig_prompt
				+ "\n\nPrevious attempt body (failing):\n" + bad_body
				+ f"\n\nFailure message: This attempt failed the {suite.capitalize()} tests. Provide a corrected function body only."
			)
			inp = repair_user
			params = {
				"max_new_tokens": _safe_int(getattr(args, "max_new_tokens", 256) or 256, 256),
				"temperature": _safe_float(getattr(args, "repair_temperature", 0.4) if getattr(args, "repair_temperature", None) is not None else 0.4, 0.4),
				"top_p": _safe_float(getattr(args, "repair_top_p", 0.95) if getattr(args, "repair_top_p", None) is not None else 0.95, 0.95),
			}
			for _ in range(repair_n):
				try:
					messages = [{"role": "system", "content": repair_system}, {"role": "user", "content": inp}]
					try:
						text = str(llm.chat(messages, **params))
					except Exception:
						text = str(llm.generate(inp, **params))
					clean = sanitize_completion(strip_fences_fn(text), orig_prompt)
					mini = {"task_id": tid, "completion": clean}
					full = {"task_id": tid, "completion": clean, "orig_prompt": orig_prompt, "input": inp}
					sf.write(__import__("json").dumps(mini) + "\n")
					vf.write(__import__("json").dumps(full) + "\n")
					rp.write(__import__("json").dumps({"task_id": tid, "repair": True}) + "\n")
					try:
						meter_cost("repair_sample", 1.0)
					except Exception:
						pass
				except Exception:
					continue

def write_suite_out(raw_out: str, args: Any, suite: str, task_ids: List[str], results_by_task: Dict[str, bool]) -> Path:
	"""Write the final per-task JSONL for a suite, handling shard suffixing."""
	from pathlib import Path as _Path
	import json as _json
	_raw_out = compute_sharded_outpath(str(raw_out), args)
	outp = _Path(_raw_out)
	outp.parent.mkdir(parents=True, exist_ok=True)
	with outp.open("w", encoding="utf-8") as f:
		for tid in task_ids:
			ok = bool(results_by_task.get(tid, False))
			f.write(_json.dumps({"suite": suite, "task_id": tid, "pass1": ok}) + "\n")
	return outp

# ---- Candidate ranking helpers ----
def pick_best_candidate(candidates: List[Dict[str, Any]], verify_risk_threshold: float) -> str:
	"""Choose best candidate using buckets: critic+precheck+lowrisk, precheck+lowrisk, critic, else length prior."""
	if not candidates:
		return ""
	thr = float(verify_risk_threshold)
	b1 = [c for c in candidates if c.get("critic") and c.get("precheck") and float(c.get("v_risk", 0.5)) <= thr]
	b2 = [c for c in candidates if c.get("precheck") and float(c.get("v_risk", 0.5)) <= thr]
	b3 = [c for c in candidates if c.get("critic")]
	b4 = candidates
	for bucket in (b1, b2, b3, b4):
		if bucket:
			best = sorted(bucket, key=lambda c: (c.get("length") or 1_000_000))[0]
			return str(best.get("body", ""))
	return ""

def rank_candidates(candidates: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
	"""Order candidates: critic+precheck first, then precheck, then critic, then rest; each by length prior."""
	b1 = [c for c in candidates if c.get("critic") and c.get("precheck")]
	b2 = [c for c in candidates if c.get("precheck") and c not in b1]
	b3 = [c for c in candidates if c.get("critic") and c not in b1 and c not in b2]
	b4 = [c for c in candidates if c not in b1 and c not in b2 and c not in b3]
	order: List[Dict[str, Any]] = []
	for bucket in (b1, b2, b3, b4):
		order.extend(sorted(bucket, key=lambda c: (c.get("length") or 1_000_000)))
	return [c for c in order if str(c.get("body", "")).strip()]

# ---- Run artifact helper ----
def write_run_artifact(
	root: Path,
	suite: str,
	run_started_ts: float,
	env: Dict[str, Any],
	prompts_hasher: Any,
	args_summary: Dict[str, Any],
	paths: Dict[str, str],
	aggregate: Any,
	metrics: Dict[str, Any],
	model: str,
	adapter_dir: str,
) -> None:
	import json as _json
	import hashlib as _hashlib
	import time as _time
	try:
		with trace_span("write_run_artifact", {"suite": suite}):
			runs_dir = Path(root / "data" / "bench" / "runs" / suite)
			runs_dir.mkdir(parents=True, exist_ok=True)
			ts_str = _time.strftime("%Y%m%dT%H%M%SZ", _time.gmtime(run_started_ts))
			art_dir = runs_dir / ts_str
			art_dir.mkdir(parents=True, exist_ok=True)
		try:
			_env_checksum = _hashlib.sha256(_json.dumps(env, sort_keys=True).encode("utf-8", errors="ignore")).hexdigest()
		except Exception:
			_env_checksum = None
		try:
			_prompt_pack_checksum = prompts_hasher.hexdigest()
		except Exception:
			_prompt_pack_checksum = None
		run_art = {
			"suite": suite,
			"started_ts": run_started_ts,
			"ended_ts": _time.time(),
			"model": model,
			"adapter_dir": adapter_dir,
			"args": args_summary,
			"paths": paths,
			"aggregate": (aggregate if isinstance(aggregate, dict) else {"raw": str(aggregate)}),
			"metrics": metrics,
			"env": env,
			"checksums": {"env": _env_checksum, "prompt_pack": _prompt_pack_checksum},
		}
		with (art_dir / "run.json").open("w", encoding="utf-8") as rf:
			rf.write(_json.dumps(run_art, ensure_ascii=False, indent=2))
	except Exception:
		pass

# ---- Run/Env helpers ----
def get_python_version() -> str:
	try:
		import sys as _sys  # type: ignore
		return str(getattr(_sys, "version", ""))
	except Exception:
		return ""

def env_fingerprint_for(mod_names: List[str]) -> Dict[str, Any]:
	_env: Dict[str, Any] = {}
	for mod_name in mod_names:
		try:
			mod = __import__(mod_name)
			_env[mod_name] = str(getattr(mod, "__version__", ""))
		except Exception:
			_env[mod_name] = None
	return _env

def build_env_fingerprint(mod_names: List[str]) -> Dict[str, Any]:
	return {"python": get_python_version(), **env_fingerprint_for(mod_names)}

def is_primary_process() -> bool:
	try:
		import os as _os
		ws = int(_os.environ.get("WORLD_SIZE", "1") or 1)
		rk = int(_os.environ.get("RANK", "0") or 0)
		return (ws <= 1) or (rk == 0)
	except Exception:
		return True

def seed_everything(seed: Optional[int]) -> None:
	try:
		if seed is None:
			return
		import random as _rnd  # type: ignore
		_rnd.seed(int(seed))
		try:
			import numpy as _np  # type: ignore
			_np.random.seed(int(seed))
		except Exception:
			pass
		try:
			import torch as _torch  # type: ignore
			_torch.manual_seed(int(seed))
			if _torch.cuda.is_available():
				try:
					_torch.cuda.manual_seed_all(int(seed))
				except Exception:
					pass
		except Exception:
			pass
	except Exception:
		pass

def get_shard_tag(args: Any) -> str:
	try:
		import os as _os
		ns = max(1, _safe_int(getattr(args, "num_shards", _os.environ.get("WORLD_SIZE", 1) or 1), 1))
		sid = max(0, _safe_int(getattr(args, "shard_id", _os.environ.get("RANK", 0) or 0), 0))
		return (f".shard{sid}-of-{ns}" if ns > 1 else "")
	except Exception:
		return ""

def init_trace_paths(root: Path, suite: str, run_started_ts: float) -> Tuple[Path, Path, str]:
	"""Create a timestamped trace directory for a suite run and return (traces_dir, trace_path, ts_str)."""
	import time as _time
	ts_str = _time.strftime("%Y%m%dT%H%M%SZ", _time.gmtime(run_started_ts))
	traces_dir = Path(root / "data" / "traces" / "bench_runs" / suite / ts_str)
	traces_dir.mkdir(parents=True, exist_ok=True)
	trace_path = traces_dir / "run.jsonl"
	return traces_dir, trace_path, ts_str

def init_suite_temp_paths(root: Path, suite: str, shard_tag: str) -> Tuple[Path, Path, Path]:
	"""Return (samples_path, verbose_path, errors_path) under bench/tmp for the suite and shard."""
	samples_path = Path(root / "data" / "bench" / "tmp" / (f"{suite}_samples" + shard_tag + ".jsonl"))
	verbose_path = Path(root / "data" / "bench" / "tmp" / (f"{suite}_samples.with_prompts" + shard_tag + ".jsonl"))
	errors_path = Path(root / "data" / "bench" / "tmp" / (f"{suite}_errors" + shard_tag + ".jsonl"))
	samples_path.parent.mkdir(parents=True, exist_ok=True)
	try:
		errors_path.write_text("")
	except Exception:
		pass
	return samples_path, verbose_path, errors_path

def read_pass_cache(root: Path, suite: str) -> Tuple[Dict[str, Dict[str, str]], Path]:
	"""Read per-suite pass cache JSONL into a dict; return (cache, path)."""
	path = Path(root / "data" / "bench" / "cache" / "results" / f"{suite}.jsonl")
	cache: Dict[str, Dict[str, str]] = {}
	try:
		if path.exists():
			for obj in read_jsonl(path):
				_tid = str(obj.get("task_id", ""))
				_comp = str(obj.get("completion", ""))
				if _tid and _comp:
					cache[_tid] = {"completion": _comp, "prompt": str(obj.get("prompt", ""))}
	except Exception:
		pass
	try:
		path.parent.mkdir(parents=True, exist_ok=True)
	except Exception:
		pass
	return cache, path

def update_pass_cache(pass_cache_path: Path, prev: Dict[str, Dict[str, str]], latest: Dict[str, Dict[str, str]]) -> None:
	merged: Dict[str, Dict[str, str]] = dict(prev)
	merged.update(latest)
	try:
		with pass_cache_path.open("w", encoding="utf-8") as pcw:
			for tid, rec in merged.items():
				pcw.write(__import__("json").dumps({"task_id": tid, "completion": rec.get("completion", ""), "prompt": rec.get("prompt", "")}, ensure_ascii=False) + "\n")
	except Exception:
		pass

def build_generation_params(args: Any) -> Tuple[Dict[str, Any], bool, int, int, int, float]:
	"""Return (params, grammar_constrained, n_samples, n_candidates, retries, retry_backoff)."""
	params: Dict[str, Any] = {
		"max_new_tokens": _safe_int(getattr(args, "max_new_tokens", 256) or 256, 256),
		"temperature": _safe_float(getattr(args, "temperature", 0.2) or 0.2, 0.2),
	}
	if getattr(args, "top_p", None) is not None:
		params["top_p"] = _safe_float(getattr(args, "top_p"), 0.95)
	if getattr(args, "top_k", None) is not None:
		params["top_k"] = _safe_int(getattr(args, "top_k"), 50)
	grammar_constrained = bool(getattr(args, "grammar_constrained", True))
	n_samples = max(1, _safe_int(getattr(args, "n_samples", 1) or 1, 1))
	n_candidates = max(1, _safe_int(getattr(args, "n_candidates", 3) or 3, 3))
	retries = max(0, _safe_int(getattr(args, "retries", 0) or 0, 0))
	retry_backoff = _safe_float(getattr(args, "retry_backoff", 0.75) or 0.75, 0.75)
	return params, grammar_constrained, n_samples, n_candidates, retries, retry_backoff

def summarize_telemetry(task_ids: List[str], results_by_task: Dict[str, bool]) -> Tuple[Dict[str, Any], Optional[float]]:
	try:
		pass_count = sum(1 for _tid, ok in results_by_task.items() if ok)
		pass1_rate = pass_count / max(1, len(task_ids))
		telemetry = {"n_tasks": len(task_ids), "pass1": pass1_rate}
		return telemetry, pass1_rate
	except Exception:
		return {}, None

def build_codebody_args_summary(args: Any) -> Dict[str, Any]:
    """Build a standard args summary dict for code-body suites."""
    return {
        "max_new_tokens": _safe_int(getattr(args, "max_new_tokens", 0) or 0, 0),
        "temperature": _safe_float(getattr(args, "temperature", 0.0) or 0.0, 0.0),
        "top_p": (_safe_float(getattr(args, "top_p"), 0.95) if getattr(args, "top_p", None
# ... truncated ...


from __future__ import annotations

from pathlib import Path
from typing import Any, Dict, List, Tuple, Optional
from agi_dw.core.ops.tracing import trace_span, meter_cost  # type: ignore

def looks_like_python_code(body: str) -> bool:
	"""Heuristic filter to check that a generated body resembles Python code.

	Shared across benchmarks to keep generation sanity checks consistent.
	"""
	b = (body or "").strip()
	if not b:
		return False
	if (b.startswith('"""') and b.endswith('"""')) or (b.startswith("'''") and b.endswith("'''")):
		return False
	# avoid degenerate bodies
	if b == "pass":
		return False
	for tok in ("return ", "= ", "=\n", "for ", "while ", "if ", "elif ", "yield ", "try:", "except "):
		if tok in b:
			return True
	# tolerate comprehensions with ':' and brackets
	if ":" in b and any(sym in b for sym in ("[", "{", "(")):
		return True
	return False

def build_python_body_chat(inp: str) -> Tuple[List[Dict[str, str]], List[str]]:
	"""Standard system+user chat and stop tokens for function-body generation."""
	messages: List[Dict[str, str]] = [
		{"role": "system", "content": "You are a Python coding assistant. Output ONLY the function body, properly indented under the provided def. No fences, no tests, no prints, no extra text. Do not include docstrings or comments."},
		{"role": "user", "content": inp},
	]
	stops = ["\n\n", "\nclass ", "\ndef ", "\nif __name__ == "]
	return messages, stops

def try_generate_python_body(llm: Any, inp: str, params: Dict[str, Any], grammar_constrained: bool) -> str:
	"""Generate a Python function body with optional grammar guidance; fall back robustly."""
	try:
		if grammar_constrained:
			try:
				import outlines  # type: ignore  # noqa: F401
				return str(llm.generate(inp, **{**params, "grammar": "python_function_body_minimal"}))
			except Exception:


	Shared across benchmarks to keep generation sanity checks consistent.
	"""
	b = (body or "").strip()
	if not b:
		return False
	if (b.startswith('"""') and b.endswith('"""')) or (b.startswith("'''") and b.endswith("'''")):
		return False
	# avoid degenerate bodies
	if b == "pass":
		return False
	for tok in ("return ", "= ", "=\n", "for ", "while ", "if ", "elif ", "yield ", "try:", "except "):
		if tok in b:
			return True
	# tolerate comprehensions with ':' and brackets
	if ":" in b and any(sym in b for sym in ("[", "{", "(")):
		return True
	return False

def build_python_body_chat(inp: str) -> Tuple[List[Dict[str, str]], List[str]]:
	"""Standard system+user chat and stop tokens for function-body generation."""
	messages: List[Dict[str, str]] = [
		{"role": "system", "content": "You are a Python coding assistant. Output ONLY the function body, properly indented under the provided def. No fences, no tests, no prints, no extra text. Do not include docstrings or comments."},
		{"role": "user", "content": inp},
	]
	stops = ["\n\n", "\nclass ", "\ndef ", "\nif __name__ == "]
	return messages, stops

def try_generate_python_body(llm: Any, inp: str, params: Dict[str, Any], grammar_constrained: bool) -> str:
	"""Generate a Python function body with optional grammar guidance; fall back robustly."""
	try:
		if grammar_constrained:
			try:
				import outlines  # type: ignore  # noqa: F401
				return str(llm.generate(inp, **{**params, "grammar": "python_function_body_minimal"}))
			except Exception:
				pass
		messages, stops = build_python_body_chat(inp)
		return str(llm.chat(messages, max_new_tokens=params.get("max_new_tokens", 256), temperature=params.get("temperature", 0.0), top_p=params.get("top_p"), top_k=params.get("top_k"), stop=stops))
	except Exception:
		return str(llm.generate(inp, **params))

def attach_adapter_from_args(llm: Any, args: Any, root: Path, default_adapter: Optional[Path] = None) -> None:
	"""Attach an adapter to the LLM based on args, adapter bank, and defaults.


		return False
	for tok in ("return ", "= ", "=\n", "for ", "while ", "if ", "elif ", "yield ", "try:", "except "):
		if tok in b:
			return True
	# tolerate comprehensions with ':' and brackets
	if ":" in b and any(sym in b for sym in ("[", "{", "(")):
		return True
	return False

def build_python_body_chat(inp: str) -> Tuple[List[Dict[str, str]], List[str]]:
	"""Standard system+user chat and stop tokens for function-body generation."""
	messages: List[Dict[str, str]] = [
		{"role": "system", "content": "You are a Python coding assistant. Output ONLY the function body, properly indented under the provided def. No fences, no tests, no prints, no extra text. Do not include docstrings or comments."},
		{"role": "user", "content": inp},
	]
	stops = ["\n\n", "\nclass ", "\ndef ", "\nif __name__ == "]
	return messages, stops

def try_generate_python_body(llm: Any, inp: str, params: Dict[str, Any], grammar_constrained: bool) -> str:
	"""Generate a Python function body with optional grammar guidance; fall back robustly."""
	try:
		if grammar_constrained:
			try:
				import outlines  # type: ignore  # noqa: F401
				return str(llm.generate(inp, **{**params, "grammar": "python_function_body_minimal"}))
			except Exception:
				pass
		messages, stops = build_python_body_chat(inp)
		return str(llm.chat(messages, max_new_tokens=params.get("max_new_tokens", 256), temperature=params.get("temperature", 0.0), top_p=params.get("top_p"), top_k=params.get("top_k"), stop=stops))
	except Exception:
		return str(llm.generate(inp, **params))

def attach_adapter_from_args(llm: Any, args: Any, root: Path, default_adapter: Optional[Path] = None) -> None:
	"""Attach an adapter to the LLM based on args, adapter bank, and defaults.

	- Respects --adapter_dir
	- If --adapter_bank is provided, chooses a planner/verifier mapping from bank
	- If adapter not specified, uses default_adapter if provided and exists
	"""
	try:
		adapter_dir = getattr(args, "adapter_dir", None)
		bank = getattr(args, "adapter_bank", None)
		if bank:
			try:
				from agi_dw.core.llm.adapter_router import pick_from_bank  # type: ignore
				mapping = pick_from_bank(root, str(bank))
				cand = mapping.get("planner") or mapping.get("verifier")
				if cand:
					adapter_dir = str(cand)
			except Exception:


	]
	stops = ["\n\n", "\nclass ", "\ndef ", "\nif __name__ == "]
	return messages, stops

def try_generate_python_body(llm: Any, inp: str, params: Dict[str, Any], grammar_constrained: bool) -> str:
	"""Generate a Python function body with optional grammar guidance; fall back robustly."""
	try:
		if grammar_constrained:
			try:
				import outlines  # type: ignore  # noqa: F401
				return str(llm.generate(inp, **{**params, "grammar": "python_function_body_minimal"}))
			except Exception:
				pass
		messages, stops = build_python_body_chat(inp)
		return str(llm.chat(messages, max_new_tokens=params.get("max_new_tokens", 256), temperature=params.get("temperature", 0.0), top_p=params.get("top_p"), top_k=params.get("top_k"), stop=stops))
	except Exception:
		return str(llm.generate(inp, **params))

def attach_adapter_from_args(llm: Any, args: Any, root: Path, default_adapter: Optional[Path] = None) -> None:
	"""Attach an adapter to the LLM based on args, adapter bank, and defaults.

	- Respects --adapter_dir
	- If --adapter_bank is provided, chooses a planner/verifier mapping from bank
	- If adapter not specified, uses default_adapter if provided and exists
	"""
	try:
		adapter_dir = getattr(args, "adapter_dir", None)
		bank = getattr(args, "adapter_bank", None)
		if bank:
			try:
				from agi_dw.core.llm.adapter_router import pick_from_bank  # type: ignore
				mapping = pick_from_bank(root, str(bank))
				cand = mapping.get("planner") or mapping.get("verifier")
				if cand:
					adapter_dir = str(cand)
			except Exception:
				pass
		if (not adapter_dir) or (isinstance(adapter_dir, str) and not str(adapter_dir).strip()):
			if default_adapter is not None and Path(default_adapter).exists():
				adapter_dir = str(default_adapter)
		llm.attach_adapter(str(adapter_dir) or None)
	except Exception:
		pass

# ---- Evaluation and results helpers ----
def _safe_int(val: Any, default: int) -> int:
	try:
		return int(val)
	except Exception:
		return int(default)

def _safe_float(val: Any, default: float) -> float:
	try:
		return float(val)
	except Exception:
		return float(default)

def parse_k_list(args: Any) -> List[int]:
	try:
		return [int(x) for x in str(getattr(args, "k", "1") or "1").split(",") if str(x).strip()]


		adapter_dir = getattr(args, "adapter_dir", None)
		bank = getattr(args, "adapter_bank", None)
		if bank:
			try:
				from agi_dw.core.llm.adapter_router import pick_from_bank  # type: ignore
				mapping = pick_from_bank(root, str(bank))
				cand = mapping.get("planner") or mapping.get("verifier")
				if cand:
					adapter_dir = str(cand)
			except Exception:
				pass
		if (not adapter_dir) or (isinstance(adapter_dir, str) and not str(adapter_dir).strip()):
			if default_adapter is not None and Path(default_adapter).exists():
				adapter_dir = str(default_adapter)
		llm.attach_adapter(str(adapter_dir) or None)
	except Exception:
		pass

# ---- Evaluation and results helpers ----
def _safe_int(val: Any, default: int) -> int:
	try:
		return int(val)
	except Exception:
		return int(default)

def _safe_float(val: Any, default: float) -> float:
	try:
		return float(val)
	except Exception:
		return float(default)

def parse_k_list(args: Any) -> List[int]:
	try:
		return [int(x) for x in str(getattr(args, "k", "1") or "1").split(",") if str(x).strip()]
	except Exception:
		return [1]

def evaluate_samples(samples_path: Path, k_list: List[int], default_workers: int, args: Any, evaluate_fn: Any) -> Tuple[Any, Path]:
	"""Run evaluation using provided evaluate_fn and return (aggregate_result, results_path)."""
	with trace_span("evaluate_samples", {"k": k_list, "timeout": _safe_float(getattr(args, "timeout", 15) or 15, 15.0)}):
		res = evaluate_fn(


				if cand:
					adapter_dir = str(cand)
			except Exception:
				pass
		if (not adapter_dir) or (isinstance(adapter_dir, str) and not str(adapter_dir).strip()):
			if default_adapter is not None and Path(default_adapter).exists():
				adapter_dir = str(default_adapter)
		llm.attach_adapter(str(adapter_dir) or None)
	except Exception:
		pass

# ---- Evaluation and results helpers ----
def _safe_int(val: Any, default: int) -> int:
	try:
		return int(val)
	except Exception:
		return int(default)

def _safe_float(val: Any, default: float) -> float:
	try:
		return float(val)
	except Exception:
		return float(default)

def parse_k_list(args: Any) -> List[int]:
	try:
		return [int(x) for x in str(getattr(args, "k", "1") or "1").split(",") if str(x).strip()]
	except Exception:
		return [1]

def evaluate_samples(samples_path: Path, k_list: List[int], default_workers: int, args: Any, evaluate_fn: Any) -> Tuple[Any, Path]:
	"""Run evaluation using provided evaluate_fn and return (aggregate_result, results_path)."""
	with trace_span("evaluate_samples", {"k": k_list, "timeout": _safe_float(getattr(args, "timeout", 15) or 15, 15.0)}):
		res = evaluate_fn(
			sample_file=str(samples_path),
			k=k_list,
			n_workers=max(1, _safe_int(getattr(args, "max_workers", default_workers) or default_workers, default_workers)),
			timeout=_safe_float(getattr(args, "timeout", 15) or 15, 15.0),
			ignore_incomplete=True,
		)
	results_path = Path(str(samples_path) + "_results.jsonl")


		llm.attach_adapter(str(adapter_dir) or None)
	except Exception:
		pass

# ---- Evaluation and results helpers ----
def _safe_int(val: Any, default: int) -> int:
	try:
		return int(val)
	except Exception:
		return int(default)

def _safe_float(val: Any, default: float) -> float:
	try:
		return float(val)
	except Exception:
		return float(default)

def parse_k_list(args: Any) -> List[int]:
	try:
		return [int(x) for x in str(getattr(args, "k", "1") or "1").split(",") if str(x).strip()]
	except Exception:
		return [1]

def evaluate_samples(samples_path: Path, k_list: List[int], default_workers: int, args: Any, evaluate_fn: Any) -> Tuple[Any, Path]:
	"""Run evaluation using provided evaluate_fn and return (aggregate_result, results_path)."""
	with trace_span("evaluate_samples", {"k": k_list, "timeout": _safe_float(getattr(args, "timeout", 15) or 15, 15.0)}):
		res = evaluate_fn(
			sample_file=str(samples_path),
			k=k_list,
			n_workers=max(1, _safe_int(getattr(args, "max_workers", default_workers) or default_workers, default_workers)),
			timeout=_safe_float(getattr(args, "timeout", 15) or 15, 15.0),
			ignore_incomplete=True,
		)
	results_path = Path(str(samples_path) + "_results.jsonl")
	return res, results_path

def read_jsonl(path: Path) -> List[Dict[str, Any]]:
	rows: List[Dict[str, Any]] = []
	if not path.exists():
		return rows


	try:
		return int(val)
	except Exception:
		return int(default)

def _safe_float(val: Any, default: float) -> float:
	try:
		return float(val)
	except Exception:
		return float(default)

def parse_k_list(args: Any) -> List[int]:
	try:
		return [int(x) for x in str(getattr(args, "k", "1") or "1").split(",") if str(x).strip()]
	except Exception:
		return [1]

def evaluate_samples(samples_path: Path, k_list: List[int], default_workers: int, args: Any, evaluate_fn: Any) -> Tuple[Any, Path]:
	"""Run evaluation using provided evaluate_fn and return (aggregate_result, results_path)."""
	with trace_span("evaluate_samples", {"k": k_list, "timeout": _safe_float(getattr(args, "timeout", 15) or 15, 15.0)}):
		res = evaluate_fn(
			sample_file=str(samples_path),
			k=k_list,
			n_workers=max(1, _safe_int(getattr(args, "max_workers", default_workers) or default_workers, default_workers)),
			timeout=_safe_float(getattr(args, "timeout", 15) or 15, 15.0),
			ignore_incomplete=True,
		)
	results_path = Path(str(samples_path) + "_results.jsonl")
	return res, results_path

def read_jsonl(path: Path) -> List[Dict[str, Any]]:
	rows: List[Dict[str, Any]] = []
	if not path.exists():
		return rows
	with path.open("r", encoding="utf-8") as f:
		for line in f:
			li = line.strip()
			if not li:
				continue
			try:
				rows.append(__import__("json").loads(li))
			except Exception:
				continue
	return rows

def build_results_by_task(results_path: Path) -> Dict[str, bool]:
	by_task: Dict[str, bool] = {}


	try:
		return [int(x) for x in str(getattr(args, "k", "1") or "1").split(",") if str(x).strip()]
	except Exception:
		return [1]

def evaluate_samples(samples_path: Path, k_list: List[int], default_workers: int, args: Any, evaluate_fn: Any) -> Tuple[Any, Path]:
	"""Run evaluation using provided evaluate_fn and return (aggregate_result, results_path)."""
	with trace_span("evaluate_samples", {"k": k_list, "timeout": _safe_float(getattr(args, "timeout", 15) or 15, 15.0)}):
		res = evaluate_fn(
			sample_file=str(samples_path),
			k=k_list,
			n_workers=max(1, _safe_int(getattr(args, "max_workers", default_workers) or default_workers, default_workers)),
			timeout=_safe_float(getattr(args, "timeout", 15) or 15, 15.0),
			ignore_incomplete=True,
		)
	results_path = Path(str(samples_path) + "_results.jsonl")
	return res, results_path

def read_jsonl(path: Path) -> List[Dict[str, Any]]:
	rows: List[Dict[str, Any]] = []
	if not path.exists():
		return rows
	with path.open("r", encoding="utf-8") as f:
		for line in f:
			li = line.strip()
			if not li:
				continue
			try:
				rows.append(__import__("json").loads(li))
			except Exception:
				continue
	return rows

def build_results_by_task(results_path: Path) -> Dict[str, bool]:
	by_task: Dict[str, bool] = {}
	for row in read_jsonl(results_path):
		tid = str(row.get("task_id", ""))
		passed = bool(row.get("passed", False))
		if tid:
			by_task[tid] = bool(by_task.get(tid, False) or passed)
	return by_task

def dedupe_by_passed(samples_path: Path, verbose_path: Path, results_path: Path) -> None:
	"""Keep only passing completions per task in samples and verbose sidecar."""
	passed_choice: Dict[str, str] = {}
	for row in read_jsonl(results_path):
		if bool(row.get("passed", False)):
			_tid = str(row.get("task_id", ""))
			_comp = str(row.get("completion", ""))
			if _tid and _comp and _tid not in passed_choice:


	results_path = Path(str(samples_path) + "_results.jsonl")
	return res, results_path

def read_jsonl(path: Path) -> List[Dict[str, Any]]:
	rows: List[Dict[str, Any]] = []
	if not path.exists():
		return rows
	with path.open("r", encoding="utf-8") as f:
		for line in f:
			li = line.strip()
			if not li:
				continue
			try:
				rows.append(__import__("json").loads(li))
			except Exception:
				continue
	return rows

def build_results_by_task(results_path: Path) -> Dict[str, bool]:
	by_task: Dict[str, bool] = {}
	for row in read_jsonl(results_path):
		tid = str(row.get("task_id", ""))
		passed = bool(row.get("passed", False))
		if tid:
			by_task[tid] = bool(by_task.get(tid, False) or passed)
	return by_task

def dedupe_by_passed(samples_path: Path, verbose_path: Path, results_path: Path) -> None:
	"""Keep only passing completions per task in samples and verbose sidecar."""
	passed_choice: Dict[str, str] = {}
	for row in read_jsonl(results_path):
		if bool(row.get("passed", False)):
			_tid = str(row.get("task_id", ""))
			_comp = str(row.get("completion", ""))
			if _tid and _comp and _tid not in passed_choice:
				passed_choice[_tid] = _comp
	if not passed_choice:
		return
	keep: List[Dict[str, str]] = []
	for obj in read_jsonl(samples_path):
		_tid = str(obj.get("task_id", ""))
		_comp = str(obj.get("completion", ""))
		if _tid in passed_choice and _comp == passed_choice[_tid]:
			keep.append({"task_id": _tid, "completion": _comp})
	with samples_path.open("w", encoding="utf-8") as sfw:


			li = line.strip()
			if not li:
				continue
			try:
				rows.append(__import__("json").loads(li))
			except Exception:
				continue
	return rows

def build_results_by_task(results_path: Path) -> Dict[str, bool]:
	by_task: Dict[str, bool] = {}
	for row in read_jsonl(results_path):
		tid = str(row.get("task_id", ""))
		passed = bool(row.get("passed", False))
		if tid:
			by_task[tid] = bool(by_task.get(tid, False) or passed)
	return by_task

def dedupe_by_passed(samples_path: Path, verbose_path: Path, results_path: Path) -> None:
	"""Keep only passing completions per task in samples and verbose sidecar."""
	passed_choice: Dict[str, str] = {}
	for row in read_jsonl(results_path):
		if bool(row.get("passed", False)):
			_tid = str(row.get("task_id", ""))
			_comp = str(row.get("completion", ""))
			if _tid and _comp and _tid not in passed_choice:
				passed_choice[_tid] = _comp
	if not passed_choice:
		return
	keep: List[Dict[str, str]] = []
	for obj in read_jsonl(samples_path):
		_tid = str(obj.get("task_id", ""))
		_comp = str(obj.get("completion", ""))
		if _tid in passed_choice and _comp == passed_choice[_tid]:
			keep.append({"task_id": _tid, "completion": _comp})
	with samples_path.open("w", encoding="utf-8") as sfw:
		for obj in keep:
			sfw.write(__import__("json").dumps(obj) + "\n")
	keep_verbose: List[Dict[str, Any]] = []
	for obj in read_jsonl(verbose_path):
		_tid = str(obj.get("task_id", ""))
		_comp = str(obj.get("completion", ""))
		if _tid in passed_choice and _comp == passed_choice[_tid]:
			keep_verbose.append(obj)
	with verbose_path.open("w", encoding="utf-8") as vfw:
		for obj in keep_verbose:
			vfw.write(__import__("json").dumps(obj) + "\n")

# ---- Sharding helpers ----
def shard_task_ids(task_ids: List[str], args: Any) -> List[str]:
	"""Shard a list of task_ids across processes using num_shards/shard_id or torchrun envs."""
	try:
		import os as _os
		num_shards = max(1, _safe_int(getattr(args, "num_shards", _os.environ.get("WORLD_SIZE", 1) or 1), 1))
		shard_id = max(0, _safe_int(getattr(args, "shard_id", _os.environ.get("RANK", 0) or 0), 0))
		if num_shards > 1:
			return [tid for i, tid in enumerate(task_ids) if (i % num_shards) == shard_id]
	except Exception:
		pass
	return task_ids

def compute_sharded_outpath(raw_out: str, args: Any) -> str:
	"""Append shard suffix to output path when sharded."""
	try:
		import os as _os


	for obj in read_jsonl(samples_path):
		_tid = str(obj.get("task_id", ""))
		_comp = str(obj.get("completion", ""))
		if _tid in passed_choice and _comp == passed_choice[_tid]:
			keep.append({"task_id": _tid, "completion": _comp})
	with samples_path.open("w", encoding="utf-8") as sfw:
		for obj in keep:
			sfw.write(__import__("json").dumps(obj) + "\n")
	keep_verbose: List[Dict[str, Any]] = []
	for obj in read_jsonl(verbose_path):
		_tid = str(obj.get("task_id", ""))
		_comp = str(obj.get("completion", ""))
		if _tid in passed_choice and _comp == passed_choice[_tid]:
			keep_verbose.append(obj)
	with verbose_path.open("w", encoding="utf-8") as vfw:
		for obj in keep_verbose:
			vfw.write(__import__("json").dumps(obj) + "\n")

# ---- Sharding helpers ----
def shard_task_ids(task_ids: List[str], args: Any) -> List[str]:
	"""Shard a list of task_ids across processes using num_shards/shard_id or torchrun envs."""
	try:
		import os as _os
		num_shards = max(1, _safe_int(getattr(args, "num_shards", _os.environ.get("WORLD_SIZE", 1) or 1), 1))
		shard_id = max(0, _safe_int(getattr(args, "shard_id", _os.environ.get("RANK", 0) or 0), 0))
		if num_shards > 1:
			return [tid for i, tid in enumerate(task_ids) if (i % num_shards) == shard_id]
	except Exception:
		pass
	return task_ids

def compute_sharded_outpath(raw_out: str, args: Any) -> str:
	"""Append shard suffix to output path when sharded."""
	try:
		import os as _os
		num_shards = max(1, _safe_int(getattr(args, "num_shards", _os.environ.get("WORLD_SIZE", 1) or 1), 1))
		shard_id = max(0, _safe_int(getattr(args, "shard_id", _os.environ.get("RANK", 0) or 0), 0))
		if num_shards > 1:
			p = Path(raw_out)
			return str(p.with_name(p.stem + f".shard{shard_id}-of-{num_shards}" + p.suffix))
	except Exception:
		pass
	return str(raw_out)

# ---- Trace helpers ----
def write_trace(
	trace_path: Path,
	suite: str,


			keep_verbose.append(obj)
	with verbose_path.open("w", encoding="utf-8") as vfw:
		for obj in keep_verbose:
			vfw.write(__import__("json").dumps(obj) + "\n")

# ---- Sharding helpers ----
def shard_task_ids(task_ids: List[str], args: Any) -> List[str]:
	"""Shard a list of task_ids across processes using num_shards/shard_id or torchrun envs."""
	try:
		import os as _os
		num_shards = max(1, _safe_int(getattr(args, "num_shards", _os.environ.get("WORLD_SIZE", 1) or 1), 1))
		shard_id = max(0, _safe_int(getattr(args, "shard_id", _os.environ.get("RANK", 0) or 0), 0))
		if num_shards > 1:
			return [tid for i, tid in enumerate(task_ids) if (i % num_shards) == shard_id]
	except Exception:
		pass
	return task_ids

def compute_sharded_outpath(raw_out: str, args: Any) -> str:
	"""Append shard suffix to output path when sharded."""
	try:
		import os as _os
		num_shards = max(1, _safe_int(getattr(args, "num_shards", _os.environ.get("WORLD_SIZE", 1) or 1), 1))
		shard_id = max(0, _safe_int(getattr(args, "shard_id", _os.environ.get("RANK", 0) or 0), 0))
		if num_shards > 1:
			p = Path(raw_out)
			return str(p.with_name(p.stem + f".shard{shard_id}-of-{num_shards}" + p.suffix))
	except Exception:
		pass
	return str(raw_out)

# ---- Trace helpers ----
def write_trace(
	trace_path: Path,
	suite: str,
	task_id: str,
	attempt_idx: Any,
	latency_ms: Optional[int],
	seed: Optional[int],
	env_fingerprint: Dict[str, Any],
	prompt: str,
	completion: str,
	meta: Optional[Dict[str, Any]] = None,
) -> None:
	"""Append a single trace record line (JSON) to the given path."""
	try:
		rec: Dict[str, Any] = {
			"suite": suite,
			"task_id": task_id,


	except Exception:
		pass
	return task_ids

def compute_sharded_outpath(raw_out: str, args: Any) -> str:
	"""Append shard suffix to output path when sharded."""
	try:
		import os as _os
		num_shards = max(1, _safe_int(getattr(args, "num_shards", _os.environ.get("WORLD_SIZE", 1) or 1), 1))
		shard_id = max(0, _safe_int(getattr(args, "shard_id", _os.environ.get("RANK", 0) or 0), 0))
		if num_shards > 1:
			p = Path(raw_out)
			return str(p.with_name(p.stem + f".shard{shard_id}-of-{num_shards}" + p.suffix))
	except Exception:
		pass
	return str(raw_out)

# ---- Trace helpers ----
def write_trace(
	trace_path: Path,
	suite: str,
	task_id: str,
	attempt_idx: Any,
	latency_ms: Optional[int],
	seed: Optional[int],
	env_fingerprint: Dict[str, Any],
	prompt: str,
	completion: str,
	meta: Optional[Dict[str, Any]] = None,
) -> None:
	"""Append a single trace record line (JSON) to the given path."""
	try:
		rec: Dict[str, Any] = {
			"suite": suite,
			"task_id": task_id,
			"attempt_idx": attempt_idx,
			"latency_ms": (int(latency_ms) if latency_ms is not None else None),
			"seed": (int(seed) if seed is not None else None),
			"env_fingerprint": env_fingerprint,
			"prompt": prompt,
			"completion": completion,
		}
		if meta is not None:
			rec["meta"] = meta
		with trace_path.open("a", encoding="utf-8") as tf:
			tf.write(__import__("json").dumps(rec) + "\n")
	except Exception:
		pass

def repair_code_failures(
	suite: str,
	failed_rows: List[Dict[str, Any]],
	args: Any,
	llm: Any,
	sanitize_completion: Any,
	strip_fences_fn: Any,
	samples_path: Path,
	verbose_path: Path,
	root: Path,
) -> None:
	"""Generic repair loop for code-body tasks. Appends repaired samples and sidecar rows.

	This mirrors existing HumanEval behavior while remaining reusable for other suites.
	"""
	repair_n = max(1, _safe_int(getattr(args, "repair_samples", 2) or 2, 2))
	if not failed_rows:
		return


	"""Append a single trace record line (JSON) to the given path."""
	try:
		rec: Dict[str, Any] = {
			"suite": suite,
			"task_id": task_id,
			"attempt_idx": attempt_idx,
			"latency_ms": (int(latency_ms) if latency_ms is not None else None),
			"seed": (int(seed) if seed is not None else None),
			"env_fingerprint": env_fingerprint,
			"prompt": prompt,
			"completion": completion,
		}
		if meta is not None:
			rec["meta"] = meta
		with trace_path.open("a", encoding="utf-8") as tf:
			tf.write(__import__("json").dumps(rec) + "\n")
	except Exception:
		pass

def repair_code_failures(
	suite: str,
	failed_rows: List[Dict[str, Any]],
	args: Any,
	llm: Any,
	sanitize_completion: Any,
	strip_fences_fn: Any,
	samples_path: Path,
	verbose_path: Path,
	root: Path,
) -> None:
	"""Generic repair loop for code-body tasks. Appends repaired samples and sidecar rows.

	This mirrors existing HumanEval behavior while remaining reusable for other suites.
	"""
	repair_n = max(1, _safe_int(getattr(args, "repair_samples", 2) or 2, 2))
	if not failed_rows:
		return
	repairs_path = Path(root / "data" / "bench" / "tmp" / f"{suite}_repairs.jsonl")
	with trace_span("repair_code_failures", {"suite": suite, "n_failed": len(failed_rows)}), repairs_path.open("w", encoding="utf-8") as rp, samples_path.open("a", encoding="utf-8") as sf, verbose_path.open("a", encoding="utf-8") as vf:
		for row in failed_rows:
			tid = str(row.get("task_id", ""))
			orig_prompt = str(row.get("orig_prompt", ""))
			bad_body = str(row.get("completion", ""))
			repair_system = "You are a Python coding assistant. Fix the function body to pass tests. Output ONLY the corrected function body."
			repair_user = (
				"Task prompt (do not rewrite def):\n" + orig_prompt
				+ "\n\nPrevious attempt body (failing):\n" + bad_body
				+ f"\n\nFailure message: This attempt failed the {suite.capitalize()} tests. Provide a corrected function body only."
			)
			inp = repair_user
			params = {
				"max_new_tokens": _safe_int(getattr(args, "max_new_tokens", 256) or 256, 256),
				"temperature": _safe_float(getattr(args, "repair_temperature", 0.4) if getattr(args, "repair_temperature", None) is not None else 0.4, 0.4),
				"top_p": _safe_float(getattr(args, "repair_top_p", 0.95) if getattr(args, "repair_top_p", None) is not None else 0.95, 0.95),
			}
			for _ in range(repair_n):
				try:
					messages = [{"role": "system", "content": repair_system}, {"role": "user", "content": inp}]
					try:
						text = str(llm.chat(messages, **params))
					except Exception:
						text = str(llm.generate(inp, **params))
					clean = sanitize_completion(strip_fences_fn(text), orig_prompt)
					mini = {"task_id": tid, "completion": clean}
					full = {"task_id": tid, "completion": clean, "orig_prompt": orig_prompt, "input": inp}
					sf.write(__import__("json").dumps(mini) + "\n")
					vf.write(__import__("json").dumps(full) + "\n")
					rp.write(__import__("json").dumps({"task_id": tid, "repair": True}) + "\n")
					try:
						meter_cost("repair_sample", 1.0)
					except Exception:
						pass
				except Exception:
					continue

def write_suite_out(raw_out: str, args: Any, suite: str, task_ids: List[str], results_by_task: Dict[str, bool]) -> Path:
	"""Write the final per-task JSONL for a suite, handling shard suffixing."""
	from pathlib import Path as _Path
	import json as _json
	_raw_out = compute_sharded_outpath(str(raw_out), args)
	outp = _Path(_raw_out)
	outp.parent.mkdir(parents=True, exist_ok=True)
	with outp.open("w", encoding="utf-8") as f:
		for tid in task_ids:
			ok = bool(results_by_task.get(tid, False))
			f.write(_json.dumps({"suite": suite, "task_id": tid, "pass1": ok}) + "\n")
	return outp

# ---- Candidate ranking helpers ----
def pick_best_candidate(candidates: List[Dict[str, Any]], verify_risk_threshold: float) -> str:
	"""Choose best candidate using buckets: critic+precheck+lowrisk, precheck+lowrisk, critic, else length prior."""
	if not candidates:


				try:
					messages = [{"role": "system", "content": repair_system}, {"role": "user", "content": inp}]
					try:
						text = str(llm.chat(messages, **params))
					except Exception:
						text = str(llm.generate(inp, **params))
					clean = sanitize_completion(strip_fences_fn(text), orig_prompt)
					mini = {"task_id": tid, "completion": clean}
					full = {"task_id": tid, "completion": clean, "orig_prompt": orig_prompt, "input": inp}
					sf.write(__import__("json").dumps(mini) + "\n")
					vf.write(__import__("json").dumps(full) + "\n")
					rp.write(__import__("json").dumps({"task_id": tid, "repair": True}) + "\n")
					try:
						meter_cost("repair_sample", 1.0)
					except Exception:
						pass
				except Exception:
					continue

def write_suite_out(raw_out: str, args: Any, suite: str, task_ids: List[str], results_by_task: Dict[str, bool]) -> Path:
	"""Write the final per-task JSONL for a suite, handling shard suffixing."""
	from pathlib import Path as _Path
	import json as _json
	_raw_out = compute_sharded_outpath(str(raw_out), args)
	outp = _Path(_raw_out)
	outp.parent.mkdir(parents=True, exist_ok=True)
	with outp.open("w", encoding="utf-8") as f:
		for tid in task_ids:
			ok = bool(results_by_task.get(tid, False))
			f.write(_json.dumps({"suite": suite, "task_id": tid, "pass1": ok}) + "\n")
	return outp

# ---- Candidate ranking helpers ----
def pick_best_candidate(candidates: List[Dict[str, Any]], verify_risk_threshold: float) -> str:
	"""Choose best candidate using buckets: critic+precheck+lowrisk, precheck+lowrisk, critic, else length prior."""
	if not candidates:
		return ""
	thr = float(verify_risk_threshold)
	b1 = [c for c in candidates if c.get("critic") and c.get("precheck") and float(c.get("v_risk", 0.5)) <= thr]
	b2 = [c for c in candidates if c.get("precheck") and float(c.get("v_risk", 0.5)) <= thr]
	b3 = [c for c in candidates if c.get("critic")]
	b4 = candidates
	for bucket in (b1, b2, b3, b4):
		if bucket:
			best = sorted(bucket, key=lambda c: (c.get("length") or 1_000_000))[0]
			return str(best.get("body", ""))
	return ""

def rank_candidates(candidates: List[Dict[str, Any]]) -> List[Dict[str, Any]]:


						pass
				except Exception:
					continue

def write_suite_out(raw_out: str, args: Any, suite: str, task_ids: List[str], results_by_task: Dict[str, bool]) -> Path:
	"""Write the final per-task JSONL for a suite, handling shard suffixing."""
	from pathlib import Path as _Path
	import json as _json
	_raw_out = compute_sharded_outpath(str(raw_out), args)
	outp = _Path(_raw_out)
	outp.parent.mkdir(parents=True, exist_ok=True)
	with outp.open("w", encoding="utf-8") as f:
		for tid in task_ids:
			ok = bool(results_by_task.get(tid, False))
			f.write(_json.dumps({"suite": suite, "task_id": tid, "pass1": ok}) + "\n")
	return outp

# ---- Candidate ranking helpers ----
def pick_best_candidate(candidates: List[Dict[str, Any]], verify_risk_threshold: float) -> str:
	"""Choose best candidate using buckets: critic+precheck+lowrisk, precheck+lowrisk, critic, else length prior."""
	if not candidates:
		return ""
	thr = float(verify_risk_threshold)
	b1 = [c for c in candidates if c.get("critic") and c.get("precheck") and float(c.get("v_risk", 0.5)) <= thr]
	b2 = [c for c in candidates if c.get("precheck") and float(c.get("v_risk", 0.5)) <= thr]
	b3 = [c for c in candidates if c.get("critic")]
	b4 = candidates
	for bucket in (b1, b2, b3, b4):
		if bucket:
			best = sorted(bucket, key=lambda c: (c.get("length") or 1_000_000))[0]
			return str(best.get("body", ""))
	return ""

def rank_candidates(candidates: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
	"""Order candidates: critic+precheck first, then precheck, then critic, then rest; each by length prior."""
	b1 = [c for c in candidates if c.get("critic") and c.get("precheck")]
	b2 = [c for c in candidates if c.get("precheck") and c not in b1]
	b3 = [c for c in candidates if c.get("critic") and c not in b1 and c not in b2]
	b4 = [c for c in candidates if c not in b1 and c not in b2 and c not in b3]
	order: List[Dict[str, Any]] = []
	for bucket in (b1, b2, b3, b4):
		order.extend(sorted(bucket, key=lambda c: (c.get("length") or 1_000_000)))
	return [c for c in order if str(c.get("body", "")).strip()]

# ---- Run artifact helper ----
def write_run_artifact(
	root: Path,
	suite: str,
	run_started_ts: float,
	env: Dict[str, Any],


	return outp

# ---- Candidate ranking helpers ----
def pick_best_candidate(candidates: List[Dict[str, Any]], verify_risk_threshold: float) -> str:
	"""Choose best candidate using buckets: critic+precheck+lowrisk, precheck+lowrisk, critic, else length prior."""
	if not candidates:
		return ""
	thr = float(verify_risk_threshold)
	b1 = [c for c in candidates if c.get("critic") and c.get("precheck") and float(c.get("v_risk", 0.5)) <= thr]
	b2 = [c for c in candidates if c.get("precheck") and float(c.get("v_risk", 0.5)) <= thr]
	b3 = [c for c in candidates if c.get("critic")]
	b4 = candidates
	for bucket in (b1, b2, b3, b4):
		if bucket:
			best = sorted(bucket, key=lambda c: (c.get("length") or 1_000_000))[0]
			return str(best.get("body", ""))
	return ""

def rank_candidates(candidates: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
	"""Order candidates: critic+precheck first, then precheck, then critic, then rest; each by length prior."""
	b1 = [c for c in candidates if c.get("critic") and c.get("precheck")]
	b2 = [c for c in candidates if c.get("precheck") and c not in b1]
	b3 = [c for c in candidates if c.get("critic") and c not in b1 and c not in b2]
	b4 = [c for c in candidates if c not in b1 and c not in b2 and c not in b3]
	order: List[Dict[str, Any]] = []
	for bucket in (b1, b2, b3, b4):
		order.extend(sorted(bucket, key=lambda c: (c.get("length") or 1_000_000)))
	return [c for c in order if str(c.get("body", "")).strip()]

# ---- Run artifact helper ----
def write_run_artifact(
	root: Path,
	suite: str,
	run_started_ts: float,
	env: Dict[str, Any],
	prompts_hasher: Any,
	args_summary: Dict[str, Any],
	paths: Dict[str, str],
	aggregate: Any,
	metrics: Dict[str, Any],
	model: str,
	adapter_dir: str,
) -> None:
	import json as _json
	import hashlib as _hashlib
	import time as _time
	try:


	for bucket in (b1, b2, b3, b4):
		if bucket:
			best = sorted(bucket, key=lambda c: (c.get("length") or 1_000_000))[0]
			return str(best.get("body", ""))
	return ""

def rank_candidates(candidates: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
	"""Order candidates: critic+precheck first, then precheck, then critic, then rest; each by length prior."""
	b1 = [c for c in candidates if c.get("critic") and c.get("precheck")]
	b2 = [c for c in candidates if c.get("precheck") and c not in b1]
	b3 = [c for c in candidates if c.get("critic") and c not in b1 and c not in b2]
	b4 = [c for c in candidates if c not in b1 and c not in b2 and c not in b3]
	order: List[Dict[str, Any]] = []
	for bucket in (b1, b2, b3, b4):
		order.extend(sorted(bucket, key=lambda c: (c.get("length") or 1_000_000)))
	return [c for c in order if str(c.get("body", "")).strip()]

# ---- Run artifact helper ----
def write_run_artifact(
	root: Path,
	suite: str,
	run_started_ts: float,
	env: Dict[str, Any],
	prompts_hasher: Any,
	args_summary: Dict[str, Any],
	paths: Dict[str, str],
	aggregate: Any,
	metrics: Dict[str, Any],
	model: str,
	adapter_dir: str,
) -> None:
	import json as _json
	import hashlib as _hashlib
	import time as _time
	try:
		with trace_span("write_run_artifact", {"suite": suite}):
			runs_dir = Path(root / "data" / "bench" / "runs" / suite)
			runs_dir.mkdir(parents=True, exist_ok=True)
			ts_str = _time.strftime("%Y%m%dT%H%M%SZ", _time.gmtime(run_started_ts))
			art_dir = runs_dir / ts_str
			art_dir.mkdir(parents=True, exist_ok=True)
		try:
			_env_checksum = _hashlib.sha256(_json.dumps(env, sort_keys=True).encode("utf-8", errors="ignore")).hexdigest()
		except Exception:
			_env_checksum = None
		try:
			_prompt_pack_checksum = prompts_hasher.hexdigest()
		except Exception:
			_prompt_pack_checksum = None
		run_art = {
			"suite": suite,
			"started_ts": run_started_ts,
			"ended_ts": _time.time(),
			"model": model,
			"adapter_dir": adapter_dir,
			"args": args_summary,
			"paths": paths,
			"aggregate": (aggregate if isinstance(aggregate, dict) else {"raw": str(aggregate)}),
			"metrics": metrics,
			"env": env,
			"checksums": {"env": _env_checksum, "prompt_pack": _prompt_pack_checksum},
		}
		with (art_dir / "run.json").open("w", encoding="utf-8") as rf:
			rf.write(_json.dumps(run_art, ensure_ascii=False, indent=2))
	except Exception:
		pass

# ---- Run/Env helpers ----
def get_python_version() -> str:
	try:
		import sys as _sys  # type: ignore
		return str(getattr(_sys, "version", ""))
	except Exception:
		return ""

def env_fingerprint_for(mod_names: List[str]) -> Dict[str, Any]:
	_env: Dict[str, Any] = {}
	for mod_name in mod_names:
		try:
			mod = __import__(mod_name)
			_env[mod_name] = str(getattr(mod, "__version__", ""))
		except Exception:
			_env[mod_name] = None


			"suite": suite,
			"started_ts": run_started_ts,
			"ended_ts": _time.time(),
			"model": model,
			"adapter_dir": adapter_dir,
			"args": args_summary,
			"paths": paths,
			"aggregate": (aggregate if isinstance(aggregate, dict) else {"raw": str(aggregate)}),
			"metrics": metrics,
			"env": env,
			"checksums": {"env": _env_checksum, "prompt_pack": _prompt_pack_checksum},
		}
		with (art_dir / "run.json").open("w", encoding="utf-8") as rf:
			rf.write(_json.dumps(run_art, ensure_ascii=False, indent=2))
	except Exception:
		pass

# ---- Run/Env helpers ----
def get_python_version() -> str:
	try:
		import sys as _sys  # type: ignore
		return str(getattr(_sys, "version", ""))
	except Exception:
		return ""

def env_fingerprint_for(mod_names: List[str]) -> Dict[str, Any]:
	_env: Dict[str, Any] = {}
	for mod_name in mod_names:
		try:
			mod = __import__(mod_name)
			_env[mod_name] = str(getattr(mod, "__version__", ""))
		except Exception:
			_env[mod_name] = None
	return _env

def build_env_fingerprint(mod_names: List[str]) -> Dict[str, Any]:
	return {"python": get_python_version(), **env_fingerprint_for(mod_names)}

def is_primary_process() -> bool:
	try:
		import os as _os


			"metrics": metrics,
			"env": env,
			"checksums": {"env": _env_checksum, "prompt_pack": _prompt_pack_checksum},
		}
		with (art_dir / "run.json").open("w", encoding="utf-8") as rf:
			rf.write(_json.dumps(run_art, ensure_ascii=False, indent=2))
	except Exception:
		pass

# ---- Run/Env helpers ----
def get_python_version() -> str:
	try:
		import sys as _sys  # type: ignore
		return str(getattr(_sys, "version", ""))
	except Exception:
		return ""

def env_fingerprint_for(mod_names: List[str]) -> Dict[str, Any]:
	_env: Dict[str, Any] = {}
	for mod_name in mod_names:
		try:
			mod = __import__(mod_name)
			_env[mod_name] = str(getattr(mod, "__version__", ""))
		except Exception:
			_env[mod_name] = None
	return _env

def build_env_fingerprint(mod_names: List[str]) -> Dict[str, Any]:
	return {"python": get_python_version(), **env_fingerprint_for(mod_names)}

def is_primary_process() -> bool:
	try:
		import os as _os
		ws = int(_os.environ.get("WORLD_SIZE", "1") or 1)
		rk = int(_os.environ.get("RANK", "0") or 0)
		return (ws <= 1) or (rk == 0)
	except Exception:
		return True

def seed_everything(seed: Optional[int]) -> None:
	try:
		if seed is None:
			return


# ---- Run/Env helpers ----
def get_python_version() -> str:
	try:
		import sys as _sys  # type: ignore
		return str(getattr(_sys, "version", ""))
	except Exception:
		return ""

def env_fingerprint_for(mod_names: List[str]) -> Dict[str, Any]:
	_env: Dict[str, Any] = {}
	for mod_name in mod_names:
		try:
			mod = __import__(mod_name)
			_env[mod_name] = str(getattr(mod, "__version__", ""))
		except Exception:
			_env[mod_name] = None
	return _env

def build_env_fingerprint(mod_names: List[str]) -> Dict[str, Any]:
	return {"python": get_python_version(), **env_fingerprint_for(mod_names)}

def is_primary_process() -> bool:
	try:
		import os as _os
		ws = int(_os.environ.get("WORLD_SIZE", "1") or 1)
		rk = int(_os.environ.get("RANK", "0") or 0)
		return (ws <= 1) or (rk == 0)
	except Exception:
		return True

def seed_everything(seed: Optional[int]) -> None:
	try:
		if seed is None:
			return
		import random as _rnd  # type: ignore
		_rnd.seed(int(seed))
		try:
			import numpy as _np  # type: ignore


		return str(getattr(_sys, "version", ""))
	except Exception:
		return ""

def env_fingerprint_for(mod_names: List[str]) -> Dict[str, Any]:
	_env: Dict[str, Any] = {}
	for mod_name in mod_names:
		try:
			mod = __import__(mod_name)
			_env[mod_name] = str(getattr(mod, "__version__", ""))
		except Exception:
			_env[mod_name] = None
	return _env

def build_env_fingerprint(mod_names: List[str]) -> Dict[str, Any]:
	return {"python": get_python_version(), **env_fingerprint_for(mod_names)}

def is_primary_process() -> bool:
	try:
		import os as _os
		ws = int(_os.environ.get("WORLD_SIZE", "1") or 1)
		rk = int(_os.environ.get("RANK", "0") or 0)
		return (ws <= 1) or (rk == 0)
	except Exception:
		return True

def seed_everything(seed: Optional[int]) -> None:
	try:
		if seed is None:
			return
		import random as _rnd  # type: ignore
		_rnd.seed(int(seed))
		try:
			import numpy as _np  # type: ignore
			_np.random.seed(int(seed))
		except Exception:
			pass
		try:
			import torch as _torch  # type: ignore
			_torch.manual_seed(int(seed))
			if _torch.cuda.is_available():
				try:
					_torch.cuda.manual_seed_all(int(seed))
				except Exception:


			_env[mod_name] = str(getattr(mod, "__version__", ""))
		except Exception:
			_env[mod_name] = None
	return _env

def build_env_fingerprint(mod_names: List[str]) -> Dict[str, Any]:
	return {"python": get_python_version(), **env_fingerprint_for(mod_names)}

def is_primary_process() -> bool:
	try:
		import os as _os
		ws = int(_os.environ.get("WORLD_SIZE", "1") or 1)
		rk = int(_os.environ.get("RANK", "0") or 0)
		return (ws <= 1) or (rk == 0)
	except Exception:
		return True

def seed_everything(seed: Optional[int]) -> None:
	try:
		if seed is None:
			return
		import random as _rnd  # type: ignore
		_rnd.seed(int(seed))
		try:
			import numpy as _np  # type: ignore
			_np.random.seed(int(seed))
		except Exception:
			pass
		try:
			import torch as _torch  # type: ignore
			_torch.manual_seed(int(seed))
			if _torch.cuda.is_available():
				try:
					_torch.cuda.manual_seed_all(int(seed))
				except Exception:
					pass
		except Exception:
			pass
	except Exception:
		pass

def get_shard_tag(args: Any) -> str:
	try:
		import os as _os
		ns = max(1, _safe_int(getattr(args, "num_shards", _os.environ.get("WORLD_SIZE", 1) or 1), 1))
		sid = max(0, _safe_int(getattr(args, "shard_id", _os.environ.get("RANK", 0) or 0), 0))
		return (f".shard{sid}-of-{ns}" if ns > 1 else "")
	except Exception:
		return ""

def init_trace_paths(root: Path, suite: str, run_started_ts: float) -> Tuple[Path, Path, str]:
	"""Create a timestamped trace directory for a suite run and return (traces_dir, trace_path, ts_str)."""
	import time as _time
	ts_str = _time.strftime("%Y%m%dT%H%M%SZ", _time.gmtime(run_started_ts))
	traces_dir = Path(root / "data" / "traces" / "bench_runs" / suite / ts_str)
	traces_dir.mkdir(parents=True, exist_ok=True)
	trace_path = traces_dir / "run.jsonl"
	return traces_dir, trace_path, ts_str


		_rnd.seed(int(seed))
		try:
			import numpy as _np  # type: ignore
			_np.random.seed(int(seed))
		except Exception:
			pass
		try:
			import torch as _torch  # type: ignore
			_torch.manual_seed(int(seed))
			if _torch.cuda.is_available():
				try:
					_torch.cuda.manual_seed_all(int(seed))
				except Exception:
					pass
		except Exception:
			pass
	except Exception:
		pass

def get_shard_tag(args: Any) -> str:
	try:
		import os as _os
		ns = max(1, _safe_int(getattr(args, "num_shards", _os.environ.get("WORLD_SIZE", 1) or 1), 1))
		sid = max(0, _safe_int(getattr(args, "shard_id", _os.environ.get("RANK", 0) or 0), 0))
		return (f".shard{sid}-of-{ns}" if ns > 1 else "")
	except Exception:
		return ""

def init_trace_paths(root: Path, suite: str, run_started_ts: float) -> Tuple[Path, Path, str]:
	"""Create a timestamped trace directory for a suite run and return (traces_dir, trace_path, ts_str)."""
	import time as _time
	ts_str = _time.strftime("%Y%m%dT%H%M%SZ", _time.gmtime(run_started_ts))
	traces_dir = Path(root / "data" / "traces" / "bench_runs" / suite / ts_str)
	traces_dir.mkdir(parents=True, exist_ok=True)
	trace_path = traces_dir / "run.jsonl"
	return traces_dir, trace_path, ts_str

def init_suite_temp_paths(root: Path, suite: str, shard_tag: str) -> Tuple[Path, Path, Path]:
	"""Return (samples_path, verbose_path, errors_path) under bench/tmp for the suite and shard."""
	samples_path = Path(root / "data" / "bench" / "tmp" / (f"{suite}_samples" + shard_tag + ".jsonl"))
	verbose_path = Path(root / "data" / "bench" / "tmp" / (f"{suite}_samples.with_prompts" + shard_tag + ".jsonl"))
	errors_path = Path(root / "data" / "bench" / "tmp" / (f"{suite}_errors" + shard_tag + ".jsonl"))
	samples_path.parent.mkdir(parents=True, exist_ok=True)
	try:
		errors_path.write_text("")


				try:
					_torch.cuda.manual_seed_all(int(seed))
				except Exception:
					pass
		except Exception:
			pass
	except Exception:
		pass

def get_shard_tag(args: Any) -> str:
	try:
		import os as _os
		ns = max(1, _safe_int(getattr(args, "num_shards", _os.environ.get("WORLD_SIZE", 1) or 1), 1))
		sid = max(0, _safe_int(getattr(args, "shard_id", _os.environ.get("RANK", 0) or 0), 0))
		return (f".shard{sid}-of-{ns}" if ns > 1 else "")
	except Exception:
		return ""

def init_trace_paths(root: Path, suite: str, run_started_ts: float) -> Tuple[Path, Path, str]:
	"""Create a timestamped trace directory for a suite run and return (traces_dir, trace_path, ts_str)."""
	import time as _time
	ts_str = _time.strftime("%Y%m%dT%H%M%SZ", _time.gmtime(run_started_ts))
	traces_dir = Path(root / "data" / "traces" / "bench_runs" / suite / ts_str)
	traces_dir.mkdir(parents=True, exist_ok=True)
	trace_path = traces_dir / "run.jsonl"
	return traces_dir, trace_path, ts_str

def init_suite_temp_paths(root: Path, suite: str, shard_tag: str) -> Tuple[Path, Path, Path]:
	"""Return (samples_path, verbose_path, errors_path) under bench/tmp for the suite and shard."""
	samples_path = Path(root / "data" / "bench" / "tmp" / (f"{suite}_samples" + shard_tag + ".jsonl"))
	verbose_path = Path(root / "data" / "bench" / "tmp" / (f"{suite}_samples.with_prompts" + shard_tag + ".jsonl"))
	errors_path = Path(root / "data" / "bench" / "tmp" / (f"{suite}_errors" + shard_tag + ".jsonl"))
	samples_path.parent.mkdir(parents=True, exist_ok=True)
	try:
		errors_path.write_text("")
	except Exception:
		pass
	return samples_path, verbose_path, errors_path

def read_pass_cache(root: Path, suite: str) -> Tuple[Dict[str, Dict[str, str]], Path]:
	"""Read per-suite pass cache JSONL into a dict; return (cache, path)."""
	path = Path(root / "data" / "bench" / "cache" / "results" / f"{suite}.jsonl")
	cache: Dict[str, Dict[str, str]] = {}
	try:


def get_shard_tag(args: Any) -> str:
	try:
		import os as _os
		ns = max(1, _safe_int(getattr(args, "num_shards", _os.environ.get("WORLD_SIZE", 1) or 1), 1))
		sid = max(0, _safe_int(getattr(args, "shard_id", _os.environ.get("RANK", 0) or 0), 0))
		return (f".shard{sid}-of-{ns}" if ns > 1 else "")
	except Exception:
		return ""

def init_trace_paths(root: Path, suite: str, run_started_ts: float) -> Tuple[Path, Path, str]:
	"""Create a timestamped trace directory for a suite run and return (traces_dir, trace_path, ts_str)."""
	import time as _time
	ts_str = _time.strftime("%Y%m%dT%H%M%SZ", _time.gmtime(run_started_ts))
	traces_dir = Path(root / "data" / "traces" / "bench_runs" / suite / ts_str)
	traces_dir.mkdir(parents=True, exist_ok=True)
	trace_path = traces_dir / "run.jsonl"
	return traces_dir, trace_path, ts_str

def init_suite_temp_paths(root: Path, suite: str, shard_tag: str) -> Tuple[Path, Path, Path]:
	"""Return (samples_path, verbose_path, errors_path) under bench/tmp for the suite and shard."""
	samples_path = Path(root / "data" / "bench" / "tmp" / (f"{suite}_samples" + shard_tag + ".jsonl"))
	verbose_path = Path(root / "data" / "bench" / "tmp" / (f"{suite}_samples.with_prompts" + shard_tag + ".jsonl"))
	errors_path = Path(root / "data" / "bench" / "tmp" / (f"{suite}_errors" + shard_tag + ".jsonl"))
	samples_path.parent.mkdir(parents=True, exist_ok=True)
	try:
		errors_path.write_text("")
	except Exception:
		pass
	return samples_path, verbose_path, errors_path

def read_pass_cache(root: Path, suite: str) -> Tuple[Dict[str, Dict[str, str]], Path]:
	"""Read per-suite pass cache JSONL into a dict; return (cache, path)."""
	path = Path(root / "data" / "bench" / "cache" / "results" / f"{suite}.jsonl")
	cache: Dict[str, Dict[str, str]] = {}
	try:
		if path.exists():
			for obj in read_jsonl(path):
				_tid = str(obj.get("task_id", ""))
				_comp = str(obj.get("completion", ""))
				if _tid and _comp:
					cache[_tid] = {"completion": _comp, "prompt": str(obj.get("prompt", ""))}
	except Exception:
		pass
	try:
		path.parent.mkdir(parents=True, exist_ok=True)
	except Exception:
		pass
	return cache, path


	ts_str = _time.strftime("%Y%m%dT%H%M%SZ", _time.gmtime(run_started_ts))
	traces_dir = Path(root / "data" / "traces" / "bench_runs" / suite / ts_str)
	traces_dir.mkdir(parents=True, exist_ok=True)
	trace_path = traces_dir / "run.jsonl"
	return traces_dir, trace_path, ts_str

def init_suite_temp_paths(root: Path, suite: str, shard_tag: str) -> Tuple[Path, Path, Path]:
	"""Return (samples_path, verbose_path, errors_path) under bench/tmp for the suite and shard."""
	samples_path = Path(root / "data" / "bench" / "tmp" / (f"{suite}_samples" + shard_tag + ".jsonl"))
	verbose_path = Path(root / "data" / "bench" / "tmp" / (f"{suite}_samples.with_prompts" + shard_tag + ".jsonl"))
	errors_path = Path(root / "data" / "bench" / "tmp" / (f"{suite}_errors" + shard_tag + ".jsonl"))
	samples_path.parent.mkdir(parents=True, exist_ok=True)
	try:
		errors_path.write_text("")
	except Exception:
		pass
	return samples_path, verbose_path, errors_path

def read_pass_cache(root: Path, suite: str) -> Tuple[Dict[str, Dict[str, str]], Path]:
	"""Read per-suite pass cache JSONL into a dict; return (cache, path)."""
	path = Path(root / "data" / "bench" / "cache" / "results" / f"{suite}.jsonl")
	cache: Dict[str, Dict[str, str]] = {}
	try:
		if path.exists():
			for obj in read_jsonl(path):
				_tid = str(obj.get("task_id", ""))
				_comp = str(obj.get("completion", ""))
				if _tid and _comp:
					cache[_tid] = {"completion": _comp, "prompt": str(obj.get("prompt", ""))}
	except Exception:
		pass
	try:
		path.parent.mkdir(parents=True, exist_ok=True)
	except Exception:
		pass
	return cache, path

def update_pass_cache(pass_cache_path: Path, prev: Dict[str, Dict[str, str]], latest: Dict[str, Dict[str, str]]) -> None:
	merged: Dict[str, Dict[str, str]] = dict(prev)
	merged.update(latest)
	try:
		with pass_cache_path.open("w", encoding="utf-8") as pcw:
			for tid, rec in merged.items():
				pcw.write(__import__("json").dumps({"task_id": tid, "completion": rec.get("completion", ""), "prompt": rec.get("prompt", "")}, ensure_ascii=False) + "\n")
	except Exception:
		pass

def build_generation_params(args: Any) -> Tuple[Dict[str, Any], bool, int, int, int, float]:
	"""Return (params, grammar_constrained, n_samples, n_candidates, retries, retry_backoff)."""
	params: Dict[str, Any] = {
		"max_new_tokens": _safe_int(getattr(args, "max_new_tokens", 256) or 256, 256),
		"temperature": _safe_float(getattr(args, "temperature", 0.2) or 0.2, 0.2),
	}
	if getattr(args, "top_p", None) is not None:


def read_pass_cache(root: Path, suite: str) -> Tuple[Dict[str, Dict[str, str]], Path]:
	"""Read per-suite pass cache JSONL into a dict; return (cache, path)."""
	path = Path(root / "data" / "bench" / "cache" / "results" / f"{suite}.jsonl")
	cache: Dict[str, Dict[str, str]] = {}
	try:
		if path.exists():
			for obj in read_jsonl(path):
				_tid = str(obj.get("task_id", ""))
				_comp = str(obj.get("completion", ""))
				if _tid and _comp:
					cache[_tid] = {"completion": _comp, "prompt": str(obj.get("prompt", ""))}
	except Exception:
		pass
	try:
		path.parent.mkdir(parents=True, exist_ok=True)
	except Exception:
		pass
	return cache, path

def update_pass_cache(pass_cache_path: Path, prev: Dict[str, Dict[str, str]], latest: Dict[str, Dict[str, str]]) -> None:
	merged: Dict[str, Dict[str, str]] = dict(prev)
	merged.update(latest)
	try:
		with pass_cache_path.open("w", encoding="utf-8") as pcw:
			for tid, rec in merged.items():
				pcw.write(__import__("json").dumps({"task_id": tid, "completion": rec.get("completion", ""), "prompt": rec.get("prompt", "")}, ensure_ascii=False) + "\n")
	except Exception:
		pass

def build_generation_params(args: Any) -> Tuple[Dict[str, Any], bool, int, int, int, float]:
	"""Return (params, grammar_constrained, n_samples, n_candidates, retries, retry_backoff)."""
	params: Dict[str, Any] = {
		"max_new_tokens": _safe_int(getattr(args, "max_new_tokens", 256) or 256, 256),
		"temperature": _safe_float(getattr(args, "temperature", 0.2) or 0.2, 0.2),
	}
	if getattr(args, "top_p", None) is not None:
		params["top_p"] = _safe_float(getattr(args, "top_p"), 0.95)
	if getattr(args, "top_k", None) is not None:
		params["top_k"] = _safe_int(getattr(args, "top_k"), 50)
	grammar_constrained = bool(getattr(args, "grammar_constrained", True))
	n_samples = max(1, _safe_int(getattr(args, "n_samples", 1) or 1, 1))
	n_candidates = max(1, _safe_int(getattr(args, "n_candidates", 3) or 3, 3))
	retries = max(0, _safe_int(getattr(args, "retries", 0) or 0, 0))
	retry_backoff = _safe_float(getattr(args, "retry_backoff", 0.75) or 0.75, 0.75)
	return params, grammar_constrained, n_samples, n_candidates, retries, retry_backoff


	except Exception:
		pass
	try:
		path.parent.mkdir(parents=True, exist_ok=True)
	except Exception:
		pass
	return cache, path

def update_pass_cache(pass_cache_path: Path, prev: Dict[str, Dict[str, str]], latest: Dict[str, Dict[str, str]]) -> None:
	merged: Dict[str, Dict[str, str]] = dict(prev)
	merged.update(latest)
	try:
		with pass_cache_path.open("w", encoding="utf-8") as pcw:
			for tid, rec in merged.items():
				pcw.write(__import__("json").dumps({"task_id": tid, "completion": rec.get("completion", ""), "prompt": rec.get("prompt", "")}, ensure_ascii=False) + "\n")
	except Exception:
		pass

def build_generation_params(args: Any) -> Tuple[Dict[str, Any], bool, int, int, int, float]:
	"""Return (params, grammar_constrained, n_samples, n_candidates, retries, retry_backoff)."""
	params: Dict[str, Any] = {
		"max_new_tokens": _safe_int(getattr(args, "max_new_tokens", 256) or 256, 256),
		"temperature": _safe_float(getattr(args, "temperature", 0.2) or 0.2, 0.2),
	}
	if getattr(args, "top_p", None) is not None:
		params["top_p"] = _safe_float(getattr(args, "top_p"), 0.95)
	if getattr(args, "top_k", None) is not None:
		params["top_k"] = _safe_int(getattr(args, "top_k"), 50)
	grammar_constrained = bool(getattr(args, "grammar_constrained", True))
	n_samples = max(1, _safe_int(getattr(args, "n_samples", 1) or 1, 1))
	n_candidates = max(1, _safe_int(getattr(args, "n_candidates", 3) or 3, 3))
	retries = max(0, _safe_int(getattr(args, "retries", 0) or 0, 0))
	retry_backoff = _safe_float(getattr(args, "retry_backoff", 0.75) or 0.75, 0.75)
	return params, grammar_constrained, n_samples, n_candidates, retries, retry_backoff

def summarize_telemetry(task_ids: List[str], results_by_task: Dict[str, bool]) -> Tuple[Dict[str, Any], Optional[float]]:
	try:
		pass_count = sum(1 for _tid, ok in results_by_task.items() if ok)
		pass1_rate = pass_count / max(1, len(task_ids))
		telemetry = {"n_tasks": len(task_ids), "pass1": pass1_rate}
		return telemetry, pass1_rate
	except Exception:
		return {}, None

def build_codebody_args_summary(args: Any) -> Dict[str, Any]:
    """Build a standard args summary dict for code-body suites."""
    return {
        "max_new_tokens": _safe_int(getattr(args, "max_new_tokens", 0) or 0, 0),
        "temperature": _safe_float(getattr(args, "temperature", 0.0) or 0.0, 0.0),
        "top_p": (_safe_float(getattr(args, "top_p"), 0.95) if getattr(args, "top_p", None) is not None else None),
        "top_k": (_safe_int(getattr(args, "top_k"), 50) if getattr(args, "top_k", None) is not None else None),
        "n_samples": _safe_int(getattr(args, "n_samples", 1) or 1, 1),


def build_generation_params(args: Any) -> Tuple[Dict[str, Any], bool, int, int, int, float]:
	"""Return (params, grammar_constrained, n_samples, n_candidates, retries, retry_backoff)."""
	params: Dict[str, Any] = {
		"max_new_tokens": _safe_int(getattr(args, "max_new_tokens", 256) or 256, 256),
		"temperature": _safe_float(getattr(args, "temperature", 0.2) or 0.2, 0.2),
	}
	if getattr(args, "top_p", None) is not None:
		params["top_p"] = _safe_float(getattr(args, "top_p"), 0.95)
	if getattr(args, "top_k", None) is not None:
		params["top_k"] = _safe_int(getattr(args, "top_k"), 50)
	grammar_constrained = bool(getattr(args, "grammar_constrained", True))
	n_samples = max(1, _safe_int(getattr(args, "n_samples", 1) or 1, 1))
	n_candidates = max(1, _safe_int(getattr(args, "n_candidates", 3) or 3, 3))
	retries = max(0, _safe_int(getattr(args, "retries", 0) or 0, 0))
	retry_backoff = _safe_float(getattr(args, "retry_backoff", 0.75) or 0.75, 0.75)
	return params, grammar_constrained, n_samples, n_candidates, retries, retry_backoff

def summarize_telemetry(task_ids: List[str], results_by_task: Dict[str, bool]) -> Tuple[Dict[str, Any], Optional[float]]:
	try:
		pass_count = sum(1 for _tid, ok in results_by_task.items() if ok)
		pass1_rate = pass_count / max(1, len(task_ids))
		telemetry = {"n_tasks": len(task_ids), "pass1": pass1_rate}
		return telemetry, pass1_rate
	except Exception:
		return {}, None

def build_codebody_args_summary(args: Any) -> Dict[str, Any]:
    """Build a standard args summary dict for code-body suites."""
    return {
        "max_new_tokens": _safe_int(getattr(args, "max_new_tokens", 0) or 0, 0),
        "temperature": _safe_float(getattr(args, "temperature", 0.0) or 0.0, 0.0),
        "top_p": (_safe_float(getattr(args, "top_p"), 0.95) if getattr(args, "top_p", None) is not None else None),
        "top_k": (_safe_int(getattr(args, "top_k"), 50) if getattr(args, "top_k", None) is not None else None),
        "n_samples": _safe_int(getattr(args, "n_samples", 1) or 1, 1),
        "k_list": [int(x) for x in str(getattr(args, "k", "1") or "1").split(",") if str(x).strip()],
        "limit": _safe_int(getattr(args, "limit", 0) or 0, 0),
        "timeout": _safe_float(getattr(args, "timeout", 0.0) or 0.0, 0.0),
        "seed": (_safe_int(getattr(args, "seed"), 0) if getattr(args, "seed", None) is not None else None),
        "repair": bool(getattr(args, "repair", False)),
        "repair_samples": _safe_int(getattr(args, "repair_samples", 0) or 0, 0),
        "n_candidates": _safe_int(getattr(args, "n_candidates", 1) or 1, 1),
        "flake_retries": _safe_int(getattr(args, "flake_retries", 0) or 0, 0),
        "grammar_constrained": bool(getattr(args, "grammar_constrained", False)),
    }


	if getattr(args, "top_k", None) is not None:
		params["top_k"] = _safe_int(getattr(args, "top_k"), 50)
	grammar_constrained = bool(getattr(args, "grammar_constrained", True))
	n_samples = max(1, _safe_int(getattr(args, "n_samples", 1) or 1, 1))
	n_candidates = max(1, _safe_int(getattr(args, "n_candidates", 3) or 3, 3))
	retries = max(0, _safe_int(getattr(args, "retries", 0) or 0, 0))
	retry_backoff = _safe_float(getattr(args, "retry_backoff", 0.75) or 0.75, 0.75)
	return params, grammar_constrained, n_samples, n_candidates, retries, retry_backoff

def summarize_telemetry(task_ids: List[str], results_by_task: Dict[str, bool]) -> Tuple[Dict[str, Any], Optional[float]]:
	try:
		pass_count = sum(1 for _tid, ok in results_by_task.items() if ok)
		pass1_rate = pass_count / max(1, len(task_ids))
		telemetry = {"n_tasks": len(task_ids), "pass1": pass1_rate}
		return telemetry, pass1_rate
	except Exception:
		return {}, None

def build_codebody_args_summary(args: Any) -> Dict[str, Any]:
    """Build a standard args summary dict for code-body suites."""
    return {
        "max_new_tokens": _safe_int(getattr(args, "max_new_tokens", 0) or 0, 0),
        "temperature": _safe_float(getattr(args, "temperature", 0.0) or 0.0, 0.0),
        "top_p": (_safe_float(getattr(args, "top_p"), 0.95) if getattr(args, "top_p", None) is not None else None),
        "top_k": (_safe_int(getattr(args, "top_k"), 50) if getattr(args, "top_k", None) is not None else None),
        "n_samples": _safe_int(getattr(args, "n_samples", 1) or 1, 1),
        "k_list": [int(x) for x in str(getattr(args, "k", "1") or "1").split(",") if str(x).strip()],
        "limit": _safe_int(getattr(args, "limit", 0) or 0, 0),
        "timeout": _safe_float(getattr(args, "timeout", 0.0) or 0.0, 0.0),
        "seed": (_safe_int(getattr(args, "seed"), 0) if getattr(args, "seed", None) is not None else None),
        "repair": bool(getattr(args, "repair", False)),
        "repair_samples": _safe_int(getattr(args, "repair_samples", 0) or 0, 0),
        "n_candidates": _safe_int(getattr(args, "n_candidates", 1) or 1, 1),
        "flake_retries": _safe_int(getattr(args, "flake_retries", 0) or 0, 0),
        "grammar_constrained": bool(getattr(args, "grammar_constrained", False)),
    }

def flake_detection(samples_path: Path, args: Any, default_workers: int, task_ids: List[str], results_by_task: Dict[str, bool], suite: str, root: Path) -> None:
	try:
		flake_k = max(0, _safe_int(getattr(args, "flake_retries", 0) or 0, 0))
		if flake_k <= 0:
			return
		per_task_outcomes: Dict[str, set] = {tid: {bool(results_by_task.get(tid, False))} for tid in task_ids}
		from human_eval.evaluation import evaluate_functional_correctness as _eval  # type: ignore
		for _ in range(flake_k):
			try:
				_ = _eval(
					sample_file=str(samples_path),
					k=[1],
					n_workers=max(1, _safe_int(getattr(args, "max_workers", default_workers) or default_workers, default_workers)),
					timeout=_safe_float(getattr(args, "timeout", 15) or 15, 15.0),
					ignore_incomplete=True,
				)
				# Refresh per-task flags from the results file
				tmp_results_path = Path(str(samples_path) + "_results.jsonl")


def build_codebody_args_summary(args: Any) -> Dict[str, Any]:
    """Build a standard args summary dict for code-body suites."""
    return {
        "max_new_tokens": _safe_int(getattr(args, "max_new_tokens", 0) or 0, 0),
        "temperature": _safe_float(getattr(args, "temperature", 0.0) or 0.0, 0.0),
        "top_p": (_safe_float(getattr(args, "top_p"), 0.95) if getattr(args, "top_p", None) is not None else None),
        "top_k": (_safe_int(getattr(args, "top_k"), 50) if getattr(args, "top_k", None) is not None else None),
        "n_samples": _safe_int(getattr(args, "n_samples", 1) or 1, 1),
        "k_list": [int(x) for x in str(getattr(args, "k", "1") or "1").split(",") if str(x).strip()],
        "limit": _safe_int(getattr(args, "limit", 0) or 0, 0),
        "timeout": _safe_float(getattr(args, "timeout", 0.0) or 0.0, 0.0),
        "seed": (_safe_int(getattr(args, "seed"), 0) if getattr(args, "seed", None) is not None else None),
        "repair": bool(getattr(args, "repair", False)),
        "repair_samples": _safe_int(getattr(args, "repair_samples", 0) or 0, 0),
        "n_candidates": _safe_int(getattr(args, "n_candidates", 1) or 1, 1),
        "flake_retries": _safe_int(getattr(args, "flake_retries", 0) or 0, 0),
        "grammar_constrained": bool(getattr(args, "grammar_constrained", False)),
    }

def flake_detection(samples_path: Path, args: Any, default_workers: int, task_ids: List[str], results_by_task: Dict[str, bool], suite: str, root: Path) -> None:
	try:
		flake_k = max(0, _safe_int(getattr(args, "flake_retries", 0) or 0, 0))
		if flake_k <= 0:
			return
		per_task_outcomes: Dict[str, set] = {tid: {bool(results_by_task.get(tid, False))} for tid in task_ids}
		from human_eval.evaluation import evaluate_functional_correctness as _eval  # type: ignore
		for _ in range(flake_k):
			try:
				_ = _eval(
					sample_file=str(samples_path),
					k=[1],
					n_workers=max(1, _safe_int(getattr(args, "max_workers", default_workers) or default_workers, default_workers)),
					timeout=_safe_float(getattr(args, "timeout", 15) or 15, 15.0),
					ignore_incomplete=True,
				)
				# Refresh per-task flags from the results file
				tmp_results_path = Path(str(samples_path) + "_results.jsonl")
				for row in read_jsonl(tmp_results_path):
					_tid = str(row.get("task_id", ""))
					_passed = bool(row.get("passed", False))
					if _tid:
						per_task_outcomes.setdefault(_tid, set()).add(_passed)
			except Exception:
				continue
		flaky = sorted([tid for tid, outcomes in per_task_outcomes.items() if len(outcomes) > 1])
		if flaky:
			flaky_path = Path(root / "data" / "bench" / "tmp" / f"{suite}_flaky.json")
			try:
				flaky_path.write_text(__import__("json").dumps({"tasks": flaky}, ensure_ascii=False, indent=2), encoding="utf-8")
			except Exception:
				pass
	except Exception:
		pass

def online_update_and_rerun_failures(
	args: Any,
	root: Path,
	suite: str,
	default_workers: int,
	samples_path: Path,
	verbose_path: Path,
	results_path: Path,
	k_list: List[int],
	regen_failed_fn: Any,
	evalplus_evaluate: Optional[Any],
	evaluate_functional_correctness: Any,
) -> Optional[Any]:
	"""Perform optional online updates and targeted rerun of failures. Returns new aggregate result or None."""
	from subprocess import call as _call  # type: ignore
	try:
		wm_pack = Path(getattr(args, "wm_model", root / "models" / "wm_mlp" / "wm_mlp.joblib"))
		# Build WM dataset from verified traces if present


				# Refresh per-task flags from the results file
				tmp_results_path = Path(str(samples_path) + "_results.jsonl")
				for row in read_jsonl(tmp_results_path):
					_tid = str(row.get("task_id", ""))
					_passed = bool(row.get("passed", False))
					if _tid:
						per_task_outcomes.setdefault(_tid, set()).add(_passed)
			except Exception:
				continue
		flaky = sorted([tid for tid, outcomes in per_task_outcomes.items() if len(outcomes) > 1])
		if flaky:
			flaky_path = Path(root / "data" / "bench" / "tmp" / f"{suite}_flaky.json")
			try:
				flaky_path.write_text(__import__("json").dumps({"tasks": flaky}, ensure_ascii=False, indent=2), encoding="utf-8")
			except Exception:
				pass
	except Exception:
		pass

def online_update_and_rerun_failures(
	args: Any,
	root: Path,
	suite: str,
	default_workers: int,
	samples_path: Path,
	verbose_path: Path,
	results_path: Path,
	k_list: List[int],
	regen_failed_fn: Any,
	evalplus_evaluate: Optional[Any],
	evaluate_functional_correctness: Any,
) -> Optional[Any]:
	"""Perform optional online updates and targeted rerun of failures. Returns new aggregate result or None."""
	from subprocess import call as _call  # type: ignore
	try:
		wm_pack = Path(getattr(args, "wm_model", root / "models" / "wm_mlp" / "wm_mlp.joblib"))
		# Build WM dataset from verified traces if present
		_call(["python3", str(root / "scripts" / "build" / "build_wm_ds.py"), str(root / "data" / "skills" / "wm_ds.jsonl")])
		# Train/refresh WM pack (favor incremental if pack exists)
		_call(["python3", str(root / "scripts" / "build" / "build_wm_online.py"), "--out", str(wm_pack), "--incremental"])
		# Refresh verifier calibration dataset and run calibration training
		_call(["python3", str(root / "scripts" / "build" / "build_verifier_ds.py"), str(root / "data" / "skills" / "verifier_ds.jsonl")])
		_call(["python3", str(root / "scripts" / "train" / "train_verifier_calib.py"), "--out", str(root / "models" / "verifier_calib")])
		# Re-evaluate only failed tasks with updated WM-assisted selection
		failed_ids: List[str] = []
		for row in read_jsonl(results_path):
			_tid = str(row.get("task_id", ""))
			if _tid and (not bool(row.get("passed", False))):
				failed_ids.append(_tid)
		if not failed_ids:
			return None
		from concurrent.futures import ThreadPoolExecutor as _TPE, as_completed as _as_completed  # type: ignore
		with _TPE(max_workers=max(1, _safe_int(getattr(args, "max_workers", default_workers) or default_workers, default_workers))) as ex:
			futs = {ex.submit(regen_failed_fn, tid): tid for tid in failed_ids}
			for fut in _as_completed(futs):
				_tid = futs[fut]
				try:
					rows = fut.result()
					# Append rows to sidecar and chosen samples
					with samples_path.open("a", encoding="utf-8") as sf, verbose_path.open("a", encoding="utf-8") as vf:
						for row in rows:
							try:
								vf.write(__import__("json").dumps(row) + "\n")
								if str(row.get("role", "chosen")) == "chosen":
									mini = {"task_id": row.get("task_id"), "completion": row.get("completion")}
									sf.write(__import__("json").dumps(mini) + "\n")
							except Exception:
								pass
				except Exception:
					pass
		# Re-run evaluation for the updated samples
		try:
			use_evalplus = bool(getattr(args, "use_evalplus", False)) and (evalplus_evaluate is not None)
			if use_evalplus:
				return evalplus_evaluate(sample_file=str(samples_path), k=k_list, n_workers=max(1, _safe_int(getattr(args, "max_workers", default_workers) or default_workers, default_workers)), timeout=_safe_float(getattr(args, "timeout", 15) or 15, 15.0))  # type: ignore
			else:
				return evaluate_functional_correctness(
					sample_file=str(samples_path),
					k=k_list,
					n_workers=max(1, _safe_int(getattr(args, "max_workers", default_workers) or default_workers, default_workers)),
					timeout=_safe_float(getattr(args, "timeout", 15) or 15, 15.0),
					ignore_incomplete=True,
				)
		except Exception:
			return None
	except Exception:
		return None

def emit_training_datasets(root: Path, suite: str, verbose_path: Path, results_path: Path) -> None:
	"""Emit coder_ds, wm_ds, verifier_ds based on verbose and results."""
	try:
		coder_ds = Path(root / "data" / "traces" / "coder_ds.jsonl")
		wm_ds = Path(root / "data" / "skills" / "wm_ds.jsonl")
		ver_ds = Path(root / "data" / "skills" / "verifier_ds.jsonl")
		coder_ds.parent.mkdir(parents=True, exist_ok=True)
		wm_ds.parent.mkdir(parents=True, exist_ok=True)
		ver_ds.parent.mkdir(parents=True, exist_ok=True)
		# Build quick lookup: (task_id, completion) -> passed
		passed_map: Dict[str, set] = {}
		for row in read_jsonl(results_path):
			_tid = str(row.get("task_id", ""))
			if not _tid:
				continue
			passed_map.setdefault(_tid, set())
			if bool(row.get("passed", False)):
				passed_map[_tid].add(str(row.get("completion", "")))