def choose_letter(text: str, valid: str = "ABCD") -> Optional[str]:
	t = text.strip().upper()
	for ch in valid:
		if ch in t:
			return ch
	# fallbacks
	if "YES" in t and "NO" not in t:
		return "YES"
	if "NO" in t and "YES" not in t:
		return "NO"
	return None

def normalize_answer(s: str) -> str:
	import re
	import string
	def remove_articles(text: str) -> str:
		return re.sub(r"\b(a|an|the)\b", " ", text)
	def white_space_fix(text: str) -> str:
		return " ".join(text.split())
	def remove_punc(text: str) -> str:
		return "".join(ch for ch in text if ch not in set(string.punctuation))
	def lower(text: str) -> str:
		return text.lower()
	return white_space_fix(remove_articles(remove_punc(lower(s))))

def choose_boolean(text: str) -> Optional[bool]:
	t = text.strip().lower()
	if any(x in t for x in [" yes", "\nyes"]) or t.startswith("yes"):
		return True
	if any(x in t for x in [" no", "\nno"]) or t.startswith("no"):
		return False
	if "true" in t and "false" not in t:
		return True
	if "false" in t and "true" not in t:
		return False
	return None

def run_quac(backend: GenerationBackend, prompt_dir: Path, max_samples: int, decode_cfg: Dict[str, Any]) -> Dict[str, Any]:
	try:


lower

	for ch in valid:
		if ch in t:
			return ch
	# fallbacks
	if "YES" in t and "NO" not in t:
		return "YES"
	if "NO" in t and "YES" not in t:
		return "NO"
	return None

def normalize_answer(s: str) -> str:
	import re
	import string
	def remove_articles(text: str) -> str:
		return re.sub(r"\b(a|an|the)\b", " ", text)
	def white_space_fix(text: str) -> str:
		return " ".join(text.split())
	def remove_punc(text: str) -> str:
		return "".join(ch for ch in text if ch not in set(string.punctuation))
	def lower(text: str) -> str:
		return text.lower()
	return white_space_fix(remove_articles(remove_punc(lower(s))))

def choose_boolean(text: str) -> Optional[bool]:
	t = text.strip().lower()
	if any(x in t for x in [" yes", "\nyes"]) or t.startswith("yes"):
		return True
	if any(x in t for x in [" no", "\nno"]) or t.startswith("no"):
		return False
	if "true" in t and "false" not in t:
		return True
	if "false" in t and "true" not in t:
		return False
	return None

def run_quac(backend: GenerationBackend, prompt_dir: Path, max_samples: int, decode_cfg: Dict[str, Any]) -> Dict[str, Any]:
	try:
		from datasets import load_dataset  # type: ignore
	except Exception as e:


		stem = str(it.get("problem", it.get("question", "")))
		options = [str(x) for x in (it.get("options") or [])]
		answer = str(it.get("answer", ""))
		# Build prompt
		choices = "\n".join([f"{chr(ord('A')+j)}) {opt}" for j, opt in enumerate(options)])
		template = load_template(prompt_dir, "agieval", "Question: {q}\n\nChoices:\n{choices}\n\nAnswer with the letter.")
		prompt = template.format(q=stem, choices=choices)
		out = backend.generate(prompt, max_tokens=int(decode_cfg.get("max_tokens", 32)), temperature=float(decode_cfg.get("temperature", 0.0)), top_p=float(decode_cfg.get("top_p", 0.95)))
		pick = choose_letter(out, valid="ABCDE") or ""
		if pick and pick[0] == answer.strip().upper()[:1]:
			correct += 1
	return {"status": "ok", "timing_sec": round(time.time() - t0, 3), "num_samples": n, "correct": correct, "acc": correct / n if n else 0.0}

def run_crass(backend: GenerationBackend, prompt_dir: Path, max_samples: int, decode_cfg: Dict[str, Any]) -> Dict[str, Any]:
	# Placeholder: CRASS requires specialized counterfactual reasoning dataset and scorer
	return {"status": "skipped", "error": "crass_not_implemented", "timing_sec": 0.0, "num_samples": 0}

def f1_score(prediction: str, ground_truths: List[str]) -> float:
	def f1(p: str, g: str) -> float:
		p_tokens = normalize_answer(p).split()
		g_tokens = normalize_answer(g).split()
		if len(p_tokens) == 0 or len(g_tokens) == 0:
			return 0.0
		common = {}
		for t in p_tokens:
			common[t] = common.get(t, 0) + 1
		num_same = 0
		for t in g_tokens:
			if common.get(t, 0) > 0:
				num_same += 1
				common[t] -= 1
		if num_same == 0:
			return 0.0
		precision = num_same / len(p_tokens)
		recall = num_same / len(g_tokens)
		return 2 * precision * recall / (precision + recall)
	if not ground_truths:
		return 0.0
	return max(f1(prediction, gt) for gt in ground_truths)

def run_hellaswag(backend: GenerationBackend, prompt_dir: Path, max_samples: int, decode_cfg: Dict[str, Any]) -> Dict[str, Any]:
	try:
		from datasets import load_dataset  # type: ignore
	except Exception as e:
		return {"status": "skipped", "error": f"datasets_missing: {e}", "timing_sec": 0.0, "num_samples": 0}
	ds = load_dataset("hellaswag", split="validation", trust_remote_code=True)
	template = load_template(prompt_dir, "hellaswag", "Context: {ctx}\n\nChoices:\nA) {a}\nB) {b}\nC) {c}\nD) {d}\n\nChoose the best ending (A-D) and only output the letter.")
	correct = 0
	n = min(max_samples, len(ds))
	t0 = time.time()
	ds_fp = getattr(ds, "_fingerprint", None)
	for i in range(n):
		item = ds[i]
		ctx = (item.get("ctx_a", "") + " " + item.get("ctx_b", "")).strip()
		opts = item.get("endings", item.get("ending_options")) or item.get("endings", [])


from __future__ import annotations
import logging

import argparse
import json
from pathlib import Path
import hashlib
from typing import Any, Dict, List, Tuple

def _read_jsonl(path: Path) -> List[Dict[str, Any]]:
    rows: List[Dict[str, Any]] = []
    try:
        with path.open("r", encoding="utf-8") as f:
            for line in f:
                s = line.strip()
                if not s:
                    continue
                try:
                    rows.append(json.loads(s))
                except Exception:
                    continue
    except Exception:
        return []
    return rows

def _read_run_artifacts(runs_dir: Path) -> List[Tuple[Path, Path]]:
    """Discover historical (sidecar, results) artifact paths from prior runs.

    Returns a list of tuples (sidecar_path, results_path). Missing files are skipped.
    """
    pairs: List[Tuple[Path, Path]] = []
    try:
        if not runs_dir.exists():
            return pairs
        # Each run directory should contain a run.json with paths
        for run_sub in sorted(runs_dir.glob("*/run.json")):
            try:
                obj = json.loads(run_sub.read_text(encoding="utf-8"))
            except Exception:
                continue
            paths = obj.get("paths", {}) if isinstance(obj, dict) else {}
            side = paths.get("sidecar")
            res = paths.get("results")
            if side and res:
                sp = Path(str(side))
                rp = Path(str(res))
                if sp.exists() and rp.exists():
                    pairs.append((sp, rp))
    except Exception:
        return pairs
    return pairs

def _pair_results_with_sidecar(results: List[Dict[str, Any]], sidecar: List[Dict[str, Any]]) -> Dict[str, List[Dict[str, Any]]]:
    by_task: Dict[str, List[Dict[str, Any]]] = {}
    # Build pass map by exact completion string
    result_map: Dict[Tuple[str, str], Dict[str, Any]] = {}
    for r in results:
        task_id = str(r.get("task_id"))
        comp = str(r.get("completion", ""))
        result_map[(task_id, comp)] = r
    # Build per-task any-pass map for fallback
    any_pass: Dict[str, bool] = {}
    for r in results:
        tid = str(r.get("task_id"))
        if tid:
            any_pass[tid] = bool(any_pass.get(tid, False) or r.get("passed", False))

    for sc in sidecar:
        task_id = str(sc.get("task_id"))
        comp = str(sc.get("completion", ""))
        # Prefer exact completion match; fallback to task-level any-pass
        passed = bool(result_map.get((task_id, comp), {}).get("passed", False))
        if not passed:
            passed = bool(any_pass.get(task_id, False))
        L = by_task.setdefault(task_id, [])
        L.append({
            "task_id": task_id,
            "input": str(sc.get("input", "")),
            "orig_prompt": str(sc.get("orig_prompt", "")),
            "completion": comp,
            "passed": passed,
        })
    return by_task

def _sha256_text(s: str) -> str:
    try:
        return hashlib.sha256(s.encode("utf-8")).hexdigest()
    except Exception:
        return ""

def main() -> int:
    ap = argparse.ArgumentParser(description="Build SFT and DPO datasets from latest HumanEval sidecar/results")
    root = Path(__file__).resolve().parents[2]
    tmp_dir = root / "data" / "bench" / "tmp"
    ap.add_argument("--sidecar", default=str(tmp_dir / "humaneval_samples.with_prompts.jsonl"))
    ap.add_argument("--results", default=str(tmp_dir / "humaneval_samples.jsonl_results.jsonl"))
    ap.add_argument("--use-history", action="store_true", help="Include all prior runs in data aggregation")
    ap.add_argument("--max-runs", type=int, default=0, help="Limit number of historical runs (0 = all)")
    out_dir = root / "data" / "sandbox"
    ap.add_argument("--sft-out", default=str(out_dir / "humaneval_sft.jsonl"))
    ap.add_argument("--dpo-out", default=str(out_dir / "humaneval_dpo.jsonl"))
    ap.add_argument("--echo", action="store_true", help="Print each example as it is written")
    ap.add_argument("--print-sft", type=int, default=0, help="Print first N SFT examples (0 = none)")
    ap.add_argument("--print-dpo", type=int, default=0, help="Print first N DPO examples (0 = none)")
    ap.add_argument("--dpo-max-negatives", type=int, default=3, help="Max negatives to pair per positive per task")
    ap.add_argument("--public-hashes", default="", help="Path to newline-delimited SHA256 hashes of public solutions to exclude/flag")
    ap.add_argument("--exclude-public", action="store_true", help="Exclude completions that match public solution hashes")
    ap.add_argument("--inspiration-root", default=str((Path(__file__).resolve().parents[2] / "data" / "sandbox" / "inspiration")), help="Root containing inspiration index or raw files for overlap checks")
    ap.add_argument("--exclude-inspiration-overlap", action="store_true", default=True, help="Exclude completions that overlap with inspiration corpus (by SHA256)")
    ap.add_argument("--card-out", default=str((Path(__file__).resolve().parents[2] / "data" / "sandbox" / "humaneval_datacard.json")), help="Where to write dataset data card JSON")
    ap.add_argument("--exclude-flaky", action="store_true", help="Exclude tasks listed in tmp/humaneval_flaky.json from SFT/DPO")
    args = ap.parse_args()

    sidecar_path = Path(args.sidecar)
    results_path = Path(args.results)
    out_dir.mkdir(parents=True, exist_ok=True)

    # Aggregate current run and optional historical runs
    sidecar: List[Dict[str, Any]] = []
    results: List[Dict[str, Any]] = []
    sidecar.extend(_read_jsonl(sidecar_path))
    results.extend(_read_jsonl(results_path))
    if bool(getattr(args, "use_history", False)):
        run_pairs = _read_run_artifacts(root / "data" / "bench" / "runs" / "humaneval")
        if int(getattr(args, "max_runs", 0) or 0) > 0:
            run_pairs = run_pairs[-int(args.max_runs) :]
        for sp, rp in run_pairs:
            sidecar.extend(_read_jsonl(sp))
            results.extend(_read_jsonl(rp))

    # Dedupe by (task_id, completion)
    def _dedupe(rows: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
        seen: set[Tuple[str, str]] = set()
        out: List[Dict[str, Any]] = []
        for r in rows:
            tid = str(r.get("task_id", ""))
            comp = str(r.get("completion", ""))
            key = (tid, comp)
            if key in seen:
                continue
            seen.add(key)
            out.append(r)
        return out

    sidecar = _dedupe(sidecar)
    results = _dedupe(results)
    by_task = _pair_results_with_sidecar(results, sidecar)

    # Load flaky task ids
    flaky_tasks: set[str] = set()
    if bool(getattr(args, "exclude_flaky", False)):
        try:
            root = Path(__file__).resolve().parents[2]
            flaky_path = root / "data" / "bench" / "tmp" / "humaneval_flaky.json"
            if flaky_path.exists():
                obj = json.loads(flaky_path.read_text(encoding="utf-8"))
                ids = obj.get("tasks", []) if isinstance(obj, dict) else []
                flaky_tasks = set([str(x) for x in ids])
        except Exception:
            flaky_tasks = set()

    # Load public solution hashes (optional)
    public_hashes: set[str] = set()
    public_hash_path = str(getattr(args, "public_hashes", "") or "").strip()
    if public_hash_path:
        try:
            with Path(public_hash_path).open("r", encoding="utf-8") as ph:
                for line in ph:
                    h = line.strip()
                    if h:
                        public_hashes.add(h)
        except Exception:
            public_hashes = set()

    # Inspiration leakage hashes (best-effort)
    inspiration_hashes: set[str] = set()
    try:
        insp_root = Path(str(getattr(args, "inspiration_root")))
        idx_path = insp_root / "index.json"
        if idx_path.exists():
            try:
                obj = json.loads(idx_path.read_text(encoding="utf-8"))
                # Hash function source names where available; index stores only names, so also hash names for a weak guard
                funcs = dict(obj.get("functions", {})) if isinstance(obj, dict) else {}
                for fpath, defs in funcs.items():
                    try:
                        p = Path(fpath)
                        if p.exists():
                            text = p.read_text(encoding="utf-8", errors="ignore")
                            # Hash whole file to catch verbatim overlaps quickly
                            inspiration_hashes.add(_sha256_text(text))
                    except Exception:
                        continue
                # Additionally hash simple name lists as extremely weak heuristic
                try:
                    names_joined = "\n".join([d.get("name", "") for lst in funcs.values() for d in (lst or [])])
                    if names_joined.strip():
                        inspiration_hashes.add(_sha256_text(names_joined))
                except Exception:
                    pass
            except Exception:
                # If the index file cannot be read or is malformed, ignore and proceed.
                pass
        else:
            # Fallback: hash all .py files under inspiration root if available
            if insp_root.exists() and insp_root.is_dir():
                for p in insp_root.rglob("*.py"):
                    try:
                        inspiration_hashes.add(_sha256_text(p.read_text(encoding="utf-8", errors="ignore")))
                    except Exception:
                        continue
    except Exception:
        inspiration_hashes = set()

    # Emit SFT: body-only pairs for passed samples (skip empty bodies; add lineage)
    sft_out = Path(args.sft_out)
    with sft_out.open("w", encoding="utf-8") as f:
        written_sft = 0
        printed_sft = 0
        public_overlap_sft = 0
        for task_id, lst in by_task.items():
            if task_id in flaky_tasks:
                continue
            for rec in lst:
                if rec.get("passed"):
                    row = {
                        "suite": "humaneval",
                        "task_id": task_id,
                        "prompt": rec.get("input"),
                        "response": rec.get("completion"),
                        "source": "humaneval_run",
                    }
                    # Skip empty completions to keep dataset clean
                    if str(row.get("response", "")).strip():
                        # Public solution hash check
                        chash = _sha256_text(str(row.get("response", "")))
                        is_pub = bool(chash and (chash in public_hashes))
                        # Inspiration overlap check: compare against file-level hashes; best-effort
                        is_insp = bool(chash and (chash in inspiration_hashes))
                        if (is_pub and bool(getattr(args, "exclude_public", False))) or (is_insp and bool(getattr(args, "exclude_inspiration_overlap", False))):
                            public_overlap_sft += 1
                        else:
                            f.write(json.dumps({**row, **({"hash": chash} if chash else {})}, ensure_ascii=False) + "\n")
                            written_sft += 1
                    if args.echo:
                        print(f"[SFT] {task_id}")
                    if args.print_sft and printed_sft < int(args.print_sft):
                        print(json.dumps(row, ensure_ascii=False))
                        printed_sft += 1

    # Emit DPO: choose a passing completion vs up to K failing ones for the same task
    dpo_out = Path(args.dpo_out)
    with dpo_out.open("w", encoding="utf-8") as f:
        written_dpo = 0
        printed_dpo = 0
        public_overlap_dpo = 0
        for task_id, lst in by_task.items():
            if task_id in flaky_tasks:
                continue
            pos = [rec for rec in lst if rec.get("passed")]
            neg = [rec for rec in lst if not rec.get("passed")]
            if not pos or not neg:
                continue
            # Hard negatives: sort by frequency (desc) to up-weight common failures
            freq: Dict[str, int] = {}
            for r in neg:
                c = str(r.get("completion", ""))
                freq[c] = freq.get(c, 0) + 1
            neg_sorted = sorted(neg, key=lambda r: freq.get(str(r.get("completion", "")), 0), reverse=True)
            max_negs = max(1, int(getattr(args, "dpo_max_negatives", 3) or 3))
            chosen = pos[0]
            for rejected in neg_sorted[:max_negs]:
                row = {
                    "suite": "humaneval",
                    "task_id": task_id,
                    "prompt": chosen.get("input"),
                    "chosen": chosen.get("completion"),
                    "rejected": rejected.get("completion"),
                }
                # Public overlap checks for both sides
                chash = _sha256_text(str(row.get("chosen", "")))
                rhash = _sha256_text(str(row.get("rejected", "")))
                is_pub = bool((chash and chash in public_hashes) or (rhash and rhash in public_hashes))
                is_insp = bool((chash and chash in inspiration_hashes) or (rhash and rhash in inspiration_hashes))
                if (is_pub and bool(getattr(args, "exclude_public", False))) or (is_insp and bool(getattr(args, "exclude_inspiration_overlap", False))):
                    public_overlap_dpo += 1
                    continue
                f.write(json.dumps({**row, **({"chosen_hash": chash} if chash else {}), **({"rejected_hash": rhash} if rhash else {})}, ensure_ascii=False) + "\n")
                written_dpo += 1
                if args.echo:
                    print(f"[DPO] {task_id}")
                if args.print_dpo and printed_dpo < int(args.print_dpo):
                    print(json.dumps(row, ensure_ascii=False))
                    printed_dpo += 1

    # Write a small summary
    # Data card with counts and simple checksums
    sidecar_count = len(sidecar)
    results_count = len(results)
    card = {
        "ok": True,
        "sidecar_count": sidecar_count,
        "results_count": results_count,
        "sft_rows": written_sft,
        "dpo_rows": written_dpo,
        "public_overlap": {
            "sft": (locals().get("public_overlap_sft", 0)),
            "dpo": (locals().get("public_overlap_dpo", 0)),
        },
        "inputs": {
            "sidecar": str(sidecar_path),
            "results": str(results_path),
        },
        "outputs": {
            "sft_out": str(sft_out),
            "dpo_out": str(dpo_out),
        },
        "hashes": {
            "sidecar_sha256": _sha256_text("\n".join([json.dumps(r, sort_keys=True) for r in sidecar])[:1_000_000]),
            "results_sha256": _sha256_text("\n".join([json.dumps(r, sort_keys=True) for r in results])[:1_000_000]),
        },
    }
    try:
        Path(str(getattr(args, "card_out"))).parent.mkdir(parents=True, exist_ok=True)
        Path(str(getattr(args, "card_out"))).write_text(json.dumps(card, ensure_ascii=False, indent=2), encoding="utf-8")
    except Exception:
        pass

    summary = {
        "ok": True,
        "sft_rows": written_sft,
        "dpo_rows": written_dpo,
        "sidecar": str(sidecar_path),
        "results": str(results_path),
        "sft_out": str(sft_out),
        "dpo_out": str(dpo_out),
    }
    print(json.dumps(summary))
    return 0

if __name__ == "__main__":
    raise SystemExit(main())


from __future__ import annotations
import logging

import argparse
import json
from pathlib import Path
import hashlib
from typing import Any, Dict, List, Tuple

def _read_jsonl(path: Path) -> List[Dict[str, Any]]:
    rows: List[Dict[str, Any]] = []
    try:
        with path.open("r", encoding="utf-8") as f:
            for line in f:
                s = line.strip()
                if not s:
                    continue
                try:
                    rows.append(json.loads(s))
                except Exception:
                    continue
    except Exception:
        return []
    return rows

def _read_run_artifacts(runs_dir: Path) -> List[Tuple[Path, Path]]:
    """Discover historical (sidecar, results) artifact paths from prior runs.

    Returns a list of tuples (sidecar_path, results_path). Missing files are skipped.
    """
    pairs: List[Tuple[Path, Path]] = []
    try:
        if not runs_dir.exists():
            return pairs
        # Each run directory should contain a run.json with paths
        for run_sub in sorted(runs_dir.glob("*/run.json")):
            try:
                obj = json.loads(run_sub.read_text(encoding="utf-8"))
            except Exception:
                continue
            paths = obj.get("paths", {}) if isinstance(obj, dict) else {}
            side = paths.get("sidecar")
            res = paths.get("results")


from typing import Any, Dict, List, Tuple

def _read_jsonl(path: Path) -> List[Dict[str, Any]]:
    rows: List[Dict[str, Any]] = []
    try:
        with path.open("r", encoding="utf-8") as f:
            for line in f:
                s = line.strip()
                if not s:
                    continue
                try:
                    rows.append(json.loads(s))
                except Exception:
                    continue
    except Exception:
        return []
    return rows

def _read_run_artifacts(runs_dir: Path) -> List[Tuple[Path, Path]]:
    """Discover historical (sidecar, results) artifact paths from prior runs.

    Returns a list of tuples (sidecar_path, results_path). Missing files are skipped.
    """
    pairs: List[Tuple[Path, Path]] = []
    try:
        if not runs_dir.exists():
            return pairs
        # Each run directory should contain a run.json with paths
        for run_sub in sorted(runs_dir.glob("*/run.json")):
            try:
                obj = json.loads(run_sub.read_text(encoding="utf-8"))
            except Exception:
                continue
            paths = obj.get("paths", {}) if isinstance(obj, dict) else {}
            side = paths.get("sidecar")
            res = paths.get("results")
            if side and res:
                sp = Path(str(side))
                rp = Path(str(res))
                if sp.exists() and rp.exists():
                    pairs.append((sp, rp))
    except Exception:
        return pairs
    return pairs

def _pair_results_with_sidecar(results: List[Dict[str, Any]], sidecar: List[Dict[str, Any]]) -> Dict[str, List[Dict[str, Any]]]:
    by_task: Dict[str, List[Dict[str, Any]]] = {}
    # Build pass map by exact completion string
    result_map: Dict[Tuple[str, str], Dict[str, Any]] = {}
    for r in results:
        task_id = str(r.get("task_id"))
        comp = str(r.get("completion", ""))
        result_map[(task_id, comp)] = r
    # Build per-task any-pass map for fallback
    any_pass: Dict[str, bool] = {}
    for r in results:
        tid = str(r.get("task_id"))
        if tid:
            any_pass[tid] = bool(any_pass.get(tid, False) or r.get("passed", False))

    for sc in sidecar:
        task_id = str(sc.get("task_id"))
        comp = str(sc.get("completion", ""))


            return pairs
        # Each run directory should contain a run.json with paths
        for run_sub in sorted(runs_dir.glob("*/run.json")):
            try:
                obj = json.loads(run_sub.read_text(encoding="utf-8"))
            except Exception:
                continue
            paths = obj.get("paths", {}) if isinstance(obj, dict) else {}
            side = paths.get("sidecar")
            res = paths.get("results")
            if side and res:
                sp = Path(str(side))
                rp = Path(str(res))
                if sp.exists() and rp.exists():
                    pairs.append((sp, rp))
    except Exception:
        return pairs
    return pairs

def _pair_results_with_sidecar(results: List[Dict[str, Any]], sidecar: List[Dict[str, Any]]) -> Dict[str, List[Dict[str, Any]]]:
    by_task: Dict[str, List[Dict[str, Any]]] = {}
    # Build pass map by exact completion string
    result_map: Dict[Tuple[str, str], Dict[str, Any]] = {}
    for r in results:
        task_id = str(r.get("task_id"))
        comp = str(r.get("completion", ""))
        result_map[(task_id, comp)] = r
    # Build per-task any-pass map for fallback
    any_pass: Dict[str, bool] = {}
    for r in results:
        tid = str(r.get("task_id"))
        if tid:
            any_pass[tid] = bool(any_pass.get(tid, False) or r.get("passed", False))

    for sc in sidecar:
        task_id = str(sc.get("task_id"))
        comp = str(sc.get("completion", ""))
        # Prefer exact completion match; fallback to task-level any-pass
        passed = bool(result_map.get((task_id, comp), {}).get("passed", False))
        if not passed:
            passed = bool(any_pass.get(task_id, False))
        L = by_task.setdefault(task_id, [])
        L.append({
            "task_id": task_id,
            "input": str(sc.get("input", "")),
            "orig_prompt": str(sc.get("orig_prompt", "")),
            "completion": comp,
            "passed": passed,
        })
    return by_task

def _sha256_text(s: str) -> str:
    try:
        return hashlib.sha256(s.encode("utf-8")).hexdigest()
    except Exception:
        return ""

def main() -> int:
    ap = argparse.ArgumentParser(description="Build SFT and DPO datasets from latest HumanEval sidecar/results")
    root = Path(__file__).resolve().parents[2]
    tmp_dir = root / "data" / "bench" / "tmp"
    ap.add_argument("--sidecar", default=str(tmp_dir / "humaneval_samples.with_prompts.jsonl"))
    ap.add_argument("--results", default=str(tmp_dir / "humaneval_samples.jsonl_results.jsonl"))
    ap.add_argument("--use-history", action="store_true", help="Include all prior runs in data aggregation")
    ap.add_argument("--max-runs", type=int, default=0, help="Limit number of historical runs (0 = all)")
    out_dir = root / "data" / "sandbox"
    ap.add_argument("--sft-out", default=str(out_dir / "humaneval_sft.jsonl"))
    ap.add_argument("--dpo-out", default=str(out_dir / "humaneval_dpo.jsonl"))


            any_pass[tid] = bool(any_pass.get(tid, False) or r.get("passed", False))

    for sc in sidecar:
        task_id = str(sc.get("task_id"))
        comp = str(sc.get("completion", ""))
        # Prefer exact completion match; fallback to task-level any-pass
        passed = bool(result_map.get((task_id, comp), {}).get("passed", False))
        if not passed:
            passed = bool(any_pass.get(task_id, False))
        L = by_task.setdefault(task_id, [])
        L.append({
            "task_id": task_id,
            "input": str(sc.get("input", "")),
            "orig_prompt": str(sc.get("orig_prompt", "")),
            "completion": comp,
            "passed": passed,
        })
    return by_task

def _sha256_text(s: str) -> str:
    try:
        return hashlib.sha256(s.encode("utf-8")).hexdigest()
    except Exception:
        return ""

def main() -> int:
    ap = argparse.ArgumentParser(description="Build SFT and DPO datasets from latest HumanEval sidecar/results")
    root = Path(__file__).resolve().parents[2]
    tmp_dir = root / "data" / "bench" / "tmp"
    ap.add_argument("--sidecar", default=str(tmp_dir / "humaneval_samples.with_prompts.jsonl"))
    ap.add_argument("--results", default=str(tmp_dir / "humaneval_samples.jsonl_results.jsonl"))
    ap.add_argument("--use-history", action="store_true", help="Include all prior runs in data aggregation")
    ap.add_argument("--max-runs", type=int, default=0, help="Limit number of historical runs (0 = all)")
    out_dir = root / "data" / "sandbox"
    ap.add_argument("--sft-out", default=str(out_dir / "humaneval_sft.jsonl"))
    ap.add_argument("--dpo-out", default=str(out_dir / "humaneval_dpo.jsonl"))
    ap.add_argument("--echo", action="store_true", help="Print each example as it is written")
    ap.add_argument("--print-sft", type=int, default=0, help="Print first N SFT examples (0 = none)")
    ap.add_argument("--print-dpo", type=int, default=0, help="Print first N DPO examples (0 = none)")
    ap.add_argument("--dpo-max-negatives", type=int, default=3, help="Max negatives to pair per positive per task")
    ap.add_argument("--public-hashes", default="", help="Path to newline-delimited SHA256 hashes of public solutions to exclude/flag")
    ap.add_argument("--exclude-public", action="store_true", help="Exclude completions that match public solution hashes")
    ap.add_argument("--inspiration-root", default=str((Path(__file__).resolve().parents[2] / "data" / "sandbox" / "inspiration")), help="Root containing inspiration index or raw files for overlap checks")


main

        if not passed:
            passed = bool(any_pass.get(task_id, False))
        L = by_task.setdefault(task_id, [])
        L.append({
            "task_id": task_id,
            "input": str(sc.get("input", "")),
            "orig_prompt": str(sc.get("orig_prompt", "")),
            "completion": comp,
            "passed": passed,
        })
    return by_task

def _sha256_text(s: str) -> str:
    try:
        return hashlib.sha256(s.encode("utf-8")).hexdigest()
    except Exception:
        return ""

def main() -> int:
    ap = argparse.ArgumentParser(description="Build SFT and DPO datasets from latest HumanEval sidecar/results")
    root = Path(__file__).resolve().parents[2]
    tmp_dir = root / "data" / "bench" / "tmp"
    ap.add_argument("--sidecar", default=str(tmp_dir / "humaneval_samples.with_prompts.jsonl"))
    ap.add_argument("--results", default=str(tmp_dir / "humaneval_samples.jsonl_results.jsonl"))
    ap.add_argument("--use-history", action="store_true", help="Include all prior runs in data aggregation")
    ap.add_argument("--max-runs", type=int, default=0, help="Limit number of historical runs (0 = all)")
    out_dir = root / "data" / "sandbox"
    ap.add_argument("--sft-out", default=str(out_dir / "humaneval_sft.jsonl"))
    ap.add_argument("--dpo-out", default=str(out_dir / "humaneval_dpo.jsonl"))
    ap.add_argument("--echo", action="store_true", help="Print each example as it is written")
    ap.add_argument("--print-sft", type=int, default=0, help="Print first N SFT examples (0 = none)")
    ap.add_argument("--print-dpo", type=int, default=0, help="Print first N DPO examples (0 = none)")
    ap.add_argument("--dpo-max-negatives", type=int, default=3, help="Max negatives to pair per positive per task")
    ap.add_argument("--public-hashes", default="", help="Path to newline-delimited SHA256 hashes of public solutions to exclude/flag")
    ap.add_argument("--exclude-public", action="store_true", help="Exclude completions that match public solution hashes")
    ap.add_argument("--inspiration-root", default=str((Path(__file__).resolve().parents[2] / "data" / "sandbox" / "inspiration")), help="Root containing inspiration index or raw files for overlap checks")
    ap.add_argument("--exclude-inspiration-overlap", action="store_true", default=True, help="Exclude completions that overlap with inspiration corpus (by SHA256)")
    ap.add_argument("--card-out", default=str((Path(__file__).resolve().parents[2] / "data" / "sandbox" / "humaneval_datacard.json")), help="Where to write dataset data card JSON")
    ap.add_argument("--exclude-flaky", action="store_true", help="Exclude tasks listed in tmp/humaneval_flaky.json from SFT/DPO")
    args = ap.parse_args()

    sidecar_path = Path(args.sidecar)
    results_path = Path(args.results)
    out_dir.mkdir(parents=True, exist_ok=True)

    # Aggregate current run and optional historical runs
    sidecar: List[Dict[str, Any]] = []
    results: List[Dict[str, Any]] = []
    sidecar.extend(_read_jsonl(sidecar_path))
    results.extend(_read_jsonl(results_path))
    if bool(getattr(args, "use_history", False)):
        run_pairs = _read_run_artifacts(root / "data" / "bench" / "runs" / "humaneval")
        if int(getattr(args, "max_runs", 0) or 0) > 0:
            run_pairs = run_pairs[-int(args.max_runs) :]
        for sp, rp in run_pairs:
            sidecar.extend(_read_jsonl(sp))
            results.extend(_read_jsonl(rp))

    # Dedupe by (task_id, completion)
    def _dedupe(rows: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
        seen: set[Tuple[str, str]] = set()
        out: List[Dict[str, Any]] = []
        for r in rows:
            tid = str(r.get("task_id", ""))
            comp = str(r.get("completion", ""))
            key = (tid, comp)
            if key in seen:
                continue
            seen.add(key)
            out.append(r)
        return out

    sidecar = _dedupe(sidecar)
    results = _dedupe(results)
    by_task = _pair_results_with_sidecar(results, sidecar)

    # Load flaky task ids
    flaky_tasks: set[str] = set()
    if bool(getattr(args, "exclude_flaky", False)):
        try:
            root = Path(__file__).resolve().parents[2]
            flaky_path = root / "data" / "bench" / "tmp" / "humaneval_flaky.json"
            if flaky_path.exists():
                obj = json.loads(flaky_path.read_text(encoding="utf-8"))
                ids = obj.get("tasks", []) if isinstance(obj, dict) else []
                flaky_tasks = set([str(x) for x in ids])
        except Exception:
            flaky_tasks = set()

    # Load public solution hashes (optional)
    public_hashes: set[str] = set()
    public_hash_path = str(getattr(args, "public_hashes", "") or "").strip()
    if public_hash_path:
        try:
            with Path(public_hash_path).open("r", encoding="utf-8") as ph:
                for line in ph:
                    h = line.strip()
                    if h:
                        public_hashes.add(h)
        except Exception:
            public_hashes = set()

    # Inspiration leakage hashes (best-effort)
    inspiration_hashes: set[str] = set()
    try:
        insp_root = Path(str(getattr(args, "inspiration_root")))
        idx_path = insp_root / "index.json"
        if idx_path.exists():
            try:
                obj = json.loads(idx_path.read_text(encoding="utf-8"))
                # Hash function source names where available; index stores only names, so also hash names for a weak guard
                funcs = dict(obj.get("functions", {})) if isinstance(obj, dict) else {}
                for fpath, defs in funcs.items():
                    try:
                        p = Path(fpath)
                        if p.exists():
                            text = p.read_text(encoding="utf-8", errors="ignore")
                            # Hash whole file to catch verbatim overlaps quickly
                            inspiration_hashes.add(_sha256_text(text))
                    except Exception:
                        continue
                # Additionally hash simple name lists as extremely weak heuristic
                try:
                    names_joined = "\n".join([d.get("name", "") for lst in funcs.values() for d in (lst or [])])
                    if names_joined.strip():
                        inspiration_hashes.add(_sha256_text(names_joined))
                except Exception:
                    pass
            except Exception:
                # If the index file cannot be read or is malformed, ignore and proceed.
                pass
        else:
            # Fallback: hash all .py files under inspiration root if available
            if insp_root.exists() and insp_root.is_dir():
                for p in insp_root.rglob("*.py"):
                    try:
                        inspiration_hashes.add(_sha256_text(p.read_text(encoding="utf-8", errors="ignore")))
                    except Exception:
                        continue
    except Exception:
        inspiration_hashes = set()

    # Emit SFT: body-only pairs for passed samples (skip empty bodies; add lineage)
    sft_out = Path(args.sft_out)
    with sft_out.open("w", encoding="utf-8") as f:
        written_sft = 0
        printed_sft = 0
        public_overlap_sft = 0
        for task_id, lst in by_task.items():
            if task_id in flaky_tasks:
                continue
            for rec in lst:
                if rec.get("passed"):
                    row = {
                        "suite": "humaneval",
                        "task_id": task_id,
                        "prompt": rec.get("input"),
                        "response": rec.get("completion"),
                        "source": "humaneval_run",
                    }
                    # Skip empty completions to keep dataset clean
                    if str(row.get("response", "")).strip():
                        # Public solution hash check
                        chash = _sha256_text(str(row.get("response", "")))
                        is_pub = bool(chash and (chash in public_hashes))
                        # Inspiration overlap check: compare against file-level hashes; best-effort
                        is_insp = bool(chash and (chash in inspiration_hashes))
                        if (is_pub and bool(getattr(args, "exclude_public", False))) or (is_insp and bool(getattr(args, "exclude_inspiration_overlap", False))):
                            public_overlap_sft += 1
                        else:
                            f.write(json.dumps({**row, **({"hash": chash} if chash else {})}, ensure_ascii=False) + "\n")
                            written_sft += 1
                    if args.echo:
                        print(f"[SFT] {task_id}")
                    if args.print_sft and printed_sft < int(args.print_sft):
                        print(json.dumps(row, ensure_ascii=False))
                        printed_sft += 1

    # Emit DPO: choose a passing completion vs up to K failing ones for the same task
    dpo_out = Path(args.dpo_out)
    with dpo_out.open("w", encoding="utf-8") as f:
        written_dpo = 0
        printed_dpo = 0
        public_overlap_dpo = 0
        for task_id, lst in by_task.items():
            if task_id in flaky_tasks:
                continue
            pos = [rec for rec in lst if rec.get("passed")]
            neg = [rec for rec in lst if not rec.get("passed")]
            if not pos or not neg:
                continue
            # Hard negatives: sort by frequency (desc) to up-weight common failures
            freq: Dict[str, int] = {}
            for r in neg:
                c = str(r.get("completion", ""))
                freq[c] = freq.get(c, 0) + 1
            neg_sorted = sorted(neg, key=lambda r: freq.get(str(r.get("completion", "")), 0), reverse=True)
            max_negs = max(1, int(getattr(args, "dpo_max_negatives", 3) or 3))
            chosen = pos[0]
            for rejected in neg_sorted[:max_negs]:
                row = {
                    "suite": "humaneval",
                    "task_id": task_id,
                    "prompt": chosen.get("input"),
                    "chosen": chosen.get("completion"),
                    "rejected": rejected.get("completion"),
                }
                # Public overlap checks for both sides
                chash = _sha256_text(str(row.get("chosen", "")))
                rhash = _sha256_text(str(row.get("rejected", "")))
                is_pub = bool((chash and chash in public_hashes) or (rhash and rhash in public_hashes))
                is_insp = bool((chash and chash in inspiration_hashes) or (rhash and rhash in inspiration_hashes))
                if (is_pub and bool(getattr(args, "exclude_public", False))) or (is_insp and bool(getattr(args, "exclude_inspiration_overlap", False))):
                    public_overlap_dpo += 1
                    continue
                f.write(json.dumps({**row, **({"chosen_hash": chash} if chash else {}), **({"rejected_hash": rhash} if rhash else {})}, ensure_ascii=False) + "\n")
                written_dpo += 1
                if args.echo:
                    print(f"[DPO] {task_id}")
                if args.print_dpo and printed_dpo < int(args.print_dpo):
                    print(json.dumps(row, ensure_ascii=False))
                    printed_dpo += 1

    # Write a small summary
    # Data card with counts and simple checksums
    sidecar_count = len(sidecar)
    results_count = len(results)
    card = {
        "ok": True,
        "sidecar_count": sidecar_count,
        "results_count": results_count,
        "sft_rows": written_sft,
        "dpo_rows": written_dpo,
        "public_overlap": {
            "sft": (locals().get("public_overlap_sft", 0)),
            "dpo": (locals().get("public_overlap_dpo", 0)),
        },
        "inputs": {
            "sidecar": str(sidecar_path),
            "results": str(results_path),
        },
        "outputs": {
            "sft_out": str(sft_out),
            "dpo_out": str(dpo_out),
        },
        "hashes": {
            "sidecar_sha256": _sha256_text("\n".join([json.dumps(r, sort_keys=True) for r in sidecar])[:1_000_000]),
            "results_sha256": _sha256_text("\n".join([json.dumps(r, sort_keys=True) for r in results])[:1_000_000]),
        },
    }
    try:
        Path(str(getattr(args, "card_out"))).parent.mkdir(parents=True, exist_ok=True)
        Path(str(getattr(args, "card_out"))).write_text(json.dumps(card, ensure_ascii=False, indent=2), encoding="utf-8")
    except Exception:
        pass

    summary = {
        "ok": True,
        "sft_rows": written_sft,
        "dpo_rows": written_dpo,
        "sidecar": str(sidecar_path),
        "results": str(results_path),
        "sft_out": str(sft_out),
        "dpo_out": str(dpo_out),
    }
    print(json.dumps(summary))
    return 0

if __name__ == "__main__":
    raise SystemExit(main())


    args = ap.parse_args()

    sidecar_path = Path(args.sidecar)
    results_path = Path(args.results)
    out_dir.mkdir(parents=True, exist_ok=True)

    # Aggregate current run and optional historical runs
    sidecar: List[Dict[str, Any]] = []
    results: List[Dict[str, Any]] = []
    sidecar.extend(_read_jsonl(sidecar_path))
    results.extend(_read_jsonl(results_path))
    if bool(getattr(args, "use_history", False)):
        run_pairs = _read_run_artifacts(root / "data" / "bench" / "runs" / "humaneval")
        if int(getattr(args, "max_runs", 0) or 0) > 0:
            run_pairs = run_pairs[-int(args.max_runs) :]
        for sp, rp in run_pairs:
            sidecar.extend(_read_jsonl(sp))
            results.extend(_read_jsonl(rp))

    # Dedupe by (task_id, completion)
    def _dedupe(rows: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
        seen: set[Tuple[str, str]] = set()
        out: List[Dict[str, Any]] = []
        for r in rows:
            tid = str(r.get("task_id", ""))
            comp = str(r.get("completion", ""))
            key = (tid, comp)
            if key in seen:
                continue
            seen.add(key)
            out.append(r)
        return out

    sidecar = _dedupe(sidecar)
    results = _dedupe(results)
    by_task = _pair_results_with_sidecar(results, sidecar)

    # Load flaky task ids
    flaky_tasks: set[str] = set()
    if bool(getattr(args, "exclude_flaky", False)):
        try:
            root = Path(__file__).resolve().parents[2]
            flaky_path = root / "data" / "bench" / "tmp" / "humaneval_flaky.json"
            if flaky_path.exists():
                obj = json.loads(flaky_path.read_text(encoding="utf-8"))
                ids = obj.get("tasks", []) if isinstance(obj, dict) else []
                flaky_tasks = set([str(x) for x in ids])
        except Exception:
            flaky_tasks = set()

    # Load public solution hashes (optional)
    public_hashes: set[str] = set()


from __future__ import annotations
import logging

import argparse
import json
from pathlib import Path
from typing import Dict

def iter_jsonl(p: Path):
	if not p.exists():
		return
	for line in p.read_text(encoding="utf-8").splitlines():
		s = line.strip()
		if not s:
			continue
		try:
			yield json.loads(s)
		except Exception:
			continue

def main() -> int:
	ap = argparse.ArgumentParser()
	root = Path(__file__).resolve().parents[2]
	ap.add_argument("--results-dir", default=str(root / "data" / "bench" / "results"))
	ap.add_argument("--out", default=str(root / "data" / "dashboards" / "benchmarks.json"))
	args = ap.parse_args()

	res_dir = Path(args.results_dir)
	summary: Dict[str, Dict[str, float]] = {}
	# Aggregate known suites present in results dir (APPS removed)
	for name in ("humaneval", "mbpp", "swebench_lite"):
		p = res_dir / f"{name}.jsonl"
		total = ok = passk_ok = 0
		el: list[float] = []
		for rec in iter_jsonl(p) or []:
			total += 1
			if bool(rec.get("pass1")):
				ok += 1
			if bool(rec.get("passk")):
				passk_ok += 1
			try:
				el.append(float(rec.get("elapsed_sec", 0.0)))
			except Exception:
				pass
		def p90(xs: list[float]) -> float:
			if not xs:
				return 0.0
			xs2 = sorted(xs)
			idx = min(len(xs2) - 1, max(0, int(round(0.9 * (len(xs2) - 1)))))
			return float(xs2[idx])
		summary[name] = {
			"pass1": round(ok / max(1, total), 4),
			"passk": round(passk_ok / max(1, total), 4),
			"p90": round(p90(el), 3),
		}

	outp = Path(args.out)
	outp.parent.mkdir(parents=True, exist_ok=True)
	outp.write_text(json.dumps({"summary": summary}, ensure_ascii=False, indent=2), encoding="utf-8")
	print(json.dumps({"ok": True, "out": str(outp)}))
	return 0

if __name__ == "__main__":
	raise SystemExit(main())


from __future__ import annotations
import logging

import argparse
import json
from pathlib import Path
from typing import Dict

def iter_jsonl(p: Path):
	if not p.exists():
		return
	for line in p.read_text(encoding="utf-8").splitlines():
		s = line.strip()
		if not s:
			continue
		try:
			yield json.loads(s)
		except Exception:
			continue

def main() -> int:
	ap = argparse.ArgumentParser()
	root = Path(__file__).resolve().parents[2]
	ap.add_argument("--results-dir", default=str(root / "data" / "bench" / "results"))
	ap.add_argument("--out", default=str(root / "data" / "dashboards" / "benchmarks.json"))
	args = ap.parse_args()

	res_dir = Path(args.results_dir)
	summary: Dict[str, Dict[str, float]] = {}
	# Aggregate known suites present in results dir (APPS removed)
	for name in ("humaneval", "mbpp", "swebench_lite"):
		p = res_dir / f"{name}.jsonl"
		total = ok = passk_ok = 0
		el: list[float] = []
		for rec in iter_jsonl(p) or []:
			total += 1
			if bool(rec.get("pass1")):
				ok += 1


import logging

import argparse
import json
from pathlib import Path
from typing import Dict

def iter_jsonl(p: Path):
	if not p.exists():
		return
	for line in p.read_text(encoding="utf-8").splitlines():
		s = line.strip()
		if not s:
			continue
		try:
			yield json.loads(s)
		except Exception:
			continue

def main() -> int:
	ap = argparse.ArgumentParser()
	root = Path(__file__).resolve().parents[2]
	ap.add_argument("--results-dir", default=str(root / "data" / "bench" / "results"))
	ap.add_argument("--out", default=str(root / "data" / "dashboards" / "benchmarks.json"))
	args = ap.parse_args()

	res_dir = Path(args.results_dir)
	summary: Dict[str, Dict[str, float]] = {}
	# Aggregate known suites present in results dir (APPS removed)
	for name in ("humaneval", "mbpp", "swebench_lite"):
		p = res_dir / f"{name}.jsonl"
		total = ok = passk_ok = 0
		el: list[float] = []
		for rec in iter_jsonl(p) or []:
			total += 1
			if bool(rec.get("pass1")):
				ok += 1
			if bool(rec.get("passk")):
				passk_ok += 1
			try:
				el.append(float(rec.get("elapsed_sec", 0.0)))
			except Exception:
				pass
		def p90(xs: list[float]) -> float:
			if not xs:
				return 0.0
			xs2 = sorted(xs)
			idx = min(len(xs2) - 1, max(0, int(round(0.9 * (len(xs2) - 1)))))
			return float(xs2[idx])
		summary[name] = {
			"pass1": round(ok / max(1, total), 4),
			"passk": round(passk_ok / max(1, total), 4),
			"p90": round(p90(el), 3),
		}

	outp = Path(args.out)
	outp.parent.mkdir(parents=True, exist_ok=True)
	outp.write_text(json.dumps({"summary": summary}, ensure_ascii=False, indent=2), encoding="utf-8")
	print(json.dumps({"ok": True, "out": str(outp)}))
	return 0

if __name__ == "__main__":
	raise SystemExit(main())


	ap.add_argument("--out", default=str(root / "data" / "dashboards" / "benchmarks.json"))
	args = ap.parse_args()

	res_dir = Path(args.results_dir)
	summary: Dict[str, Dict[str, float]] = {}
	# Aggregate known suites present in results dir (APPS removed)
	for name in ("humaneval", "mbpp", "swebench_lite"):
		p = res_dir / f"{name}.jsonl"
		total = ok = passk_ok = 0
		el: list[float] = []
		for rec in iter_jsonl(p) or []:
			total += 1
			if bool(rec.get("pass1")):
				ok += 1
			if bool(rec.get("passk")):
				passk_ok += 1
			try:
				el.append(float(rec.get("elapsed_sec", 0.0)))
			except Exception:
				pass
		def p90(xs: list[float]) -> float:
			if not xs:
				return 0.0
			xs2 = sorted(xs)
			idx = min(len(xs2) - 1, max(0, int(round(0.9 * (len(xs2) - 1)))))
			return float(xs2[idx])
		summary[name] = {
			"pass1": round(ok / max(1, total), 4),
			"passk": round(passk_ok / max(1, total), 4),
			"p90": round(p90(el), 3),
		}

	outp = Path(args.out)
	outp.parent.mkdir(parents=True, exist_ok=True)
	outp.write_text(json.dumps({"summary": summary}, ensure_ascii=False, indent=2), encoding="utf-8")
	print(json.dumps({"ok": True, "out": str(outp)}))
	return 0

if __name__ == "__main__":
	raise SystemExit(main())


from __future__ import annotations

import logging
import hashlib
import json
from pathlib import Path
from typing import Optional

class LLMCache:
	"""Simple file-based cache mapping (model,prompt,params)->response."""

	def __init__(self, root_dir: str | Path) -> None:
		self.dir = Path(root_dir)
		self.dir.mkdir(parents=True, exist_ok=True)

	def _key(self, model: str, prompt: str, params: dict) -> str:
		obj = {"m": model, "p": prompt, "a": params}
		dig = hashlib.sha256(json.dumps(obj, sort_keys=True).encode("utf-8")).hexdigest()
		return dig

	def get(self, model: str, prompt: str, params: dict) -> Optional[str]:
		k = self._key(model, prompt, params)
		p = self.dir / f"{k}.txt"
		if p.exists():
			try:
				return p.read_text(encoding="utf-8")
			except Exception:
				return None
		return None

	def put(self, model: str, prompt: str, params: dict, text: str) -> None:
		k = self._key(model, prompt, params)
		p = self.dir / f"{k}.txt"
		try:
			p.write_text(text, encoding="utf-8")
		except Exception:
			return


from __future__ import annotations

import logging
import hashlib
import json
from pathlib import Path
from typing import Optional

class LLMCache:
	"""Simple file-based cache mapping (model,prompt,params)->response."""

	def __init__(self, root_dir: str | Path) -> None:
		self.dir = Path(root_dir)
		self.dir.mkdir(parents=True, exist_ok=True)

	def _key(self, model: str, prompt: str, params: dict) -> str:
		obj = {"m": model, "p": prompt, "a": params}
		dig = hashlib.sha256(json.dumps(obj, sort_keys=True).encode("utf-8")).hexdigest()
		return dig

	def get(self, model: str, prompt: str, params: dict) -> Optional[str]:
		k = self._key(model, prompt, params)
		p = self.dir / f"{k}.txt"
		if p.exists():
			try:
				return p.read_text(encoding="utf-8")
			except Exception:
				return None
		return None

	def put(self, model: str, prompt: str, params: dict, text: str) -> None:
		k = self._key(model, prompt, params)
		p = self.dir / f"{k}.txt"
		try:


get

import logging
import hashlib
import json
from pathlib import Path
from typing import Optional

class LLMCache:
	"""Simple file-based cache mapping (model,prompt,params)->response."""

	def __init__(self, root_dir: str | Path) -> None:
		self.dir = Path(root_dir)
		self.dir.mkdir(parents=True, exist_ok=True)

	def _key(self, model: str, prompt: str, params: dict) -> str:
		obj = {"m": model, "p": prompt, "a": params}
		dig = hashlib.sha256(json.dumps(obj, sort_keys=True).encode("utf-8")).hexdigest()
		return dig

	def get(self, model: str, prompt: str, params: dict) -> Optional[str]:
		k = self._key(model, prompt, params)
		p = self.dir / f"{k}.txt"
		if p.exists():
			try:
				return p.read_text(encoding="utf-8")
			except Exception:
				return None
		return None

	def put(self, model: str, prompt: str, params: dict, text: str) -> None:
		k = self._key(model, prompt, params)
		p = self.dir / f"{k}.txt"
		try:
			p.write_text(text, encoding="utf-8")
		except Exception:
			return


put

	def __init__(self, root_dir: str | Path) -> None:
		self.dir = Path(root_dir)
		self.dir.mkdir(parents=True, exist_ok=True)

	def _key(self, model: str, prompt: str, params: dict) -> str:
		obj = {"m": model, "p": prompt, "a": params}
		dig = hashlib.sha256(json.dumps(obj, sort_keys=True).encode("utf-8")).hexdigest()
		return dig

	def get(self, model: str, prompt: str, params: dict) -> Optional[str]:
		k = self._key(model, prompt, params)
		p = self.dir / f"{k}.txt"
		if p.exists():
			try:
				return p.read_text(encoding="utf-8")
			except Exception:
				return None
		return None

	def put(self, model: str, prompt: str, params: dict, text: str) -> None:
		k = self._key(model, prompt, params)
		p = self.dir / f"{k}.txt"
		try:
			p.write_text(text, encoding="utf-8")
		except Exception:
			return


#!/usr/bin/env python3
from __future__ import annotations
import logging

import json
from pathlib import Path
from typing import Dict, Any, List

def main() -> int:
    root = Path(__file__).resolve().parents[2]
    out = root / "data" / "sandbox" / "tmp" / "feats_mbpp.json"
    feats: List[Dict[str, Any]] = []
    try:
        from agi_dw.scripts.bench.mbpp_dataset import MBPPDataset  # type: ignore
        ds = MBPPDataset()
        for row in ds.get_dataset():  # type: ignore[attr-defined]
            try:
                task_id = str(row.get("task_id") or row.get("id") or "")
                starter = str(row.get("starter_code", ""))
                fn_name = None
                for ln in starter.splitlines():
                    ln = ln.strip()
                    if ln.startswith("def ") and "(" in ln:
                        fn_name = ln.split("def ", 1)[1].split("(", 1)[0].strip()
                        break
                feats.append({"task_id": task_id, "function": fn_name, "context": starter[:400]})
            except Exception:
                continue
    except Exception:
        feats = []
    out.parent.mkdir(parents=True, exist_ok=True)
    out.write_text(json.dumps({"ok": True, "n": len(feats), "items": feats}, ensure_ascii=False, indent=2), encoding="utf-8")
    print(json.dumps({"ok": True, "out": str(out), "n": len(feats)}))
    return 0

if __name__ == "__main__":
    raise SystemExit(main())


from __future__ import annotations

import argparse
import importlib
import json
import os
from pathlib import Path
from typing import Any, Dict

def parse_args():
    ap = argparse.ArgumentParser(description="Generic stub bench runner (placeholder)")
    root = Path(__file__).resolve().parents[2]
    ap.add_argument("--suite", required=True, help="Suite key, e.g., dycodeeval, ppm, mconala")
    ap.add_argument("--out", required=False, default=None)
    return ap.parse_args()

def _sharded_out_path(out_path: Path) -> Path:
    raw = str(out_path)
    try:
        ws = max(1, int(os.environ.get("WORLD_SIZE", "1") or 1))
        rk = max(0, int(os.environ.get("RANK", "0") or 0))
        if ws > 1:
            p = out_path
            raw = str(p.with_name(p.stem + f".shard{rk}-of-{ws}" + p.suffix))
    except Exception:
        pass
    p = Path(raw)
    p.parent.mkdir(parents=True, exist_ok=True)
    return p

def _try_run_plugin(suite: str, out_path: Path) -> bool:
    """Attempt to import and run a plugin for this suite.

    Plugin module path: agi_dw.scripts.bench.plugins.<suite>
    Expected function: run(out_path: Path) -> int
    Returns True if plugin executed, False if unavailable.
    """
    mod_name = f"agi_dw.scripts.bench.plugins.{suite}"
    try:
        mod = importlib.import_module(mod_name)
        run_fn = getattr(mod, "run", None)
        if callable(run_fn):
            rc = int(run_fn(out_path))
            # Consider any execution (even non-zero) as handled
            return True
    except Exception:
        return False
    return False

def main() -> int:
    args = parse_args()
    root = Path(__file__).resolve().parents[2]
    default_out = root / "data" / "bench" / "results" / f"{args.suite}.jsonl"
    outp = _sharded_out_path(Path(args.out) if args.out else default_out)

    # Try plugin first
    if _try_run_plugin(args.suite, outp):
        print(json.dumps({"ok": True, "out": str(outp), "suite": args.suite, "note": "plugin"}))
        return 0

    # Fallback to stub row
    try:
        with outp.open("w", encoding="utf-8") as f:
            f.write(json.dumps({"suite": args.suite, "status": "not_implemented"}) + "\n")
        print(json.dumps({"ok": True, "out": str(outp), "suite": args.suite, "note": "stub"}))
        return 0
    except Exception as e:
        print(json.dumps({"ok": False, "error": str(e)}))
        return 2

if __name__ == "__main__":
    raise SystemExit(main())


from __future__ import annotations

import argparse
import importlib
import json
import os
from pathlib import Path
from typing import Any, Dict

def parse_args():
    ap = argparse.ArgumentParser(description="Generic stub bench runner (placeholder)")
    root = Path(__file__).resolve().parents[2]
    ap.add_argument("--suite", required=True, help="Suite key, e.g., dycodeeval, ppm, mconala")
    ap.add_argument("--out", required=False, default=None)
    return ap.parse_args()

def _sharded_out_path(out_path: Path) -> Path:
    raw = str(out_path)
    try:
        ws = max(1, int(os.environ.get("WORLD_SIZE", "1") or 1))
        rk = max(0, int(os.environ.get("RANK", "0") or 0))
        if ws > 1:
            p = out_path
            raw = str(p.with_name(p.stem + f".shard{rk}-of-{ws}" + p.suffix))
    except Exception:
        pass
    p = Path(raw)
    p.parent.mkdir(parents=True, exist_ok=True)
    return p

def _try_run_plugin(suite: str, out_path: Path) -> bool:
    """Attempt to import and run a plugin for this suite.


from __future__ import annotations

import argparse
import importlib
import json
import os
from pathlib import Path
from typing import Any, Dict

def parse_args():
    ap = argparse.ArgumentParser(description="Generic stub bench runner (placeholder)")
    root = Path(__file__).resolve().parents[2]
    ap.add_argument("--suite", required=True, help="Suite key, e.g., dycodeeval, ppm, mconala")
    ap.add_argument("--out", required=False, default=None)
    return ap.parse_args()

def _sharded_out_path(out_path: Path) -> Path:
    raw = str(out_path)
    try:
        ws = max(1, int(os.environ.get("WORLD_SIZE", "1") or 1))
        rk = max(0, int(os.environ.get("RANK", "0") or 0))
        if ws > 1:
            p = out_path
            raw = str(p.with_name(p.stem + f".shard{rk}-of-{ws}" + p.suffix))
    except Exception:
        pass
    p = Path(raw)
    p.parent.mkdir(parents=True, exist_ok=True)
    return p

def _try_run_plugin(suite: str, out_path: Path) -> bool:
    """Attempt to import and run a plugin for this suite.

    Plugin module path: agi_dw.scripts.bench.plugins.<suite>
    Expected function: run(out_path: Path) -> int
    Returns True if plugin executed, False if unavailable.
    """
    mod_name = f"agi_dw.scripts.bench.plugins.{suite}"
    try:
        mod = importlib.import_module(mod_name)
        run_fn = getattr(mod, "run", None)
        if callable(run_fn):
            rc = int(run_fn(out_path))
            # Consider any execution (even non-zero) as handled
            return True
    except Exception:
        return False
    return False


    ap.add_argument("--suite", required=True, help="Suite key, e.g., dycodeeval, ppm, mconala")
    ap.add_argument("--out", required=False, default=None)
    return ap.parse_args()

def _sharded_out_path(out_path: Path) -> Path:
    raw = str(out_path)
    try:
        ws = max(1, int(os.environ.get("WORLD_SIZE", "1") or 1))
        rk = max(0, int(os.environ.get("RANK", "0") or 0))
        if ws > 1:
            p = out_path
            raw = str(p.with_name(p.stem + f".shard{rk}-of-{ws}" + p.suffix))
    except Exception:
        pass
    p = Path(raw)
    p.parent.mkdir(parents=True, exist_ok=True)
    return p

def _try_run_plugin(suite: str, out_path: Path) -> bool:
    """Attempt to import and run a plugin for this suite.

    Plugin module path: agi_dw.scripts.bench.plugins.<suite>
    Expected function: run(out_path: Path) -> int
    Returns True if plugin executed, False if unavailable.
    """
    mod_name = f"agi_dw.scripts.bench.plugins.{suite}"
    try:
        mod = importlib.import_module(mod_name)
        run_fn = getattr(mod, "run", None)
        if callable(run_fn):
            rc = int(run_fn(out_path))
            # Consider any execution (even non-zero) as handled
            return True
    except Exception:
        return False
    return False

def main() -> int:
    args = parse_args()
    root = Path(__file__).resolve().parents[2]
    default_out = root / "data" / "bench" / "results" / f"{args.suite}.jsonl"
    outp = _sharded_out_path(Path(args.out) if args.out else default_out)

    # Try plugin first
    if _try_run_plugin(args.suite, outp):
        print(json.dumps({"ok": True, "out": str(outp), "suite": args.suite, "note": "plugin"}))
        return 0

    # Fallback to stub row
    try:
        with outp.open("w", encoding="utf-8") as f:
            f.write(json.dumps({"suite": args.suite, "status": "not_implemented"}) + "\n")
        print(json.dumps({"ok": True, "out": str(outp), "suite": args.suite, "note": "stub"}))
        return 0
    except Exception as e:


main

def _try_run_plugin(suite: str, out_path: Path) -> bool:
    """Attempt to import and run a plugin for this suite.

    Plugin module path: agi_dw.scripts.bench.plugins.<suite>
    Expected function: run(out_path: Path) -> int
    Returns True if plugin executed, False if unavailable.
    """
    mod_name = f"agi_dw.scripts.bench.plugins.{suite}"
    try:
        mod = importlib.import_module(mod_name)
        run_fn = getattr(mod, "run", None)
        if callable(run_fn):
            rc = int(run_fn(out_path))
            # Consider any execution (even non-zero) as handled
            return True
    except Exception:
        return False
    return False

def main() -> int:
    args = parse_args()
    root = Path(__file__).resolve().parents[2]
    default_out = root / "data" / "bench" / "results" / f"{args.suite}.jsonl"
    outp = _sharded_out_path(Path(args.out) if args.out else default_out)

    # Try plugin first
    if _try_run_plugin(args.suite, outp):
        print(json.dumps({"ok": True, "out": str(outp), "suite": args.suite, "note": "plugin"}))
        return 0

    # Fallback to stub row
    try:
        with outp.open("w", encoding="utf-8") as f:
            f.write(json.dumps({"suite": args.suite, "status": "not_implemented"}) + "\n")
        print(json.dumps({"ok": True, "out": str(outp), "suite": args.suite, "note": "stub"}))
        return 0
    except Exception as e:
        print(json.dumps({"ok": False, "error": str(e)}))
        return 2

if __name__ == "__main__":
    raise SystemExit(main())


import logging
import argparse
from pathlib import Path
from typing import List, Dict
import json

from agi_dw.core.llm.hf_client import HFClient
from agi_dw.core.utils.critic import get_critic

def main() -> int:
	ap = argparse.ArgumentParser()
	root = Path(__file__).resolve().parents[1]
	ap.add_argument("--model", default="meta-llama/Llama-3.2-3B")
	ap.add_argument("--diff", default=str(root / "data" / "traces" / "last_diff.patch"))
	ap.add_argument("--lint", default=str(root / "data" / "traces" / "last_lints.txt"))
	args = ap.parse_args()

	diff = Path(args.diff).read_text(encoding="utf-8") if Path(args.diff).exists() else ""
	lints = Path(args.lint).read_text(encoding="utf-8") if Path(args.lint).exists() else ""
	critic = get_critic(args.model)
	ok, feedback = critic.review(diff, lints)
	print(json.dumps({"ok": ok, "feedback": feedback}))
	return 0

if __name__ == "__main__":
	raise SystemExit(main())


from __future__ import annotations

import argparse
import concurrent.futures as _cf
import json
import math
import os
import shutil
import subprocess
import sys
from pathlib import Path
from typing import List, Tuple

def _detect_gpu_count() -> int:
    # Prefer CUDA_VISIBLE_DEVICES if set
    cvd = os.environ.get("CUDA_VISIBLE_DEVICES")
    if cvd is not None:
        try:
            toks = [t for t in cvd.split(",") if t.strip()]
            return max(0, len(toks))
        except Exception:
            pass
    # Fallback: nvidia-smi
    if shutil.which("nvidia-smi"):
        try:
            out = subprocess.check_output(["nvidia-smi", "--query-gpu=name", "--format=csv,noheader"], stderr=subprocess.DEVNULL)
            lines = [l for l in out.decode("utf-8", errors="ignore").splitlines() if l.strip()]
            return max(0, len(lines))
        except Exception:
            return 0
    return 0

def _parallel_workers(auto: bool, requested: int | None, n_suites: int) -> int:
    if not auto and isinstance(requested, int) and requested > 0:
        return min(requested, max(1, n_suites))
    # Auto mode: prefer GPU count, else CPU count; cap by number of suites
    gpus = _detect_gpu_count()
    if gpus > 0:
        return min(n_suites, gpus)
    cpus = os.cpu_count() or 1
    # leave some headroom
    return min(n_suites, max(1, math.floor(cpus)))

def _run_suite(cmd: List[str]) -> Tuple[str, int]:
    suite = "unknown"
    try:
        # extract suite name from args
        for i, t in enumerate(cmd):
            if t == "--suite" and i + 1 < len(cmd):
                suite = cmd[i + 1]
                break
        proc = subprocess.run(cmd)
        return suite, proc.returncode
    except Exception:
        return suite, 1

def main(argv: List[str] | None = None) -> int:
	ap = argparse.ArgumentParser(description="Run all benchmarks defined in bench/registry.json via the registry harness")
	root = Path(__file__).resolve().parents[2]
    ap.add_argument("--results-dir", default=str(root / "data" / "bench" / "results"))
	ap.add_argument("--include", nargs="*", default=None, help="Optional suite names to include (default: all)")
	ap.add_argument("--exclude", nargs="*", default=None, help="Optional suite names to exclude")
	# Pass-through extra args for each suite run (e.g., --limit 5 --k 1)
    ap.add_argument("--parallel", default="auto", help="Number of suites to run concurrently (int) or 'auto' for GPU/CPU-based")
    args, extra = ap.parse_known_args(argv)

	from agi_dw.bench.common.registry import load_registry  # type: ignore
	reg = load_registry(root)
	suites = sorted((reg.get("suites") or {}).keys())
	if args.include:
		suites = [s for s in suites if s in set(args.include)]
	if args.exclude:
		blocked = set(args.exclude)
		suites = [s for s in suites if s not in blocked]
	results_dir = Path(args.results_dir)
	results_dir.mkdir(parents=True, exist_ok=True)

	ok_all = True
	# Compute concurrency
	par_auto = str(args.parallel).strip().lower() == "auto"
	try:
		par_req = None if par_auto else int(args.parallel)
	except Exception:
		par_req = None
	max_workers = _parallel_workers(par_auto, par_req, len(suites))

	# Ensure a model is provided for code_body suites: use env default or fallback
	default_model = os.environ.get("AGI_DEFAULT_MODEL", "meta-llama/Llama-3.2-3B")
	def _ensure_model_arg(xtra: list[str]) -> list[str]:
		has_flag = False

		for i, tok in enumerate(xtra):
			if tok == "--model":
				has_flag = True
				break
			if isinstance(tok, str) and tok.startswith("--model="):
				has_flag = True
				break
		if not has_flag and default_model:
			return xtra + ["--model", default_model]
		return xtra
    # Dispatch in parallel
    def _build_cmd(suite: str) -> List[str]:
        out = results_dir / f"{suite}.jsonl"
        cmd = [
            "python",
            str(root / "scripts" / "bench" / "run_registry.py"),
            "--suite",
            suite,
            "--out",
            str(out),
        ]
        cmd.extend(_ensure_model_arg(list(extra)))
        return cmd

    print(json.dumps({"parallel": max_workers, "n_suites": len(suites)}))
    with _cf.ThreadPoolExecutor(max_workers=max_workers) as ex:
        futs = {ex.submit(_run_suite, _build_cmd(s)): s for s in suites}
        for fut in _cf.as_completed(futs):
            s = futs[fut]
            suite, rc = fut.result()
            if rc != 0:
                ok_all = False
                print(json.dumps({"ok": False, "suite": suite, "error": "runner_nonzero_exit", "returncode": rc}))

    print(json.dumps({"ok": ok_all, "n_suites": len(suites), "parallel": max_workers}))
	return 0 if ok_all else 2

if __name__ == "__main__":
	raise SystemExit(main())


#!/usr/bin/env python3
from __future__ import annotations
import logging

import json
import re
from pathlib import Path
from typing import Any, Dict, List

def simple_target_files_from_title(title: str, repo_dir: Path) -> List[str]:
    """Heuristic targeted file selection based on title keywords.

    Picks files whose names contain any keyword tokens (case-insensitive),
    limited to Python files for safety in this scaffold.
    """
    tokens = [t for t in re.split(r"\W+", title.lower()) if t and len(t) > 2][:6]
    matched: List[str] = []
    try:
        for p in repo_dir.rglob("*.py"):
            name = p.name.lower()
            if any(tok in name for tok in tokens):
                matched.append(str(p.resolve()))
            if len(matched) >= 50:
                break
    except Exception:
        matched = []
    return matched

def main() -> int:
    root = Path(__file__).resolve().parents[2]
    cache_root = root / "data" / "bench" / "swe_lite"
    cache_root.mkdir(parents=True, exist_ok=True)

    # Attempt to load lite instances via official loader; fallback to stub
    instances: List[Dict[str, Any]] = []
    try:
        from swebench.harness.utils import load_swebench_dataset  # type: ignore
        instances = load_swebench_dataset("SWE-bench/SWE-bench_Lite", split="test")  # type: ignore
    except Exception:
        instances = []

    selected: List[Dict[str, Any]] = []
    for it in instances:
        try:
            iid = str(it.get("instance_id", ""))
            title = str(it.get("title", ""))
            repo = str(it.get("repo", ""))
            # In this scaffold we do not clone; just write a placeholder and pick targets heuristically under a local mirror if present
            repo_dir = root / "data" / "bench" / "repos" / repo.replace("/", "_")
            targets = simple_target_files_from_title(title, repo_dir) if repo_dir.exists() else []
            selected.append({"instance_id": iid, "repo": repo, "targets": targets, "title": title})
        except Exception:
            continue

    out_meta = cache_root / "retrieval.json"
    out_meta.write_text(json.dumps({"ok": True, "items": selected}, ensure_ascii=False, indent=2), encoding="utf-8")
    print(json.dumps({"ok": True, "out": str(out_meta), "n": len(selected)}))
    return 0

if __name__ == "__main__":
    raise SystemExit(main())


#!/usr/bin/env python3
from __future__ import annotations
import logging

import json
import re
from pathlib import Path
from typing import Any, Dict, List

def simple_target_files_from_title(title: str, repo_dir: Path) -> List[str]:
    """Heuristic targeted file selection based on title keywords.

    Picks files whose names contain any keyword tokens (case-insensitive),
    limited to Python files for safety in this scaffold.
    """
    tokens = [t for t in re.split(r"\W+", title.lower()) if t and len(t) > 2][:6]
    matched: List[str] = []
    try:
        for p in repo_dir.rglob("*.py"):
            name = p.name.lower()
            if any(tok in name for tok in tokens):
                matched.append(str(p.resolve()))
            if len(matched) >= 50:
                break
    except Exception:
        matched = []
    return matched

def main() -> int:
    root = Path(__file__).resolve().parents[2]
    cache_root = root / "data" / "bench" / "swe_lite"
    cache_root.mkdir(parents=True, exist_ok=True)

    # Attempt to load lite instances via official loader; fallback to stub
    instances: List[Dict[str, Any]] = []
    try:
        from swebench.harness.utils import load_swebench_dataset  # type: ignore
        instances = load_swebench_dataset("SWE-bench/SWE-bench_Lite", split="test")  # type: ignore
    except Exception:
        instances = []

    selected: List[Dict[str, Any]] = []
    for it in instances:
        try:
            iid = str(it.get("instance_id", ""))
            title = str(it.get("title", ""))


def simple_target_files_from_title(title: str, repo_dir: Path) -> List[str]:
    """Heuristic targeted file selection based on title keywords.

    Picks files whose names contain any keyword tokens (case-insensitive),
    limited to Python files for safety in this scaffold.
    """
    tokens = [t for t in re.split(r"\W+", title.lower()) if t and len(t) > 2][:6]
    matched: List[str] = []
    try:
        for p in repo_dir.rglob("*.py"):
            name = p.name.lower()
            if any(tok in name for tok in tokens):
                matched.append(str(p.resolve()))
            if len(matched) >= 50:
                break
    except Exception:
        matched = []
    return matched

def main() -> int:
    root = Path(__file__).resolve().parents[2]
    cache_root = root / "data" / "bench" / "swe_lite"
    cache_root.mkdir(parents=True, exist_ok=True)

    # Attempt to load lite instances via official loader; fallback to stub
    instances: List[Dict[str, Any]] = []
    try:
        from swebench.harness.utils import load_swebench_dataset  # type: ignore
        instances = load_swebench_dataset("SWE-bench/SWE-bench_Lite", split="test")  # type: ignore
    except Exception:
        instances = []

    selected: List[Dict[str, Any]] = []
    for it in instances:
        try:
            iid = str(it.get("instance_id", ""))
            title = str(it.get("title", ""))
            repo = str(it.get("repo", ""))
            # In this scaffold we do not clone; just write a placeholder and pick targets heuristically under a local mirror if present
            repo_dir = root / "data" / "bench" / "repos" / repo.replace("/", "_")
            targets = simple_target_files_from_title(title, repo_dir) if repo_dir.exists() else []
            selected.append({"instance_id": iid, "repo": repo, "targets": targets, "title": title})
        except Exception:
            continue

    out_meta = cache_root / "retrieval.json"
    out_meta.write_text(json.dumps({"ok": True, "items": selected}, ensure_ascii=False, indent=2), encoding="utf-8")
    print(json.dumps({"ok": True, "out": str(out_meta), "n": len(selected)}))
    return 0

if __name__ == "__main__":
    raise SystemExit(main())


from __future__ import annotations

import argparse
import os
import json
from pathlib import Path

def parse_args():
	ap = argparse.ArgumentParser(description="Unified bench runner")
	root = Path(__file__).resolve().parents[2]
	ap.add_argument("--suite", required=True, choices=["humaneval", "mbpp", "apps", "swebench_lite", "swebench"])
	ap.add_argument("--out", default=None)
	# Parse known args only; keep unknown args for forwarding
	args, remainder = ap.parse_known_args()
	setattr(args, "remainder", remainder)
	return args

def main() -> int:
	args = parse_args()
	# Build a simple shim namespace for the base runner functions
	# We forward only known flags (out is optional) and let each base runner parse the rest via its own argparse in scripts if needed.
	try:
		from agi_dw.bench.common.base_runner import run_registry_benchmark  # type: ignore
	except Exception as e:
		print(json.dumps({"ok": False, "error": f"base_runner import failed: {e}"}))
		return 2

	# Minimal args shim: we pass through known flags via setattr
	# We forward only known flags (out is optional) and let each base runner parse the rest via its own argparse in scripts if needed.
	class _Shim(object):
		pass

	shim = _Shim()
	# Suite-specific default outs if not provided
	root = Path(__file__).resolve().parents[2]
	default_outs = {
		"humaneval": root / "data" / "bench" / "results" / "humaneval.jsonl",
		"mbpp": root / "data" / "bench" / "results" / "mbpp.jsonl",
		"apps": root / "data" / "bench" / "results" / "apps.jsonl",
		"swebench_lite": root / "data" / "bench" / "results" / "swebench_lite.jsonl",
		"swebench": root / "data" / "bench" / "results" / "swebench.jsonl",
	}
	setattr(shim, "out", str(Path(args.out)) if args.out else str(default_outs[args.suite]))
	# Default model per suite if not provided via ARGS, overridable via env
	env_default = os.environ.get("AGI_DEFAULT_MODEL", "").strip() or None
	default_models = {
		"humaneval": env_default or "meta-llama/Llama-3.2-3B",
		"mbpp": env_default or "meta-llama/Llama-3.2-3B",
		"apps": env_default or "meta-llama/Llama-3.2-3B",
		"swebench_lite": env_default or "meta-llama/Llama-3.2-3B",
		"swebench": None,
	}
	if getattr(shim, "model", None) is None and default_models[args.suite]:
		setattr(shim, "model", default_models[args.suite])

	# Forward extra ARGS from the command line to shim attributes
	# Supports: --flag value, --flag=value, and boolean flags (--flag)
	def _coerce(val: str):
		low = str(val).strip().lower()
		if low in ("true", "false"):  # booleans
			return low == "true"
		try:
			return int(val)
		except Exception:
			pass
		try:
			return float(val)
		except Exception:
			pass
		return val

	rem = list(getattr(args, "remainder", []) or [])
	i = 0
	while i < len(rem):
		tok = rem[i]
		if isinstance(tok, str) and tok.startswith("--"):
			key = tok.lstrip("-").strip()
			val = True
			if "=" in key:
				k, v = key.split("=", 1)
				key, val = k, _coerce(v)
			elif i + 1 < len(rem) and not str(rem[i + 1]).startswith("-"):
				val = _coerce(rem[i + 1])
				i += 1
			# Argparse turns dashes into underscores for attr names in our runners
			attr = key.replace("-", "_")
			setattr(shim, attr, val)
		i += 1

	# Unified registry-driven path
	setattr(shim, "suite", args.suite)
	return int(run_registry_benchmark(shim))

if __name__ == "__main__":
	raise SystemExit(main())


from __future__ import annotations

import argparse
import os
import json
from pathlib import Path

def parse_args():
	ap = argparse.ArgumentParser(description="Unified bench runner")
	root = Path(__file__).resolve().parents[2]
	ap.add_argument("--suite", required=True, choices=["humaneval", "mbpp", "apps", "swebench_lite", "swebench"])
	ap.add_argument("--out", default=None)
	# Parse known args only; keep unknown args for forwarding
	args, remainder = ap.parse_known_args()
	setattr(args, "remainder", remainder)
	return args

def main() -> int:
	args = parse_args()
	# Build a simple shim namespace for the base runner functions
	# We forward only known flags (out is optional) and let each base runner parse the rest via its own argparse in scripts if needed.
	try:
		from agi_dw.bench.common.base_runner import run_registry_benchmark  # type: ignore
	except Exception as e:
		print(json.dumps({"ok": False, "error": f"base_runner import failed: {e}"}))
		return 2

	# Minimal args shim: we pass through known flags via setattr
	# We forward only known flags (out is optional) and let each base runner parse the rest via its own argparse in scripts if needed.
	class _Shim(object):
		pass

	shim = _Shim()
	# Suite-specific default outs if not provided
	root = Path(__file__).resolve().parents[2]


	ap.add_argument("--suite", required=True, choices=["humaneval", "mbpp", "apps", "swebench_lite", "swebench"])
	ap.add_argument("--out", default=None)
	# Parse known args only; keep unknown args for forwarding
	args, remainder = ap.parse_known_args()
	setattr(args, "remainder", remainder)
	return args

def main() -> int:
	args = parse_args()
	# Build a simple shim namespace for the base runner functions
	# We forward only known flags (out is optional) and let each base runner parse the rest via its own argparse in scripts if needed.
	try:
		from agi_dw.bench.common.base_runner import run_registry_benchmark  # type: ignore
	except Exception as e:
		print(json.dumps({"ok": False, "error": f"base_runner import failed: {e}"}))
		return 2

	# Minimal args shim: we pass through known flags via setattr
	# We forward only known flags (out is optional) and let each base runner parse the rest via its own argparse in scripts if needed.
	class _Shim(object):
		pass

	shim = _Shim()
	# Suite-specific default outs if not provided
	root = Path(__file__).resolve().parents[2]
	default_outs = {
		"humaneval": root / "data" / "bench" / "results" / "humaneval.jsonl",
		"mbpp": root / "data" / "bench" / "results" / "mbpp.jsonl",
		"apps": root / "data" / "bench" / "results" / "apps.jsonl",
		"swebench_lite": root / "data" / "bench" / "results" / "swebench_lite.jsonl",
		"swebench": root / "data" / "bench" / "results" / "swebench.jsonl",
	}
	setattr(shim, "out", str(Path(args.out)) if args.out else str(default_outs[args.suite]))
	# Default model per suite if not provided via ARGS, overridable via env
	env_default = os.environ.get("AGI_DEFAULT_MODEL", "").strip() or None
	default_models = {
		"humaneval": env_default or "meta-llama/Llama-3.2-3B",
		"mbpp": env_default or "meta-llama/Llama-3.2-3B",
		"apps": env_default or "meta-llama/Llama-3.2-3B",
		"swebench_lite": env_default or "meta-llama/Llama-3.2-3B",
		"swebench": None,


		"mbpp": root / "data" / "bench" / "results" / "mbpp.jsonl",
		"apps": root / "data" / "bench" / "results" / "apps.jsonl",
		"swebench_lite": root / "data" / "bench" / "results" / "swebench_lite.jsonl",
		"swebench": root / "data" / "bench" / "results" / "swebench.jsonl",
	}
	setattr(shim, "out", str(Path(args.out)) if args.out else str(default_outs[args.suite]))
	# Default model per suite if not provided via ARGS, overridable via env
	env_default = os.environ.get("AGI_DEFAULT_MODEL", "").strip() or None
	default_models = {
		"humaneval": env_default or "meta-llama/Llama-3.2-3B",
		"mbpp": env_default or "meta-llama/Llama-3.2-3B",
		"apps": env_default or "meta-llama/Llama-3.2-3B",
		"swebench_lite": env_default or "meta-llama/Llama-3.2-3B",
		"swebench": None,
	}
	if getattr(shim, "model", None) is None and default_models[args.suite]:
		setattr(shim, "model", default_models[args.suite])

	# Forward extra ARGS from the command line to shim attributes
	# Supports: --flag value, --flag=value, and boolean flags (--flag)
	def _coerce(val: str):
		low = str(val).strip().lower()
		if low in ("true", "false"):  # booleans
			return low == "true"
		try:
			return int(val)
		except Exception:
			pass
		try:
			return float(val)
		except Exception:
			pass
		return val

	rem = list(getattr(args, "remainder", []) or [])
	i = 0
	while i < len(rem):
		tok = rem[i]
		if isinstance(tok, str) and tok.startswith("--"):
			key = tok.lstrip("-").strip()
			val = True
			if "=" in key:
				k, v = key.split("=", 1)
				key, val = k, _coerce(v)
			elif i + 1 < len(rem) and not str(rem[i + 1]).startswith("-"):
				val = _coerce(rem[i + 1])
				i += 1
			# Argparse turns dashes into underscores for attr names in our runners
			attr = key.replace("-", "_")
			setattr(shim, attr, val)
		i += 1

	# Unified registry-driven path


from __future__ import annotations
import logging

import argparse
import tarfile
from pathlib import Path

def main() -> int:
	ap = argparse.ArgumentParser(description="Export benchmark results and dashboard to a tar.gz bundle")
	root = Path(__file__).resolve().parents[2]
	ap.add_argument("--results", default=str(root / "data" / "bench" / "results"))
	ap.add_argument("--dashboard", default=str(root / "data" / "dashboards" / "benchmarks.json"))
	ap.add_argument("--out", default=str(root / "data" / "bench" / "export.tgz"))
	args = ap.parse_args()

	res_dir = Path(args.results)
	dash = Path(args.dashboard)
	out = Path(args.out)
	out.parent.mkdir(parents=True, exist_ok=True)
	with tarfile.open(out, "w:gz") as tar:
		if res_dir.exists():
			tar.add(res_dir, arcname="results")
		if dash.exists():
			tar.add(dash, arcname="benchmarks.json")
	print(str(out))
	return 0

if __name__ == "__main__":
	raise SystemExit(main())


from __future__ import annotations
import logging

import argparse
import json
import os
from pathlib import Path
from typing import Dict, Any

def load_json(path: str | os.PathLike[str]) -> Dict[str, Any]:
	p = Path(path)
	if not p.exists():
		return {}
	try:
		return json.loads(p.read_text(encoding="utf-8"))
	except Exception:
		return {}

def main() -> int:
	ap = argparse.ArgumentParser(description="Assert benchmark acceptance thresholds")
	root = Path(__file__).resolve().parents[2]
	ap.add_argument("--dashboard", default=str(root / "data" / "dashboards" / "benchmarks.json"))
	ap.add_argument("--thresholds", default=str(root / "data" / "bench" / "acceptance_thresholds.json"))
	ap.add_argument("--suites", default="humaneval,mbpp,swebench_lite", help="Comma-separated suites to enforce")
	# Allow overrides via env or CLI
	ap.add_argument("--min-pass1", type=float, help="Override min pass@1 for a single suite usage")
	ap.add_argument("--max-p90", type=float, help="Override max p90 for a single suite usage")
	args = ap.parse_args()

	dashboard = load_json(str(args.dashboard))
	thresholds = load_json(str(args.thresholds))
	summary: Dict[str, Dict[str, float]] = dashboard.get("summary", {}) or {}

	requested = [s.strip() for s in str(getattr(args, "suites", "")).split(",") if s.strip()]
	if not requested:
		# default to all found in summary
		requested = list(summary.keys())

	ok_all = True
	report: Dict[str, Any] = {"ok": False, "details": []}

	for suite in requested:
		met = summary.get(suite, {}) or {}
		p1 = float(met.get("pass1", 0.0))
		p90 = float(met.get("p90", 0.0))

		th = thresholds.get(suite, {}) if isinstance(thresholds, dict) else {}
		min_p1 = float(os.environ.get("MIN_PASS1", th.get("min_pass1", 0.0)) if os.environ.get("SUITE", "") in (suite, "") else th.get("min_pass1", 0.0))
		max_p90 = float(os.environ.get("MAX_P90", th.get("max_p90", 9999.0)) if os.environ.get("SUITE", "") in (suite, "") else th.get("max_p90", 9999.0))

		# CLI overrides if provided and single-suite context
		if getattr(args, "min_pass1", None) is not None:
			min_p1 = float(args.min_pass1)
		if getattr(args, "max_p90", None) is not None:
			max_p90 = float(args.max_p90)

		ok = (p1 >= min_p1) and (p90 <= max_p90)
		ok_all = ok_all and ok
		report["details"].append(
			{
				"suite": suite,
				"pass1": p1,
				"p90": p90,
				"min_pass1": min_p1,
				"max_p90": max_p90,
				"ok": ok,
			}
		)

	report["ok"] = bool(ok_all)
	print(json.dumps(report))
	return 0 if ok_all else 2

if __name__ == "__main__":
	raise SystemExit(main())


from __future__ import annotations
import logging

import argparse
import json
import os
from pathlib import Path
from typing import Dict, Any

def load_json(path: str | os.PathLike[str]) -> Dict[str, Any]:
	p = Path(path)
	if not p.exists():
		return {}
	try:
		return json.loads(p.read_text(encoding="utf-8"))
	except Exception:
		return {}

def main() -> int:
	ap = argparse.ArgumentParser(description="Assert benchmark acceptance thresholds")
	root = Path(__file__).resolve().parents[2]
	ap.add_argument("--dashboard", default=str(root / "data" / "dashboards" / "benchmarks.json"))
	ap.add_argument("--thresholds", default=str(root / "data" / "bench" / "acceptance_thresholds.json"))
	ap.add_argument("--suites", default="humaneval,mbpp,swebench_lite", help="Comma-separated suites to enforce")
	# Allow overrides via env or CLI
	ap.add_argument("--min-pass1", type=float, help="Override min pass@1 for a single suite usage")
	ap.add_argument("--max-p90", type=float, help="Override max p90 for a single suite usage")
	args = ap.parse_args()

	dashboard = load_json(str(args.dashboard))
	thresholds = load_json(str(args.thresholds))
	summary: Dict[str, Dict[str, float]] = dashboard.get("summary", {}) or {}

	requested = [s.strip() for s in str(getattr(args, "suites", "")).split(",") if s.strip()]
	if not requested:
		# default to all found in summary


from __future__ import annotations

import argparse
import json
from pathlib import Path

def parse_args():
	ap = argparse.ArgumentParser(description="Run a benchmark via the registry")
	root = Path(__file__).resolve().parents[2]
	ap.add_argument("--suite", required=True)
	ap.add_argument("--out", default=None)
	args, remainder = ap.parse_known_args()
	setattr(args, "remainder", remainder)
	# Supply default out if missing
	if args.out is None:
		defaults = {
			"humaneval": root / "data" / "bench" / "results" / "humaneval.jsonl",
			"mbpp": root / "data" / "bench" / "results" / "mbpp.jsonl",
			"apps": root / "data" / "bench" / "results" / "apps.jsonl",
			"swebench_lite": root / "data" / "bench" / "results" / "swebench_lite.jsonl",
		}
		setattr(args, "out", str(defaults.get(args.suite, root / "data" / "bench" / "results" / f"{args.suite}.jsonl")))
	return args

def main() -> int:
	args = parse_args()
	try:
		from agi_dw.bench.common.base_runner import run_registry_benchmark  # type: ignore
		return int(run_registry_benchmark(args))
	except Exception as e:
		print(json.dumps({"ok": False, "error": f"registry_runner_failed: {e}"}))
		return 2

if __name__ == "__main__":
	raise SystemExit(main())


def parse_args():
	ap = argparse.ArgumentParser(description="Run a benchmark via the registry")
	root = Path(__file__).resolve().parents[2]
	ap.add_argument("--suite", required=True)
	ap.add_argument("--out", default=None)
	args, remainder = ap.parse_known_args()
	setattr(args, "remainder", remainder)
	# Supply default out if missing
	if args.out is None:
		defaults = {
			"humaneval": root / "data" / "bench" / "results" / "humaneval.jsonl",
			"mbpp": root / "data" / "bench" / "results" / "mbpp.jsonl",
			"apps": root / "data" / "bench" / "results" / "apps.jsonl",
			"swebench_lite": root / "data" / "bench" / "results" / "swebench_lite.jsonl",
		}
		setattr(args, "out", str(defaults.get(args.suite, root / "data" / "bench" / "results" / f"{args.suite}.jsonl")))
	return args

def main() -> int:
	args = parse_args()
	try:
		from agi_dw.bench.common.base_runner import run_registry_benchmark  # type: ignore
		return int(run_registry_benchmark(args))
	except Exception as e:
		print(json.dumps({"ok": False, "error": f"registry_runner_failed: {e}"}))
		return 2

if __name__ == "__main__":
	raise SystemExit(main())


from __future__ import annotations

import argparse
import json
import os
from pathlib import Path

def parse_args():
    ap = argparse.ArgumentParser(description="CodeEval-Pro wrapper (HE/MBPP Pro)")
    root = Path(__file__).resolve().parents[2]
    ap.add_argument("--suite", required=True, choices=["humaneval_pro", "mbpp_pro"])
    ap.add_argument("--out", default=None)
    ap.add_argument("--model", default=os.environ.get("AGI_DEFAULT_MODEL", "meta-llama/Llama-3.2-3B"))
    return ap.parse_args()

def _default_out(root: Path, suite: str) -> Path:
    name = "humaneval_pro" if suite == "humaneval_pro" else "mbpp_pro"
    return root / "data" / "bench" / "results" / f"{name}.jsonl"

def _sharded_path(p: Path) -> Path:
    raw = str(p)
    try:
        ws = max(1, int(os.environ.get("WORLD_SIZE", "1") or 1))
        rk = max(0, int(os.environ.get("RANK", "0") or 0))
        if ws > 1:
            q = p.with_name(p.stem + f".shard{rk}-of-{ws}" + p.suffix)
            q.parent.mkdir(parents=True, exist_ok=True)
            return q
    except Exception:
        pass
    p.parent.mkdir(parents=True, exist_ok=True)
    return p

def main() -> int:
    args = parse_args()
    root = Path(__file__).resolve().parents[2]
    outp = Path(args.out) if args.out else _default_out(root, args.suite)
    outp = _sharded_path(outp)

    # Try to import evalpro harness
    have = False
    try:
        import codeeval_pro  # type: ignore  # noqa: F401
        have = True
    except Exception:
        have = False

    if not have:
        outp.write_text("", encoding="utf-8")
        print(json.dumps({
            "ok": False,
            "status": "skipped",
            "reason": "CodeEval-Pro harness not installed",
            "out": str(outp),
        }))
        return 0

    with outp.open("w", encoding="utf-8") as f:
        f.write(json.dumps({"suite": args.suite, "status": "not_implemented"}) + "\n")
    print(json.dumps({"ok": True, "out": str(outp), "note": "harness stub"}))
    return 0

if __name__ == "__main__":
    raise SystemExit(main())


from __future__ import annotations

import argparse
import json
import os
from pathlib import Path

def parse_args():
    ap = argparse.ArgumentParser(description="CodeEval-Pro wrapper (HE/MBPP Pro)")
    root = Path(__file__).resolve().parents[2]
    ap.add_argument("--suite", required=True, choices=["humaneval_pro", "mbpp_pro"])
    ap.add_argument("--out", default=None)
    ap.add_argument("--model", default=os.environ.get("AGI_DEFAULT_MODEL", "meta-llama/Llama-3.2-3B"))
    return ap.parse_args()

def _default_out(root: Path, suite: str) -> Path:
    name = "humaneval_pro" if suite == "humaneval_pro" else "mbpp_pro"
    return root / "data" / "bench" / "results" / f"{name}.jsonl"

def _sharded_path(p: Path) -> Path:
    raw = str(p)
    try:
        ws = max(1, int(os.environ.get("WORLD_SIZE", "1") or 1))
        rk = max(0, int(os.environ.get("RANK", "0") or 0))
        if ws > 1:
            q = p.with_name(p.stem + f".shard{rk}-of-{ws}" + p.suffix)
            q.parent.mkdir(parents=True, exist_ok=True)
            return q
    except Exception:
        pass
    p.parent.mkdir(parents=True, exist_ok=True)
    return p


from __future__ import annotations

import argparse
import json
import os
from pathlib import Path

def parse_args():
    ap = argparse.ArgumentParser(description="CodeEval-Pro wrapper (HE/MBPP Pro)")
    root = Path(__file__).resolve().parents[2]
    ap.add_argument("--suite", required=True, choices=["humaneval_pro", "mbpp_pro"])
    ap.add_argument("--out", default=None)
    ap.add_argument("--model", default=os.environ.get("AGI_DEFAULT_MODEL", "meta-llama/Llama-3.2-3B"))
    return ap.parse_args()

def _default_out(root: Path, suite: str) -> Path:
    name = "humaneval_pro" if suite == "humaneval_pro" else "mbpp_pro"
    return root / "data" / "bench" / "results" / f"{name}.jsonl"

def _sharded_path(p: Path) -> Path:
    raw = str(p)
    try:
        ws = max(1, int(os.environ.get("WORLD_SIZE", "1") or 1))
        rk = max(0, int(os.environ.get("RANK", "0") or 0))
        if ws > 1:
            q = p.with_name(p.stem + f".shard{rk}-of-{ws}" + p.suffix)
            q.parent.mkdir(parents=True, exist_ok=True)
            return q
    except Exception:
        pass
    p.parent.mkdir(parents=True, exist_ok=True)
    return p

def main() -> int:
    args = parse_args()
    root = Path(__file__).resolve().parents[2]


import argparse
import json
import os
from pathlib import Path

def parse_args():
    ap = argparse.ArgumentParser(description="CodeEval-Pro wrapper (HE/MBPP Pro)")
    root = Path(__file__).resolve().parents[2]
    ap.add_argument("--suite", required=True, choices=["humaneval_pro", "mbpp_pro"])
    ap.add_argument("--out", default=None)
    ap.add_argument("--model", default=os.environ.get("AGI_DEFAULT_MODEL", "meta-llama/Llama-3.2-3B"))
    return ap.parse_args()

def _default_out(root: Path, suite: str) -> Path:
    name = "humaneval_pro" if suite == "humaneval_pro" else "mbpp_pro"
    return root / "data" / "bench" / "results" / f"{name}.jsonl"

def _sharded_path(p: Path) -> Path:
    raw = str(p)
    try:
        ws = max(1, int(os.environ.get("WORLD_SIZE", "1") or 1))
        rk = max(0, int(os.environ.get("RANK", "0") or 0))
        if ws > 1:
            q = p.with_name(p.stem + f".shard{rk}-of-{ws}" + p.suffix)
            q.parent.mkdir(parents=True, exist_ok=True)
            return q
    except Exception:
        pass
    p.parent.mkdir(parents=True, exist_ok=True)
    return p

def main() -> int:
    args = parse_args()
    root = Path(__file__).resolve().parents[2]
    outp = Path(args.out) if args.out else _default_out(root, args.suite)
    outp = _sharded_path(outp)

    # Try to import evalpro harness
    have = False
    try:
        import codeeval_pro  # type: ignore  # noqa: F401
        have = True
    except Exception:
        have = False

    if not have:
        outp.write_text("", encoding="utf-8")
        print(json.dumps({
            "ok": False,


def _default_out(root: Path, suite: str) -> Path:
    name = "humaneval_pro" if suite == "humaneval_pro" else "mbpp_pro"
    return root / "data" / "bench" / "results" / f"{name}.jsonl"

def _sharded_path(p: Path) -> Path:
    raw = str(p)
    try:
        ws = max(1, int(os.environ.get("WORLD_SIZE", "1") or 1))
        rk = max(0, int(os.environ.get("RANK", "0") or 0))
        if ws > 1:
            q = p.with_name(p.stem + f".shard{rk}-of-{ws}" + p.suffix)
            q.parent.mkdir(parents=True, exist_ok=True)
            return q
    except Exception:
        pass
    p.parent.mkdir(parents=True, exist_ok=True)
    return p

def main() -> int:
    args = parse_args()
    root = Path(__file__).resolve().parents[2]
    outp = Path(args.out) if args.out else _default_out(root, args.suite)
    outp = _sharded_path(outp)

    # Try to import evalpro harness
    have = False
    try:
        import codeeval_pro  # type: ignore  # noqa: F401
        have = True
    except Exception:
        have = False

    if not have:
        outp.write_text("", encoding="utf-8")
        print(json.dumps({
            "ok": False,
            "status": "skipped",
            "reason": "CodeEval-Pro harness not installed",
            "out": str(outp),
        }))
        return 0

    with outp.open("w", encoding="utf-8") as f:
        f.write(json.dumps({"suite": args.suite, "status": "not_implemented"}) + "\n")
    print(json.dumps({"ok": True, "out": str(outp), "note": "harness stub"}))
    return 0

if __name__ == "__main__":
    raise SystemExit(main())