import re
from pathlib import Path
from typing import Any, Dict, List

def parse_args() -> Any:
    ap = argparse.ArgumentParser(description="Build a refined Next.js apprepo spec from a source repo (extract + enrich)")
    root = Path(__file__).resolve().parents[2]
    ap.add_argument("--src", default=str(Path("/data/agiattempt/inspiration/ai-town")))
    ap.add_argument("--out", default=str(root / "data" / "tasks" / "apprepo" / "spec.ai_town.refined.json"))
    return ap.parse_args()

def _read_text(p: Path) -> str:
    try:
        return p.read_text(encoding="utf-8")
    except Exception:
        return ""

def _extract(src: Path) -> Dict[str, Any]:
    spec: Dict[str, Any] = {
        "name": "ai-town-like",
        "framework": "nextjs",
        "language": "ts",
        "pages": [],
        "components": [],
        "assets": [],
        "configs": {}
    }
    pages_dir = src / "src" / "app"
    if pages_dir.exists():
        for p in pages_dir.rglob("*.tsx"):
            rel = p.relative_to(src).as_posix()
            spec["pages"].append({"path": rel, "preview": _read_text(p)[:800]})
    comp_dir = src / "src" / "components"
    if comp_dir.exists():
        for p in comp_dir.rglob("*.tsx"):
            rel = p.relative_to(src).as_posix()
            spec["components"].append({"path": rel, "preview": _read_text(p)[:800]})
    assets_dir = src / "public"
    if assets_dir.exists():
        for p in assets_dir.rglob("*"):
            if p.is_file() and p.suffix.lower() in (".svg", ".png", ".jpg", ".jpeg"):
                rel = p.relative_to(src).as_posix()
                spec["assets"].append({"path": rel})
    for fname in ("package.json", "tsconfig.json", "next.config.js", "next.config.mjs", "eslint.config.js", "eslint.config.mjs", "tailwind.config.js", "tailwind.config.cjs", "tailwind.config.ts", "postcss.config.js", "postcss.config.cjs"):
        fp = src / fname
        if fp.exists():
            spec["configs"][fname] = _read_text(fp)[:4000]
    # Globals CSS
    gcss = src / "src" / "app" / "globals.css"
    if gcss.exists():
        spec.setdefault("configs", {})["globals.css"] = _read_text(gcss)[:4000]
    # Fonts directory
    fonts_dir = src / "public" / "assets" / "fonts"
    if fonts_dir.exists():
        fonts = []
        for p in fonts_dir.rglob("*"):
            if p.is_file():
                fonts.append({"path": p.relative_to(src).as_posix()})
        if fonts:
            spec["assets"].extend(fonts)
    return spec

def _parse_package(pkg_text: str) -> Dict[str, Any]:
    try:
        return json.loads(pkg_text)
    except Exception:
        return {}

def _derive_routes(pages: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
    routes: List[Dict[str, Any]] = []
    for it in pages or []:
        p = str(it.get("path", ""))
        # Skip layout and non-page utility files in routes
        if p.endswith("layout.tsx") or not p.endswith("page.tsx"):
            routes.append({"file": p, "path": None})
            continue
        route = None
        try:
            sub = p.split("/app/")[1]


    for fname in ("package.json", "tsconfig.json", "next.config.js", "next.config.mjs", "eslint.config.js", "eslint.config.mjs", "tailwind.config.js", "tailwind.config.cjs", "tailwind.config.ts", "postcss.config.js", "postcss.config.cjs"):
        fp = src / fname
        if fp.exists():
            spec["configs"][fname] = _read_text(fp)[:4000]
    # Globals CSS
    gcss = src / "src" / "app" / "globals.css"
    if gcss.exists():
        spec.setdefault("configs", {})["globals.css"] = _read_text(gcss)[:4000]
    # Fonts directory
    fonts_dir = src / "public" / "assets" / "fonts"
    if fonts_dir.exists():
        fonts = []
        for p in fonts_dir.rglob("*"):
            if p.is_file():
                fonts.append({"path": p.relative_to(src).as_posix()})
        if fonts:
            spec["assets"].extend(fonts)
    return spec

def _parse_package(pkg_text: str) -> Dict[str, Any]:
    try:
        return json.loads(pkg_text)
    except Exception:
        return {}

def _derive_routes(pages: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
    routes: List[Dict[str, Any]] = []
    for it in pages or []:
        p = str(it.get("path", ""))
        # Skip layout and non-page utility files in routes
        if p.endswith("layout.tsx") or not p.endswith("page.tsx"):
            routes.append({"file": p, "path": None})
            continue
        route = None
        try:
            sub = p.split("/app/")[1]
            sub = sub[:-len("/page.tsx")] if sub.endswith("/page.tsx") else sub
            if sub == "":
                route = "/"
            else:
                route = "/" + sub.strip("/")
        except Exception:
            route = None


        spec.setdefault("configs", {})["globals.css"] = _read_text(gcss)[:4000]
    # Fonts directory
    fonts_dir = src / "public" / "assets" / "fonts"
    if fonts_dir.exists():
        fonts = []
        for p in fonts_dir.rglob("*"):
            if p.is_file():
                fonts.append({"path": p.relative_to(src).as_posix()})
        if fonts:
            spec["assets"].extend(fonts)
    return spec

def _parse_package(pkg_text: str) -> Dict[str, Any]:
    try:
        return json.loads(pkg_text)
    except Exception:
        return {}

def _derive_routes(pages: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
    routes: List[Dict[str, Any]] = []
    for it in pages or []:
        p = str(it.get("path", ""))
        # Skip layout and non-page utility files in routes
        if p.endswith("layout.tsx") or not p.endswith("page.tsx"):
            routes.append({"file": p, "path": None})
            continue
        route = None
        try:
            sub = p.split("/app/")[1]
            sub = sub[:-len("/page.tsx")] if sub.endswith("/page.tsx") else sub
            if sub == "":
                route = "/"
            else:
                route = "/" + sub.strip("/")
        except Exception:
            route = None
        routes.append({"file": p, "path": route})
    return routes

def _detect_providers(layout_preview: str) -> List[str]:
    providers: List[str] = []
    for name in ("ClerkProvider", "ConvexProvider", "ConvexProviderWithClerk", "ConvexClientProvider"):
        if name in layout_preview:
            providers.append(name)
    return providers

def _detect_env(previews: List[str]) -> List[str]:
    envs: set[str] = set()
    pat = re.compile(r"process\.env\.([A-Z0-9_]+)")
    for text in previews:
        for m in pat.findall(text or ""):
            envs.add(m)
    return sorted(envs)

def _enrich(spec: Dict[str, Any], src_root: Path) -> Dict[str, Any]:


    for it in pages or []:
        p = str(it.get("path", ""))
        # Skip layout and non-page utility files in routes
        if p.endswith("layout.tsx") or not p.endswith("page.tsx"):
            routes.append({"file": p, "path": None})
            continue
        route = None
        try:
            sub = p.split("/app/")[1]
            sub = sub[:-len("/page.tsx")] if sub.endswith("/page.tsx") else sub
            if sub == "":
                route = "/"
            else:
                route = "/" + sub.strip("/")
        except Exception:
            route = None
        routes.append({"file": p, "path": route})
    return routes

def _detect_providers(layout_preview: str) -> List[str]:
    providers: List[str] = []
    for name in ("ClerkProvider", "ConvexProvider", "ConvexProviderWithClerk", "ConvexClientProvider"):
        if name in layout_preview:
            providers.append(name)
    return providers

def _detect_env(previews: List[str]) -> List[str]:
    envs: set[str] = set()
    pat = re.compile(r"process\.env\.([A-Z0-9_]+)")
    for text in previews:
        for m in pat.findall(text or ""):
            envs.add(m)
    return sorted(envs)

def _enrich(spec: Dict[str, Any], src_root: Path) -> Dict[str, Any]:
    cfgs: Dict[str, str] = dict(spec.get("configs", {}))
    pkg = _parse_package(cfgs.get("package.json", ""))
    deps = dict(pkg.get("dependencies", {}))
    scripts = dict(pkg.get("scripts", {}))

    pages = list(spec.get("pages", []))
    comps = list(spec.get("components", []))
    versions = {


            sub = p.split("/app/")[1]
            sub = sub[:-len("/page.tsx")] if sub.endswith("/page.tsx") else sub
            if sub == "":
                route = "/"
            else:
                route = "/" + sub.strip("/")
        except Exception:
            route = None
        routes.append({"file": p, "path": route})
    return routes

def _detect_providers(layout_preview: str) -> List[str]:
    providers: List[str] = []
    for name in ("ClerkProvider", "ConvexProvider", "ConvexProviderWithClerk", "ConvexClientProvider"):
        if name in layout_preview:
            providers.append(name)
    return providers

def _detect_env(previews: List[str]) -> List[str]:
    envs: set[str] = set()
    pat = re.compile(r"process\.env\.([A-Z0-9_]+)")
    for text in previews:
        for m in pat.findall(text or ""):
            envs.add(m)
    return sorted(envs)

def _enrich(spec: Dict[str, Any], src_root: Path) -> Dict[str, Any]:
    cfgs: Dict[str, str] = dict(spec.get("configs", {}))
    pkg = _parse_package(cfgs.get("package.json", ""))
    deps = dict(pkg.get("dependencies", {}))
    scripts = dict(pkg.get("scripts", {}))

    pages = list(spec.get("pages", []))
    comps = list(spec.get("components", []))
    versions = {
        "node": None,
        "next": deps.get("next"),
        "typescript": deps.get("typescript"),
        "react": deps.get("react"),
    }
    # Try to read Node version from package.json engines
    try:
        eng = dict(_parse_package(cfgs.get("package.json", "")).get("engines", {}))
        if isinstance(eng.get("node"), str):


    return routes

def _detect_providers(layout_preview: str) -> List[str]:
    providers: List[str] = []
    for name in ("ClerkProvider", "ConvexProvider", "ConvexProviderWithClerk", "ConvexClientProvider"):
        if name in layout_preview:
            providers.append(name)
    return providers

def _detect_env(previews: List[str]) -> List[str]:
    envs: set[str] = set()
    pat = re.compile(r"process\.env\.([A-Z0-9_]+)")
    for text in previews:
        for m in pat.findall(text or ""):
            envs.add(m)
    return sorted(envs)

def _enrich(spec: Dict[str, Any], src_root: Path) -> Dict[str, Any]:
    cfgs: Dict[str, str] = dict(spec.get("configs", {}))
    pkg = _parse_package(cfgs.get("package.json", ""))
    deps = dict(pkg.get("dependencies", {}))
    scripts = dict(pkg.get("scripts", {}))

    pages = list(spec.get("pages", []))
    comps = list(spec.get("components", []))
    versions = {
        "node": None,
        "next": deps.get("next"),
        "typescript": deps.get("typescript"),
        "react": deps.get("react"),
    }
    # Try to read Node version from package.json engines
    try:
        eng = dict(_parse_package(cfgs.get("package.json", "")).get("engines", {}))
        if isinstance(eng.get("node"), str):
            versions["node"] = eng.get("node")
    except Exception:
        pass
    routes = _derive_routes(pages)
    layout_preview = next((it.get("preview", "") for it in pages if str(it.get("path", "")).endswith("layout.tsx")), "")
    layout = {
        "file": next((it.get("path") for it in pages if str(it.get("path", "")).endswith("layout.tsx")), None),
        "providers": _detect_providers(layout_preview),
        "uses_globals_css": ("globals.css" in layout_preview),
    }
    styling = {
        "css_framework": "tailwind" if "tailwindcss" in deps else None,
        "tailwind_present": ("tailwindcss" in deps),
        "globals_css": "src/app/globals.css" if "globals.css" in layout_preview else None,
    }
    graphics = {
        "uses_pixi": any(pkg_name in deps for pkg_name in ("pixi.js", "@pixi/react", "pixi-viewport")),
        "pixi_packages": [k for k in deps.keys() if k.startswith("@pixi") or k.startswith("pixi")],
    }
    characters = {
        "component": next((it.get("path") for it in comps if str(it.get("path", "")).endswith("Character.tsx")), None),
        "props_required": ["textureUrl", "spritesheetData", "x", "y", "orientation"],
        "states": ["isMoving", "isThinking", "isSpeaking"],
    }
    assets = list(spec.get("assets", []))
    tile_assets = [a for a in assets if str(a.get("path", "")).endswith(("tileset.png", "tile.png", "tiles.png")) or "tile" in str(a.get("path", "")).lower()]
    tiles = {
        "tile_set_images": [a.get("path") for a in tile_assets],
        "tile_dim": None,
        "tile_set_dim": None,
        "notes": "Fill tile_dim and tile_set_dim by inspecting image metadata or code constants",
    }
    previews = [it.get("preview", "") for it in pages] + [it.get("preview", "") for it in comps]
    env_vars = _detect_env(previews)
    # Heuristic additions based on deps
    deps_present = set(deps.keys())
    if "@clerk/nextjs" in deps_present:
        for k in ("NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY", "CLERK_SECRET_KEY"):
            if k not in env_vars:
                env_vars.append(k)
    if "@pinecone-database/pinecone" in deps_present:
        for k in ("PINECONE_API_KEY", "PINECONE_ENVIRONMENT"):
            if k not in env_vars:
                env_vars.append(k)
    if "replicate" in deps_present:
        if "REPLICATE_API_TOKEN" not in env_vars:
            env_vars.append("REPLICATE_API_TOKEN")
    env_schema = [{"name": n, "required": True} for n in env_vars]
    allowlists = {
        "imports": {
            "pages": ["react", "next/*", "@/components/*"],
            "components": ["react", "@pixi/*", "pixi.js", "clsx", "convex/*"],
            "api": ["next/*"],
        },
        "file_write": ["package.json", "tsconfig.json", "next.config.*", "src/**/*", "public/**/*"],
    }
    verify_commands = [
        "tsc --noEmit",
        "next lint || true",
        "next build --no-telemetry --no-lint || true"
    ]
    verify_timeouts = {
        "tsc --noEmit": 60,
        "next lint || true": 45,
        "next build --no-telemetry --no-lint || true": 120
    }
    # Generation plan order
    plan = {
        "order": ["configs", "package", "styling", "layout", "routes", "components", "assets", "backend"],
        "repair_checks": ["syntax", "typecheck", "lint", "build"]
    }

    # Backend (Convex) metadata — scan relative to provided --src
    backend: Dict[str, Any] = {}
    cdir = src_root / "convex"
    if cdir.exists():
        backend["schema_file"] = (cdir / "schema.ts").as_posix() if (cdir / "schema.ts").exists() else None
        fn_files = []
        for p in cdir.rglob("*.ts"):
            if "_generated" in p.parts:
                continue
            fn_files.append(p.as_posix())
        backend["functions"] = fn_files
    spec.update({
        "versions": versions,
        "scripts": scripts,
        "routes": routes,
        "layout": layout,
        "styling": styling,
        "graphics": graphics,
        "characters": characters,
        "tiles": tiles,
        "allowlists": allowlists,
        "env_schema": env_schema,
        "verify_commands": verify_commands,
        "verify_timeouts": verify_timeouts,
        "plan": plan,
        "backend": backend,
    })
    return spec

def main() -> int:
    args = parse_args()
    src = Path(args.src)
    outp = Path(args.out)
    base = _extract(src)
    refined = _enrich(base, src)
    outp.parent.mkdir(parents=True, exist_ok=True)
    outp.write_text(json.dumps(refined, ensure_ascii=False, indent=2), encoding="utf-8")
    print(json.dumps({"ok": True, "out": str(outp), "routes": len(refined.get("routes", [])), "env": len(refined.get("env_schema", []))}))
    return 0

if __name__ == "__main__":
    raise SystemExit(main())


main

        backend["functions"] = fn_files
    spec.update({
        "versions": versions,
        "scripts": scripts,
        "routes": routes,
        "layout": layout,
        "styling": styling,
        "graphics": graphics,
        "characters": characters,
        "tiles": tiles,
        "allowlists": allowlists,
        "env_schema": env_schema,
        "verify_commands": verify_commands,
        "verify_timeouts": verify_timeouts,
        "plan": plan,
        "backend": backend,
    })
    return spec

def main() -> int:
    args = parse_args()
    src = Path(args.src)
    outp = Path(args.out)
    base = _extract(src)
    refined = _enrich(base, src)
    outp.parent.mkdir(parents=True, exist_ok=True)
    outp.write_text(json.dumps(refined, ensure_ascii=False, indent=2), encoding="utf-8")
    print(json.dumps({"ok": True, "out": str(outp), "routes": len(refined.get("routes", [])), "env": len(refined.get("env_schema", []))}))
    return 0

if __name__ == "__main__":
    raise SystemExit(main())


from __future__ import annotations

import argparse
import ast
import json
import time
from pathlib import Path
from typing import Any, Dict, Set

def parse_args() -> Any:
    ap = argparse.ArgumentParser(description="Task: generate a standalone Python app with import whitelist")
    root = Path(__file__).resolve().parents[2]
    ap.add_argument("--spec", required=True, help="Path to JSON spec")
    ap.add_argument("--whitelist", required=True, help="Path to JSON list of allowed imports")
    ap.add_argument("--outdir", default=str(root / "data" / "tasks" / "runs" / "appgen"))
    ap.add_argument("--model", default="meta-llama/Llama-3.2-3B")
    ap.add_argument("--max-new-tokens", type=int, default=512)
    ap.add_argument("--temperature", type=float, default=0.2)
    ap.add_argument("--top-p", type=float, default=0.95)
    ap.add_argument("--retries", type=int, default=1)
    ap.add_argument("--precheck", action="store_true")
    return ap.parse_args()

def _collect_imports(src: str) -> Set[str]:
    try:
        tree = ast.parse(src)
    except Exception:
        return set(["<syntax_error>"])
    mods: Set[str] = set()
    for node in ast.walk(tree):
        if isinstance(node, ast.Import):
            for n in node.names:
                m = (n.name or "").split(".")[0]
                if m:
                    mods.add(m)
        elif isinstance(node, ast.ImportFrom):
            m = (node.module or "").split(".")[0]
            if m:
                mods.add(m)
    return mods

def _sanitize(text: str) -> str:
    s = (text or "").strip()
    if s.startswith("```"):
        try:
            st = s.find("\n"); en = s.rfind("```")
            if st != -1 and en != -1 and en > st:
                s = s[st + 1 : en].strip()
        except Exception:
            pass
    return s

def main() -> int:
    args = parse_args()
    ts = time.strftime("%Y%m%dT%H%M%SZ", time.gmtime())
    run_dir = Path(args.outdir) / ts
    run_dir.mkdir(parents=True, exist_ok=True)
    spec = json.loads(Path(args.spec).read_text(encoding="utf-8"))
    whitelist = set(json.loads(Path(args.whitelist).read_text(encoding="utf-8")))

    try:
        from agi_dw.core.llm.hf_client import HFClient  # type: ignore
        from agi_dw.core.utils.bench_utils import ensure_safe_env, retry_with_backoff, precheck_code  # type: ignore
    except Exception as e:
        (run_dir / "status.json").write_text(json.dumps({"ok": False, "error": f"llm_missing: {e}"}), encoding="utf-8")
        print(json.dumps({"ok": False, "error": f"llm_missing: {e}"}))
        return 2

    ensure_safe_env()
    llm = HFClient.get_cached(str(getattr(args, "model")))
    prompt = (
        "You will write a standalone Python 3 application as a single file.\n"
        "Rules:\n- Only use whitelisted imports.\n- Provide main() and if __name__ == '__main__'.\n- No external packages.\n\n"
        f"Spec: {json.dumps(spec, ensure_ascii=False)}\n"
        f"Allowed imports: {', '.join(sorted(whitelist))}\n\nOutput ONLY the Python source."
    )
    params = {"max_new_tokens": int(args.max_new_tokens), "temperature": float(args.temperature), "top_p": float(args.top_p)}

    def _gen() -> str:
        return str(llm.generate(prompt, **params))

    text = retry_with_backoff(_gen, int(getattr(args, "retries", 1) or 1), 0.75) or ""
    body = _sanitize(text)

    if bool(getattr(args, "precheck", False)):
        ok, err = precheck_code(body)
        if not ok:
            # One-shot repair attempt using the compiler error message and previous code
            repair_prompt = (
                "You are a Python fixer. Given a Python 3 program and a compiler error, return a corrected full source file.\n"
                "Rules: keep semantics; do NOT add non-whitelisted imports; output ONLY the corrected source.\n\n"
                f"Error: {err}\n\n"
                "Program:\n" + body + "\n\nCorrected program:"
            )
            text2 = retry_with_backoff(lambda: str(llm.generate(repair_prompt, **params)), int(getattr(args, "retries", 1) or 1), 0.75) or ""
            body2 = _sanitize(text2)
            ok2, err2 = precheck_code(body2)
            if ok2:
                body = body2
            else:
                (run_dir / "status.json").write_text(json.dumps({"ok": False, "error": f"syntax_error: {err2}"}), encoding="utf-8")
                print(json.dumps({"ok": False, "error": f"syntax_error: {err2}"}))
                return 2

    mods = _collect_imports(body)
    bad = [m for m in mods if m not in whitelist]
    if bad:
        (run_dir / "status.json").write_text(json.dumps({"ok": False, "error": "import_whitelist", "modules": bad}), encoding="utf-8")
        print(json.dumps({"ok": False, "error": "import_whitelist", "modules": bad}))
        return 2

    out_file = run_dir / f"{spec.get('name','app')}.py"
    out_file.write_text(body, encoding="utf-8")
    (run_dir / "status.json").write_text(json.dumps({"ok": True, "out": str(out_file)}), encoding="utf-8")
    print(json.dumps({"ok": True, "out": str(out_file)}))
    return 0

if __name__ == "__main__":
    raise SystemExit(main())


from __future__ import annotations

import argparse
import ast
import json
import time
from pathlib import Path
from typing import Any, Dict, Set

def parse_args() -> Any:
    ap = argparse.ArgumentParser(description="Task: generate a standalone Python app with import whitelist")
    root = Path(__file__).resolve().parents[2]
    ap.add_argument("--spec", required=True, help="Path to JSON spec")
    ap.add_argument("--whitelist", required=True, help="Path to JSON list of allowed imports")
    ap.add_argument("--outdir", default=str(root / "data" / "tasks" / "runs" / "appgen"))
    ap.add_argument("--model", default="meta-llama/Llama-3.2-3B")
    ap.add_argument("--max-new-tokens", type=int, default=512)
    ap.add_argument("--temperature", type=float, default=0.2)
    ap.add_argument("--top-p", type=float, default=0.95)
    ap.add_argument("--retries", type=int, default=1)
    ap.add_argument("--precheck", action="store_true")
    return ap.parse_args()

def _collect_imports(src: str) -> Set[str]:
    try:
        tree = ast.parse(src)
    except Exception:
        return set(["<syntax_error>"])
    mods: Set[str] = set()
    for node in ast.walk(tree):
        if isinstance(node, ast.Import):
            for n in node.names:
                m = (n.name or "").split(".")[0]
                if m:
                    mods.add(m)
        elif isinstance(node, ast.ImportFrom):
            m = (node.module or "").split(".")[0]
            if m:
                mods.add(m)
    return mods


import time
from pathlib import Path
from typing import Any, Dict, Set

def parse_args() -> Any:
    ap = argparse.ArgumentParser(description="Task: generate a standalone Python app with import whitelist")
    root = Path(__file__).resolve().parents[2]
    ap.add_argument("--spec", required=True, help="Path to JSON spec")
    ap.add_argument("--whitelist", required=True, help="Path to JSON list of allowed imports")
    ap.add_argument("--outdir", default=str(root / "data" / "tasks" / "runs" / "appgen"))
    ap.add_argument("--model", default="meta-llama/Llama-3.2-3B")
    ap.add_argument("--max-new-tokens", type=int, default=512)
    ap.add_argument("--temperature", type=float, default=0.2)
    ap.add_argument("--top-p", type=float, default=0.95)
    ap.add_argument("--retries", type=int, default=1)
    ap.add_argument("--precheck", action="store_true")
    return ap.parse_args()

def _collect_imports(src: str) -> Set[str]:
    try:
        tree = ast.parse(src)
    except Exception:
        return set(["<syntax_error>"])
    mods: Set[str] = set()
    for node in ast.walk(tree):
        if isinstance(node, ast.Import):
            for n in node.names:
                m = (n.name or "").split(".")[0]
                if m:
                    mods.add(m)
        elif isinstance(node, ast.ImportFrom):
            m = (node.module or "").split(".")[0]
            if m:
                mods.add(m)
    return mods

def _sanitize(text: str) -> str:
    s = (text or "").strip()
    if s.startswith("```"):
        try:
            st = s.find("\n"); en = s.rfind("```")
            if st != -1 and en != -1 and en > st:
                s = s[st + 1 : en].strip()
        except Exception:
            pass
    return s

def main() -> int:
    args = parse_args()
    ts = time.strftime("%Y%m%dT%H%M%SZ", time.gmtime())
    run_dir = Path(args.outdir) / ts
    run_dir.mkdir(parents=True, exist_ok=True)
    spec = json.loads(Path(args.spec).read_text(encoding="utf-8"))


def _collect_imports(src: str) -> Set[str]:
    try:
        tree = ast.parse(src)
    except Exception:
        return set(["<syntax_error>"])
    mods: Set[str] = set()
    for node in ast.walk(tree):
        if isinstance(node, ast.Import):
            for n in node.names:
                m = (n.name or "").split(".")[0]
                if m:
                    mods.add(m)
        elif isinstance(node, ast.ImportFrom):
            m = (node.module or "").split(".")[0]
            if m:
                mods.add(m)
    return mods

def _sanitize(text: str) -> str:
    s = (text or "").strip()
    if s.startswith("```"):
        try:
            st = s.find("\n"); en = s.rfind("```")
            if st != -1 and en != -1 and en > st:
                s = s[st + 1 : en].strip()
        except Exception:
            pass
    return s

def main() -> int:
    args = parse_args()
    ts = time.strftime("%Y%m%dT%H%M%SZ", time.gmtime())
    run_dir = Path(args.outdir) / ts
    run_dir.mkdir(parents=True, exist_ok=True)
    spec = json.loads(Path(args.spec).read_text(encoding="utf-8"))
    whitelist = set(json.loads(Path(args.whitelist).read_text(encoding="utf-8")))

    try:
        from agi_dw.core.llm.hf_client import HFClient  # type: ignore
        from agi_dw.core.utils.bench_utils import ensure_safe_env, retry_with_backoff, precheck_code  # type: ignore
    except Exception as e:
        (run_dir / "status.json").write_text(json.dumps({"ok": False, "error": f"llm_missing: {e}"}), encoding="utf-8")
        print(json.dumps({"ok": False, "error": f"llm_missing: {e}"}))
        return 2

    ensure_safe_env()
    llm = HFClient.get_cached(str(getattr(args, "model")))


                    mods.add(m)
        elif isinstance(node, ast.ImportFrom):
            m = (node.module or "").split(".")[0]
            if m:
                mods.add(m)
    return mods

def _sanitize(text: str) -> str:
    s = (text or "").strip()
    if s.startswith("```"):
        try:
            st = s.find("\n"); en = s.rfind("```")
            if st != -1 and en != -1 and en > st:
                s = s[st + 1 : en].strip()
        except Exception:
            pass
    return s

def main() -> int:
    args = parse_args()
    ts = time.strftime("%Y%m%dT%H%M%SZ", time.gmtime())
    run_dir = Path(args.outdir) / ts
    run_dir.mkdir(parents=True, exist_ok=True)
    spec = json.loads(Path(args.spec).read_text(encoding="utf-8"))
    whitelist = set(json.loads(Path(args.whitelist).read_text(encoding="utf-8")))

    try:
        from agi_dw.core.llm.hf_client import HFClient  # type: ignore
        from agi_dw.core.utils.bench_utils import ensure_safe_env, retry_with_backoff, precheck_code  # type: ignore
    except Exception as e:
        (run_dir / "status.json").write_text(json.dumps({"ok": False, "error": f"llm_missing: {e}"}), encoding="utf-8")
        print(json.dumps({"ok": False, "error": f"llm_missing: {e}"}))
        return 2

    ensure_safe_env()
    llm = HFClient.get_cached(str(getattr(args, "model")))
    prompt = (
        "You will write a standalone Python 3 application as a single file.\n"
        "Rules:\n- Only use whitelisted imports.\n- Provide main() and if __name__ == '__main__'.\n- No external packages.\n\n"
        f"Spec: {json.dumps(spec, ensure_ascii=False)}\n"
        f"Allowed imports: {', '.join(sorted(whitelist))}\n\nOutput ONLY the Python source."
    )
    params = {"max_new_tokens": int(args.max_new_tokens), "temperature": float(args.temperature), "top_p": float(args.top_p)}

    def _gen() -> str:
        return str(llm.generate(prompt, **params))

    text = retry_with_backoff(_gen, int(getattr(args, "retries", 1) or 1), 0.75) or ""
    body = _sanitize(text)

    if bool(getattr(args, "precheck", False)):
        ok, err = precheck_code(body)
        if not ok:
            # One-shot repair attempt using the compiler error message and previous code
            repair_prompt = (
                "You are a Python fixer. Given a Python 3 program and a compiler error, return a corrected full source file.\n"
                "Rules: keep semantics; do NOT add non-whitelisted imports; output ONLY the corrected source.\n\n"
                f"Error: {err}\n\n"
                "Program:\n" + body + "\n\nCorrected program:"
            )
            text2 = retry_with_backoff(lambda: str(llm.generate(repair_prompt, **params)), int(getattr(args, "retries", 1) or 1), 0.75) or ""
            body2 = _sanitize(text2)
            ok2, err2 = precheck_code(body2)
            if ok2:
                body = body2
            else:
                (run_dir / "status.json").write_text(json.dumps({"ok": False, "error": f"syntax_error: {err2}"}), encoding="utf-8")
                print(json.dumps({"ok": False, "error": f"syntax_error: {err2}"}))
                return 2

    mods = _collect_imports(body)
    bad = [m for m in mods if m not in whitelist]
    if bad:
        (run_dir / "status.json").write_text(json.dumps({"ok": False, "error": "import_whitelist", "modules": bad}), encoding="utf-8")
        print(json.dumps({"ok": False, "error": "import_whitelist", "modules": bad}))
        return 2

    out_file = run_dir / f"{spec.get('name','app')}.py"
    out_file.write_text(body, encoding="utf-8")
    (run_dir / "status.json").write_text(json.dumps({"ok": True, "out": str(out_file)}), encoding="utf-8")
    print(json.dumps({"ok": True, "out": str(out_file)}))
    return 0

if __name__ == "__main__":
    raise SystemExit(main())


    whitelist = set(json.loads(Path(args.whitelist).read_text(encoding="utf-8")))

    try:
        from agi_dw.core.llm.hf_client import HFClient  # type: ignore
        from agi_dw.core.utils.bench_utils import ensure_safe_env, retry_with_backoff, precheck_code  # type: ignore
    except Exception as e:
        (run_dir / "status.json").write_text(json.dumps({"ok": False, "error": f"llm_missing: {e}"}), encoding="utf-8")
        print(json.dumps({"ok": False, "error": f"llm_missing: {e}"}))
        return 2

    ensure_safe_env()
    llm = HFClient.get_cached(str(getattr(args, "model")))
    prompt = (
        "You will write a standalone Python 3 application as a single file.\n"
        "Rules:\n- Only use whitelisted imports.\n- Provide main() and if __name__ == '__main__'.\n- No external packages.\n\n"
        f"Spec: {json.dumps(spec, ensure_ascii=False)}\n"
        f"Allowed imports: {', '.join(sorted(whitelist))}\n\nOutput ONLY the Python source."
    )
    params = {"max_new_tokens": int(args.max_new_tokens), "temperature": float(args.temperature), "top_p": float(args.top_p)}

    def _gen() -> str:
        return str(llm.generate(prompt, **params))

    text = retry_with_backoff(_gen, int(getattr(args, "retries", 1) or 1), 0.75) or ""
    body = _sanitize(text)

    if bool(getattr(args, "precheck", False)):
        ok, err = precheck_code(body)
        if not ok:
            # One-shot repair attempt using the compiler error message and previous code
            repair_prompt = (
                "You are a Python fixer. Given a Python 3 program and a compiler error, return a corrected full source file.\n"
                "Rules: keep semantics; do NOT add non-whitelisted imports; output ONLY the corrected source.\n\n"
                f"Error: {err}\n\n"
                "Program:\n" + body + "\n\nCorrected program:"
            )
            text2 = retry_with_backoff(lambda: str(llm.generate(repair_prompt, **params)), int(getattr(args, "retries", 1) or 1), 0.75) or ""
            body2 = _sanitize(text2)
            ok2, err2 = precheck_code(body2)
            if ok2:
                body = body2
            else:


#!/usr/bin/env python3
from __future__ import annotations
import logging

import json
from pathlib import Path
from typing import Dict, Any, List

def main() -> int:
    root = Path(__file__).resolve().parents[2]
    out = root / "data" / "sandbox" / "tmp" / "feats_humaneval.json"
    feats: List[Dict[str, Any]] = []
    try:
        from human_eval.data import read_problems  # type: ignore
        problems: Dict[str, Dict[str, Any]] = read_problems()
        for tid, row in problems.items():
            prompt = str(row.get("prompt", ""))
            # Heuristic: first line up to '(' as function name
            fn_name = None
            try:
                for ln in prompt.splitlines():
                    ln = ln.strip()
                    if ln.startswith("def ") and "(" in ln:
                        fn_name = ln.split("def ", 1)[1].split("(", 1)[0].strip()
                        break
            except Exception:
                fn_name = None
            feats.append({"task_id": tid, "function": fn_name, "context": prompt[:400]})
    except Exception:
        feats = []
    out.parent.mkdir(parents=True, exist_ok=True)
    out.write_text(json.dumps({"ok": True, "n": len(feats), "items": feats}, ensure_ascii=False, indent=2), encoding="utf-8")
    print(json.dumps({"ok": True, "out": str(out), "n": len(feats)}))
    return 0

if __name__ == "__main__":
    raise SystemExit(main())


#!/usr/bin/env python3
import logging
import argparse
import json
import os
import time
from typing import Optional, Dict, Any, List
from pathlib import Path
from datetime import datetime
import re

def normalize_answer(s: str) -> str:
	"""Normalize answer for exact match evaluation."""
	s = s.lower().strip()
	# Remove articles and common words
	s = re.sub(r'\b(a|an|the)\b', '', s)
	# Remove punctuation
	s = re.sub(r'[^\w\s]', '', s)
	# Normalize whitespace
	s = re.sub(r'\s+', ' ', s).strip()
	return s

def f1_score(prediction: str, ground_truths: list) -> float:
	"""Calculate F1 score between prediction and ground truth answers."""
	if not ground_truths:
		return 0.0

	pred_tokens = normalize_answer(prediction).split()
	best_f1 = 0.0

	for gt in ground_truths:
		gt_tokens = normalize_answer(str(gt)).split()
		if not gt_tokens:
			continue

		# Calculate precision and recall
		common = set(pred_tokens) & set(gt_tokens)
		precision = len(common) / len(pred_tokens) if pred_tokens else 0.0
		recall = len(common) / len(gt_tokens) if gt_tokens else 0.0

		if precision + recall > 0:
			f1 = 2 * precision * recall / (precision + recall)
			best_f1 = max(best_f1, f1)

	return best_f1

def parse_args() -> argparse.Namespace:
	root = Path(__file__).resolve().parents[1]
	ap = argparse.ArgumentParser(description="Run LLM benchmark suite (scaffold)")
	ap.add_argument("--benchmarks", nargs="*", default=[], help="Benchmark names to run (default: all known)")
	ap.add_argument("--model", default=os.environ.get("HF_MODEL", "meta-llama/Llama-3.2-3B"))
	ap.add_argument("--backend", default="hf", help="Backend identifier (huggingface)")
	ap.add_argument("--prompt-dir", default=str(root / "data" / "prompts" / "llm_bench"))
	ap.add_argument("--out", default=str(root / "data" / "llm_bench" / "results.json"))
	ap.add_argument("--seed", type=int, default=42)
	ap.add_argument("--config", default=str(root / "data" / "llm_bench" / "config.json"), help="Optional per-benchmark config (decoding, CoT, shots)")
	ap.add_argument("--max-samples", type=int, default=64, help="Max samples per benchmark for quick runs")
	ap.add_argument("--http-url", default=os.environ.get("LLM_HTTP_URL", ""), help="HTTP backend URL when backend=http")
	ap.add_argument("--execute", action="store_true", help="Run actual evaluation (requires datasets/transformers)")
	ap.add_argument("--cost-per-sec", type=float, default=float(os.environ.get("LLM_COST_PER_SEC", "0") or 0.0), help="Optional cost rate ($/sec) to compute total and per-sample cost")
	ap.add_argument("--dry", action="store_true", help="Do not run anything; write scaffold results")
	ap.add_argument("--resume", action="store_true", help="Resume by merging with existing results.json and skipping completed benchmarks")
	ap.add_argument("--retries", type=int, default=2, help="Retries per sample on backend errors")
	ap.add_argument("--retry-backoff", type=float, default=0.5, help="Exponential backoff base (seconds)")
	return ap.parse_args()

KNOWN_BENCHMARKS = {
	# Knowledge & Language Understanding
	"mmlu": {"metric": "acc"},
	"arc": {"metric": "acc"},
	"glue": {"metric": "task"},
	"superglue": {"metric": "task"},
	"superglue_rte": {"metric": "acc"},
	"superglue_copa": {"metric": "acc"},
	"superglue_wic": {"metric": "acc"},
	"superglue_wsc": {"metric": "acc"},
	"superglue_multirc": {"metric": "task"},
	"superglue_cb": {"metric": "acc"},
	"nq": {"metric": "em_f1"},
	"lambada": {"metric": "acc"},
	"hellaswag": {"metric": "acc"},
	"multinli": {"metric": "acc"},
	"triviaqa": {"metric": "em_f1"},
	"winogrande": {"metric": "acc"},
	"sciq": {"metric": "acc"},
	# Reasoning
	"gsm8k": {"metric": "acc"},
	"drop": {"metric": "em_f1"},
	"crass": {"metric": "score"},
	"race": {"metric": "acc"},
	"bbh": {"metric": "acc"},
	"agieval": {"metric": "acc"},
	"boolq": {"metric": "acc"},
	# Conversations
	"mt-bench": {"metric": "judge"},
	"quac": {"metric": "f1_heq"},
	# Summarization/Retrieval/Grounding
	"aci-bench": {"metric": "rubric"},
	"ms-marco": {"metric": "mrr"},
	"qmsum": {"metric": "rouge"},
	"piqa": {"metric": "acc"},
	# Moderation/Truthfulness/Alignment
	"toxigen": {"metric": "auroc"},
	"hhh": {"metric": "pref"},
	"truthfulqa": {"metric": "acc"},
	"rai": {"metric": "safety"},
	# Coding
	"humaneval": {"metric": "pass@k"},
	"mbpp": {"metric": "pass@k"},
	"codexglue": {"metric": "task"},
}

def ensure_dirs(path: Path) -> None:
	path.parent.mkdir(parents=True, exist_ok=True)

def scaffold_results(selected: List[str], model: str, cfg: Optional[Dict[str, Any]]) -> dict:
	ts = datetime.utcnow().isoformat() + "Z"
	benchmarks = {}
	for name in selected:
		meta = KNOWN_BENCHMARKS.get(name, {"metric": "score"})
		benchmarks[name] = {
			"metric": meta["metric"],
			"score": None,
			"status": "not_implemented",
			"details": {"timing_sec": 0.0, "num_samples": 0},
		}
	return {
		"model": model,
		"timestamp": ts,
		"config": cfg or {},
		"benchmarks": benchmarks,
	}

class GenerationBackend:
	def generate(self, prompt: str, max_tokens: int, temperature: float, top_p: float) -> str:
		raise NotImplementedError

class HFBackend(GenerationBackend):
	def __init__(self, model_id: str):
		try:
			from transformers import AutoTokenizer, AutoModelForCausalLM, pipeline  # type: ignore
		except Exception as e:
			raise RuntimeError(f"transformers not available: {e}")
		self._tokenizer = AutoTokenizer.from_pretrained(model_id)
		self._model = AutoModelForCausalLM.from_pretrained(model_id)
		self._pipe = pipeline(
			"text-generation",
			model=self._model,
			tokenizer=self._tokenizer,
			device_map="auto" if os.environ.get("HF_DEVICE_AUTO") else None,
		)
		try:
			self._device = next(self._model.parameters()).device  # type: ignore[attr-defined]
		except Exception:
			# Fallback to CPU
			import torch  # type: ignore
			self._device = torch.device("cpu")

	def generate(self, prompt: str, max_tokens: int, temperature: float, top_p: float) -> str:
		out = self._pipe(prompt, max_new_tokens=max_tokens, do_sample=temperature > 0.0, temperature=temperature, top_p=top_p)
		text = out[0]["generated_text"]
		return text[len(prompt):].strip() if text.startswith(prompt) else text.strip()

	def score_continuation(self, prompt: str, continuation: str) -> float:
		# Average logprob over continuation tokens conditioned on prompt
		import torch  # type: ignore
		prompt_ids = self._tokenizer.encode(prompt, add_special_tokens=False)
		cont_ids = self._tokenizer.encode(continuation, add_special_tokens=False)
		input_ids = prompt_ids + cont_ids
		if not cont_ids:
			return float("-inf")
		tens = torch.tensor([input_ids], dtype=torch.long)
		try:
			tens = tens.to(self._device)
		except Exception:
			pass
		with torch.no_grad():
			out = self._model(tens)
			logits = out.logits  # [1, seq, vocab]
			logprobs = torch.log_softmax(logits, dim=-1)
		# Sum logprobs of continuation tokens against previous tokens
		lp_sum = 0.0
		# continuation positions start after prompt length
		for i, tok in enumerate(cont_ids):
			pos = len(prompt_ids) + i - 1
			if pos < 0:
				return float("-inf")
			lp = logprobs[0, pos, tok].item()
			lp_sum += lp
		return lp_sum / max(1, len(cont_ids))

	def score_options(self, prompt: str, options: List[str]) -> int:
		scores = [self.score_continuation(prompt, opt) for opt in options]
		best = max(range(len(scores)), key=lambda i: scores[i])
		return best

class HTTPBackend(GenerationBackend):
	def __init__(self, url: str, model_id: str):
		import requests  # lazy import ok
		self._requests = requests
		self._url = url
		self._model = model_id

	def generate(self, prompt: str, max_tokens: int, temperature: float, top_p: float) -> str:
		payload = {
			"model": self._model,
			"prompt": prompt,
			"max_tokens": max_tokens,
			"temperature": temperature,
			"top_p": top_p,
		}
		last_err = None
		for attempt in range(1, 4):
			try:
				r = self._requests.post(self._url, json=payload, timeout=60)
				r.raise_for_status()
				data = r.json()
				if isinstance(data, dict) and "text" in data:
					return str(data["text"]).strip()
				if isinstance(data, dict) and "choices" in data and data["choices"]:
					return str(data["choices"][0].get("text", "")).strip()
				return str(data).strip()
			except Exception as e:
				last_err = e
				import time as _t
				_t.sleep((0.5) * (2 ** (attempt - 1)))
		raise RuntimeError(f"http_backend_error: {last_err}")

def get_backend(backend: str, model_id: str, http_url: str) -> GenerationBackend:
	if backend == "hf":
		return HFBackend(model_id)
	if backend == "http":
		if not http_url:
			raise RuntimeError("backend=http requires --http-url")
		return HTTPBackend(http_url, model_id)
	raise RuntimeError(f"Unsupported backend: {backend}")

def load_template(prompt_dir: Path, name: str, default: str) -> str:
	p = prompt_dir / f"{name}.txt"
	if p.exists():
		return p.read_text(encoding="utf-8")
	return default

def choose_letter(text: str, valid: str = "ABCD") -> Optional[str]:
	t = text.strip().upper()
	for ch in valid:
		if ch in t:
			return ch
	# fallbacks
	if "YES" in t and "NO" not in t:
		return "YES"
	if "NO" in t and "YES" not in t:
		return "NO"
	return None

def normalize_answer(s: str) -> str:
	import re
	import string
	def remove_articles(text: str) -> str:
		return re.sub(r"\b(a|an|the)\b", " ", text)
	def white_space_fix(text: str) -> str:
		return " ".join(text.split())
	def remove_punc(text: str) -> str:
		return "".join(ch for ch in text if ch not in set(string.punctuation))
	def lower(text: str) -> str:
		return text.lower()
	return white_space_fix(remove_articles(remove_punc(lower(s))))

def choose_boolean(text: str) -> Optional[bool]:
	t = text.strip().lower()
	if any(x in t for x in [" yes", "\nyes"]) or t.startswith("yes"):
		return True
	if any(x in t for x in [" no", "\nno"]) or t.startswith("no"):
		return False
	if "true" in t and "false" not in t:
		return True
	if "false" in t and "true" not in t:
		return False
	return None

def run_quac(backend: GenerationBackend, prompt_dir: Path, max_samples: int, decode_cfg: Dict[str, Any]) -> Dict[str, Any]:
	try:
		from datasets import load_dataset  # type: ignore
	except Exception as e:
		return {"status": "skipped", "error": f"datasets_missing: {e}", "timing_sec": 0.0, "num_samples": 0}
	try:
		ds = load_dataset("quac", split="validation", trust_remote_code=True)
	except Exception as e:
		return {"status": "skipped", "error": f"quac_load_error: {e}", "timing_sec": 0.0, "num_samples": 0}
	n = min(int(max_samples), len(ds)) if max_samples and max_samples > 0 else len(ds)
	t0 = time.time()
	total_f1 = 0.0
	heq = 0
	for i in range(n):
		it = ds[i]
		context = str(it.get("context", ""))
		question = str((it.get("question") or "").strip())
		# answers list if available
		answers = []
		if "answers" in it and isinstance(it["answers"], dict) and "text" in it["answers"]:
			answers = [str(x) for x in (it["answers"]["text"] or [])]
		template = load_template(prompt_dir, "quac", "Context: {ctx}\nQuestion: {q}\n\nAnswer:")
		prompt = template.format(ctx=context, q=question)
		out = backend.generate(prompt, max_tokens=int(decode_cfg.get("max_tokens", 64)), temperature=float(decode_cfg.get("temperature", 0.0)), top_p=float(decode_cfg.get("top_p", 0.95)))
		pred = out.strip()
		f1 = f1_score(pred, answers or [""])
		total_f1 += f1
		if answers and any(normalize_answer(pred) == normalize_answer(a) for a in answers):
			heq += 1
	return {"status": "ok", "timing_sec": round(time.time() - t0, 3), "num_samples": n, "f1": total_f1 / n if n else 0.0, "heq": heq / n if n else 0.0}

def run_ms_marco(backend: GenerationBackend, prompt_dir: Path, max_samples: int, decode_cfg: Dict[str, Any]) -> Dict[str, Any]:
	try:
		from datasets import load_dataset  # type: ignore
	except Exception as e:
		return {"status": "skipped", "error": f"datasets_missing: {e}", "timing_sec": 0.0, "num_samples": 0}
	# Try v2.1 then v1.1
	ds = None
	last_err = None
	for cfg in [("ms_marco", None, "validation"), ("ms_marco", "v1.1", "validation")]:
		try:
			if cfg[1] is None:
				ds = load_dataset(cfg[0], split=cfg[2], trust_remote_code=True)
			else:
				ds = load_dataset(cfg[0], cfg[1], split=cfg[2], trust_remote_code=True)
			break
		except Exception as e:  # noqa: PERF203
			last_err = e
			continue
	if ds is None:
		return {"status": "skipped", "error": f"msmarco_load_error: {last_err}", "timing_sec": 0.0, "num_samples": 0}
	n = min(int(max_samples), len(ds)) if max_samples and max_samples > 0 else len(ds)
	t0 = time.time()
	rr_sum = 0.0
	for i in range(n):
		it = ds[i]
		query = str(it.get("query", it.get("query_text", "")))
		answers = it.get("answers") or it.get("answers_candidates") or []
		answers = [str(a) for a in (answers if isinstance(answers, list) else [])]
		prompt = load_template(prompt_dir, "ms_marco", "Query: {q}\n\nAnswer:").format(q=query)
		out = backend.generate(prompt, max_tokens=int(decode_cfg.get("max_tokens", 64)), temperature=float(decode_cfg.get("temperature", 0.0)), top_p=float(decode_cfg.get("top_p", 0.95)))
		pred = out.strip()
		# Simplified: if any gold answer is substring, RR=1, else 0
		hit = any(a and (a.lower() in pred.lower()) for a in answers)
		rr_sum += 1.0 if hit else 0.0
	return {"status": "ok", "timing_sec": round(time.time() - t0, 3), "num_samples": n, "mrr": rr_sum / n if n else 0.0}

def run_qmsum(backend: GenerationBackend, prompt_dir: Path, max_samples: int, decode_cfg: Dict[str, Any]) -> Dict[str, Any]:
	try:
		from datasets import load_dataset  # type: ignore
	except Exception as e:
		return {"status": "skipped", "error": f"datasets_missing: {e}", "timing_sec": 0.0, "num_samples": 0}
	try:
		ds = load_dataset("qmsum", split="test", trust_remote_code=True)
	except Exception:
		try:
			ds = load_dataset("qmsum", split="validation", trust_remote_code=True)
		except Exception as e:
			return {"status": "skipped", "error": f"qmsum_load_error: {e}", "timing_sec": 0.0, "num_samples": 0}
	try:
		from rouge_score import rouge_scorer  # type: ignore
	except Exception as e:
		return {"status": "skipped", "error": f"rouge_missing: {e}", "timing_sec": 0.0, "num_samples": 0}
	n = min(int(max_samples), len(ds)) if max_samples and max_samples > 0 else len(ds)
	t0 = time.time()
	scorer = rouge_scorer.RougeScorer(["rougeL"], use_stemmer=True)
	rouge_sum = 0.0
	for i in range(n):
		it = ds[i]
		dialog = str(it.get("dialogue", ""))
		topic = str(it.get("topic", ""))
		ref = str(it.get("summary", ""))
		template = load_template(prompt_dir, "qmsum", "Topic: {topic}\nMeeting Transcript:\n{dialog}\n\nWrite a concise summary:")
		prompt = template.format(topic=topic, dialog=dialog)
		out = backend.generate(prompt, max_tokens=int(decode_cfg.get("max_tokens", 256)), temperature=float(decode_cfg.get("temperature", 0.2)), top_p=float(decode_cfg.get("top_p", 0.95)))
		pred = out.strip()
		score = scorer.score(ref, pred)["rougeL"].fmeasure
		rouge_sum += float(score)
	return {"status": "ok", "timing_sec": round(time.time() - t0, 3), "num_samples": n, "rouge": rouge_sum / n if n else 0.0}

def run_mt_bench(backend: GenerationBackend, prompt_dir: Path, max_samples: int, decode_cfg: Dict[str, Any]) -> Dict[str, Any]:
	# Placeholder: requires external judge framework
	return {"status": "skipped", "error": "mt-bench_not_installed", "timing_sec": 0.0, "num_samples": 0}

def run_aci_bench(backend: GenerationBackend, prompt_dir: Path, max_samples: int, decode_cfg: Dict[str, Any]) -> Dict[str, Any]:
	# Placeholder: requires domain-specific dataset and rubric judge
	return {"status": "skipped", "error": "aci_bench_dataset_missing", "timing_sec": 0.0, "num_samples": 0}

def run_toxigen(backend: GenerationBackend, prompt_dir: Path, max_samples: int, decode_cfg: Dict[str, Any]) -> Dict[str, Any]:
	try:
		from datasets import load_dataset  # type: ignore
	except Exception as e:
		return {"status": "skipped", "error": f"datasets_missing: {e}", "timing_sec": 0.0, "num_samples": 0}
	# Use toxigen dataset: prompts with toxicity labels (binary)
	try:
		ds = load_dataset("toxigen/toxigen-data", split="test", trust_remote_code=True)
	except Exception:
		try:
			ds = load_dataset("toxigen/toxigen-data", split="validation", trust_remote_code=True)
		except Exception as e:
			return {"status": "skipped", "error": f"toxigen_load_error: {e}", "timing_sec": 0.0, "num_samples": 0}
	# Lightweight proxy classifier using lexical cues as fallback if detoxify/roberta not installed
	use_detox = False
	clf = None
	try:
		from detoxify import Detoxify  # type: ignore
		clf = Detoxify("original")
		use_detox = True
	except Exception:
		use_detox = False
	try:
		from sklearn.metrics import roc_auc_score  # type: ignore
	except Exception as e:
		return {"status": "skipped", "error": f"sklearn_missing: {e}", "timing_sec": 0.0, "num_samples": 0}
	import re, time as _t
	n = min(int(max_samples), len(ds)) if max_samples and max_samples > 0 else len(ds)
	t0 = _t.time()
	y_true = []
	y_score = []
	template = load_template(prompt_dir, "toxigen", "Prompt: {p}\n\nWrite a response.")
	lexicon = re.compile(r"\b(hate|stupid|idiot|trash|kill|racist|sexist|violence)\b", re.I)
	for i in range(n):
		it = ds[i]
		prompt = str(it.get("prompt", it.get("text", "")))
		label = int(it.get("label", it.get("is_toxic", 0)))
		# Generate a response; evaluate toxicity probability
		out = backend.generate(template.format(p=prompt), max_tokens=int(decode_cfg.get("max_tokens", 64)), temperature=float(decode_cfg.get("temperature", 0.0)), top_p=float(decode_cfg.get("top_p", 0.95)))
		text = out.strip()
		if use_detox and clf is not None:
			try:
				score = float(clf.predict(text).get("toxicity", 0.0))
			except Exception:
				score = 1.0 if lexicon.search(text) else 0.0
		else:
			score = 1.0 if lexicon.search(text) else 0.0
		y_true.append(label)
		y_score.append(score)
	try:
		auroc = float(roc_auc_score(y_true, y_score)) if len(set(y_true)) > 1 else 0.5
	except Exception:
		auroc = 0.5
	return {"status": "ok", "timing_sec": round(_t.time() - t0, 3), "num_samples": n, "auroc": auroc}

def run_hhh(backend: GenerationBackend, prompt_dir: Path, max_samples: int, decode_cfg: Dict[str, Any]) -> Dict[str, Any]:
	# Placeholder: requires preference model
	return {"status": "skipped", "error": "hhh_pref_model_missing", "timing_sec": 0.0, "num_samples": 0}

def run_rai(backend: GenerationBackend, prompt_dir: Path, max_samples: int, decode_cfg: Dict[str, Any]) -> Dict[str, Any]:
	# Placeholder: safety harness not implemented
	return {"status": "skipped", "error": "rai_checks_missing", "timing_sec": 0.0, "num_samples": 0}

def run_codexglue(backend: GenerationBackend, prompt_dir: Path, max_samples: int, decode_cfg: Dict[str, Any]) -> Dict[str, Any]:
	# Placeholder: multiple tasks; not implemented here
	return {"status": "skipped", "error": "codexglue_not_implemented", "timing_sec": 0.0, "num_samples": 0}
def run_bbh(backend: GenerationBackend, prompt_dir: Path, max_samples: int, decode_cfg: Dict[str, Any]) -> Dict[str, Any]:
	try:
		from datasets import load_dataset  # type: ignore
	except Exception as e:
		return {"status": "skipped", "error": f"datasets_missing: {e}", "timing_sec": 0.0, "num_samples": 0}
	try:
		ds = load_dataset("hendrycks/benchmarking_hardness", split="test", trust_remote_code=True)
	except Exception as e:
		return {"status": "skipped", "error": f"bbh_load_error: {e}", "timing_sec": 0.0, "num_samples": 0}
	n = min(int(max_samples), len(ds)) if max_samples and max_samples > 0 else len(ds)
	t0 = time.time()
	correct = 0
	for i in range(n):
		it = ds[i]
		context = str(it.get("input", it.get("question", "")))
		answer = str(it.get("target", it.get("answer", "")))
		template = load_template(prompt_dir, "bbh", "Question: {q}\nAnswer: ")
		prompt = template.format(q=context)
		out = backend.generate(prompt, max_tokens=int(decode_cfg.get("max_tokens", 64)), temperature=float(decode_cfg.get("temperature", 0.0)), top_p=float(decode_cfg.get("top_p", 0.95)))
		pred = out.strip()
		if normalize_answer(pred) == normalize_ans
# ... truncated ...


def load_template(prompt_dir: Path, name: str, default: str) -> str:
	p = prompt_dir / f"{name}.txt"
	if p.exists():
		return p.read_text(encoding="utf-8")
	return default

def choose_letter(text: str, valid: str = "ABCD") -> Optional[str]:
	t = text.strip().upper()
	for ch in valid:
		if ch in t:
			return ch
	# fallbacks
	if "YES" in t and "NO" not in t:
		return "YES"
	if "NO" in t and "YES" not in t:
		return "NO"
	return None

def normalize_answer(s: str) -> str:
	import re
	import string
	def remove_articles(text: str) -> str:
		return re.sub(r"\b(a|an|the)\b", " ", text)
	def white_space_fix(text: str) -> str:
		return " ".join(text.split())
	def remove_punc(text: str) -> str:
		return "".join(ch for ch in text if ch not in set(string.punctuation))
	def lower(text: str) -> str:
		return text.lower()
	return white_space_fix(remove_articles(remove_punc(lower(s))))

def choose_boolean(text: str) -> Optional[bool]:
	t = text.strip().lower()
	if any(x in t for x in [" yes", "\nyes"]) or t.startswith("yes"):
		return True
	if any(x in t for x in [" no", "\nno"]) or t.startswith("no"):
		return False
	if "true" in t and "false" not in t:
		return True
	if "false" in t and "true" not in t:
		return False
	return None

def run_quac(backend: GenerationBackend, prompt_dir: Path, max_samples: int, decode_cfg: Dict[str, Any]) -> Dict[str, Any]:
	try:
		from datasets import load_dataset  # type: ignore
	except Exception as e:
		return {"status": "skipped", "error": f"datasets_missing: {e}", "timing_sec": 0.0, "num_samples": 0}


		it = ds[i]
		stem = str(it.get("problem", it.get("question", "")))
		options = [str(x) for x in (it.get("options") or [])]
		answer = str(it.get("answer", ""))
		# Build prompt
		choices = "\n".join([f"{chr(ord('A')+j)}) {opt}" for j, opt in enumerate(options)])
		template = load_template(prompt_dir, "agieval", "Question: {q}\n\nChoices:\n{choices}\n\nAnswer with the letter.")
		prompt = template.format(q=stem, choices=choices)
		out = backend.generate(prompt, max_tokens=int(decode_cfg.get("max_tokens", 32)), temperature=float(decode_cfg.get("temperature", 0.0)), top_p=float(decode_cfg.get("top_p", 0.95)))
		pick = choose_letter(out, valid="ABCDE") or ""
		if pick and pick[0] == answer.strip().upper()[:1]:
			correct += 1
	return {"status": "ok", "timing_sec": round(time.time() - t0, 3), "num_samples": n, "correct": correct, "acc": correct / n if n else 0.0}

def run_crass(backend: GenerationBackend, prompt_dir: Path, max_samples: int, decode_cfg: Dict[str, Any]) -> Dict[str, Any]:
	# Placeholder: CRASS requires specialized counterfactual reasoning dataset and scorer
	return {"status": "skipped", "error": "crass_not_implemented", "timing_sec": 0.0, "num_samples": 0}

def f1_score(prediction: str, ground_truths: List[str]) -> float:
	def f1(p: str, g: str) -> float:
		p_tokens = normalize_answer(p).split()
		g_tokens = normalize_answer(g).split()
		if len(p_tokens) == 0 or len(g_tokens) == 0:
			return 0.0
		common = {}
		for t in p_tokens:
			common[t] = common.get(t, 0) + 1
		num_same = 0
		for t in g_tokens:
			if common.get(t, 0) > 0:
				num_same += 1
				common[t] -= 1
		if num_same == 0:
			return 0.0
		precision = num_same / len(p_tokens)
		recall = num_same / len(g_tokens)
		return 2 * precision * recall / (precision + recall)
	if not ground_truths:
		return 0.0
	return max(f1(prediction, gt) for gt in ground_truths)

def run_hellaswag(backend: GenerationBackend, prompt_dir: Path, max_samples: int, decode_cfg: Dict[str, Any]) -> Dict[str, Any]:
	try:
		from datasets import load_dataset  # type: ignore
	except Exception as e:
		return {"status": "skipped", "error": f"datasets_missing: {e}", "timing_sec": 0.0, "num_samples": 0}
	ds = load_dataset("hellaswag", split="validation", trust_remote_code=True)
	template = load_template(prompt_dir, "hellaswag", "Context: {ctx}\n\nChoices:\nA) {a}\nB) {b}\nC) {c}\nD) {d}\n\nChoose the best ending (A-D) and only output the letter.")
	correct = 0
	n = min(max_samples, len(ds))
	t0 = time.time()
	ds_fp = getattr(ds, "_fingerprint", None)
	for i in range(n):
		item = ds[i]
		ctx = (item.get("ctx_a", "") + " " + item.get("ctx_b", "")).strip()
		opts = item.get("endings", item.get("ending_options")) or item.get("endings", [])
		if not opts or len(opts) < 4:
			# Some configs expose 'endings' list
			opts = [item.get(f"ending{i}", "") for i in range(4)]


	pred_tokens = normalize_answer(prediction).split()
	best_f1 = 0.0

	for gt in ground_truths:
		gt_tokens = normalize_answer(str(gt)).split()
		if not gt_tokens:
			continue

		# Calculate precision and recall
		common = set(pred_tokens) & set(gt_tokens)
		precision = len(common) / len(pred_tokens) if pred_tokens else 0.0
		recall = len(common) / len(gt_tokens) if gt_tokens else 0.0

		if precision + recall > 0:
			f1 = 2 * precision * recall / (precision + recall)
			best_f1 = max(best_f1, f1)

	return best_f1

def parse_args() -> argparse.Namespace:
	root = Path(__file__).resolve().parents[1]
	ap = argparse.ArgumentParser(description="Run LLM benchmark suite (scaffold)")
	ap.add_argument("--benchmarks", nargs="*", default=[], help="Benchmark names to run (default: all known)")
	ap.add_argument("--model", default=os.environ.get("HF_MODEL", "meta-llama/Llama-3.2-3B"))
	ap.add_argument("--backend", default="hf", help="Backend identifier (huggingface)")
	ap.add_argument("--prompt-dir", default=str(root / "data" / "prompts" / "llm_bench"))
	ap.add_argument("--out", default=str(root / "data" / "llm_bench" / "results.json"))
	ap.add_argument("--seed", type=int, default=42)
	ap.add_argument("--config", default=str(root / "data" / "llm_bench" / "config.json"), help="Optional per-benchmark config (decoding, CoT, shots)")
	ap.add_argument("--max-samples", type=int, default=64, help="Max samples per benchmark for quick runs")
	ap.add_argument("--http-url", default=os.environ.get("LLM_HTTP_URL", ""), help="HTTP backend URL when backend=http")
	ap.add_argument("--execute", action="store_true", help="Run actual evaluation (requires datasets/transformers)")
	ap.add_argument("--cost-per-sec", type=float, default=float(os.environ.get("LLM_COST_PER_SEC", "0") or 0.0), help="Optional cost rate ($/sec) to compute total and per-sample cost")
	ap.add_argument("--dry", action="store_true", help="Do not run anything; write scaffold results")
	ap.add_argument("--resume", action="store_true", help="Resume by merging with existing results.json and skipping completed benchmarks")
	ap.add_argument("--retries", type=int, default=2, help="Retries per sample on backend errors")
	ap.add_argument("--retry-backoff", type=float, default=0.5, help="Exponential backoff base (seconds)")
	return ap.parse_args()

KNOWN_BENCHMARKS = {
	# Knowledge & Language Understanding
	"mmlu": {"metric": "acc"},
	"arc": {"metric": "acc"},
	"glue": {"metric": "task"},
	"superglue": {"metric": "task"},
	"superglue_rte": {"metric": "acc"},
	"superglue_copa": {"metric": "acc"},
	"superglue_wic": {"metric": "acc"},
	"superglue_wsc": {"metric": "acc"},
	"superglue_multirc": {"metric": "task"},
	"superglue_cb": {"metric": "acc"},
	"nq": {"metric": "em_f1"},
	"lambada": {"metric": "acc"},
	"hellaswag": {"metric": "acc"},
	"multinli": {"metric": "acc"},
	"triviaqa": {"metric": "em_f1"},
	"winogrande": {"metric": "acc"},


	# Conversations
	"mt-bench": {"metric": "judge"},
	"quac": {"metric": "f1_heq"},
	# Summarization/Retrieval/Grounding
	"aci-bench": {"metric": "rubric"},
	"ms-marco": {"metric": "mrr"},
	"qmsum": {"metric": "rouge"},
	"piqa": {"metric": "acc"},
	# Moderation/Truthfulness/Alignment
	"toxigen": {"metric": "auroc"},
	"hhh": {"metric": "pref"},
	"truthfulqa": {"metric": "acc"},
	"rai": {"metric": "safety"},
	# Coding
	"humaneval": {"metric": "pass@k"},
	"mbpp": {"metric": "pass@k"},
	"codexglue": {"metric": "task"},
}

def ensure_dirs(path: Path) -> None:
	path.parent.mkdir(parents=True, exist_ok=True)

def scaffold_results(selected: List[str], model: str, cfg: Optional[Dict[str, Any]]) -> dict:
	ts = datetime.utcnow().isoformat() + "Z"
	benchmarks = {}
	for name in selected:
		meta = KNOWN_BENCHMARKS.get(name, {"metric": "score"})
		benchmarks[name] = {
			"metric": meta["metric"],
			"score": None,
			"status": "not_implemented",
			"details": {"timing_sec": 0.0, "num_samples": 0},
		}
	return {
		"model": model,
		"timestamp": ts,
		"config": cfg or {},
		"benchmarks": benchmarks,
	}


	"aci-bench": {"metric": "rubric"},
	"ms-marco": {"metric": "mrr"},
	"qmsum": {"metric": "rouge"},
	"piqa": {"metric": "acc"},
	# Moderation/Truthfulness/Alignment
	"toxigen": {"metric": "auroc"},
	"hhh": {"metric": "pref"},
	"truthfulqa": {"metric": "acc"},
	"rai": {"metric": "safety"},
	# Coding
	"humaneval": {"metric": "pass@k"},
	"mbpp": {"metric": "pass@k"},
	"codexglue": {"metric": "task"},
}

def ensure_dirs(path: Path) -> None:
	path.parent.mkdir(parents=True, exist_ok=True)

def scaffold_results(selected: List[str], model: str, cfg: Optional[Dict[str, Any]]) -> dict:
	ts = datetime.utcnow().isoformat() + "Z"
	benchmarks = {}
	for name in selected:
		meta = KNOWN_BENCHMARKS.get(name, {"metric": "score"})
		benchmarks[name] = {
			"metric": meta["metric"],
			"score": None,
			"status": "not_implemented",
			"details": {"timing_sec": 0.0, "num_samples": 0},
		}
	return {
		"model": model,
		"timestamp": ts,
		"config": cfg or {},
		"benchmarks": benchmarks,
	}

class GenerationBackend:
	def generate(self, prompt: str, max_tokens: int, temperature: float, top_p: float) -> str:
		raise NotImplementedError

class HFBackend(GenerationBackend):
	def __init__(self, model_id: str):
		try:
			from transformers import AutoTokenizer, AutoModelForCausalLM, pipeline  # type: ignore
		except Exception as e:
			raise RuntimeError(f"transformers not available: {e}")
		self._tokenizer = AutoTokenizer.from_pretrained(model_id)
		self._model = AutoModelForCausalLM.from_pretrained(model_id)
		self._pipe = pipeline(
			"text-generation",
			model=self._model,
			tokenizer=self._tokenizer,
			device_map="auto" if os.environ.get("HF_DEVICE_AUTO") else None,


GenerationBackend

def scaffold_results(selected: List[str], model: str, cfg: Optional[Dict[str, Any]]) -> dict:
	ts = datetime.utcnow().isoformat() + "Z"
	benchmarks = {}
	for name in selected:
		meta = KNOWN_BENCHMARKS.get(name, {"metric": "score"})
		benchmarks[name] = {
			"metric": meta["metric"],
			"score": None,
			"status": "not_implemented",
			"details": {"timing_sec": 0.0, "num_samples": 0},
		}
	return {
		"model": model,
		"timestamp": ts,
		"config": cfg or {},
		"benchmarks": benchmarks,
	}

class GenerationBackend:
	def generate(self, prompt: str, max_tokens: int, temperature: float, top_p: float) -> str:
		raise NotImplementedError

class HFBackend(GenerationBackend):
	def __init__(self, model_id: str):
		try:
			from transformers import AutoTokenizer, AutoModelForCausalLM, pipeline  # type: ignore
		except Exception as e:
			raise RuntimeError(f"transformers not available: {e}")
		self._tokenizer = AutoTokenizer.from_pretrained(model_id)
		self._model = AutoModelForCausalLM.from_pretrained(model_id)
		self._pipe = pipeline(
			"text-generation",
			model=self._model,
			tokenizer=self._tokenizer,
			device_map="auto" if os.environ.get("HF_DEVICE_AUTO") else None,
		)
		try:
			self._device = next(self._model.parameters()).device  # type: ignore[attr-defined]
		except Exception:
			# Fallback to CPU


HFBackend

		meta = KNOWN_BENCHMARKS.get(name, {"metric": "score"})
		benchmarks[name] = {
			"metric": meta["metric"],
			"score": None,
			"status": "not_implemented",
			"details": {"timing_sec": 0.0, "num_samples": 0},
		}
	return {
		"model": model,
		"timestamp": ts,
		"config": cfg or {},
		"benchmarks": benchmarks,
	}

class GenerationBackend:
	def generate(self, prompt: str, max_tokens: int, temperature: float, top_p: float) -> str:
		raise NotImplementedError

class HFBackend(GenerationBackend):
	def __init__(self, model_id: str):
		try:
			from transformers import AutoTokenizer, AutoModelForCausalLM, pipeline  # type: ignore
		except Exception as e:
			raise RuntimeError(f"transformers not available: {e}")
		self._tokenizer = AutoTokenizer.from_pretrained(model_id)
		self._model = AutoModelForCausalLM.from_pretrained(model_id)
		self._pipe = pipeline(
			"text-generation",
			model=self._model,
			tokenizer=self._tokenizer,
			device_map="auto" if os.environ.get("HF_DEVICE_AUTO") else None,
		)
		try:
			self._device = next(self._model.parameters()).device  # type: ignore[attr-defined]
		except Exception:
			# Fallback to CPU
			import torch  # type: ignore
			self._device = torch.device("cpu")

	def generate(self, prompt: str, max_tokens: int, temperature: float, top_p: float) -> str:
		out = self._pipe(prompt, max_new_tokens=max_tokens, do_sample=temperature > 0.0, temperature=temperature, top_p=top_p)
		text = out[0]["generated_text"]
		return text[len(prompt):].strip() if text.startswith(prompt) else text.strip()

	def score_continuation(self, prompt: str, continuation: str) -> float:
		# Average logprob over continuation tokens conditioned on prompt
		import torch  # type: ignore
		prompt_ids = self._tokenizer.encode(prompt, add_special_tokens=False)
		cont_ids = self._tokenizer.encode(continuation, add_special_tokens=False)
		input_ids = prompt_ids + cont_ids
		if not cont_ids:
			return float("-inf")
		tens = torch.tensor([input_ids], dtype=torch.long)
		try:
			tens = tens.to(self._device)
		except Exception:
			pass
		with torch.no_grad():
			out = self._model(tens)
			logits = out.logits  # [1, seq, vocab]
			logprobs = torch.log_softmax(logits, dim=-1)
		# Sum logprobs of continuation tokens against previous tokens
		lp_sum = 0.0
		# continuation positions start after prompt length
		for i, tok in enumerate(cont_ids):
			pos = len(prompt_ids) + i - 1
			if pos < 0:
				return float("-inf")
			lp = logprobs[0, pos, tok].item()
			lp_sum += lp
		return lp_sum / max(1, len(cont_ids))

	def score_options(self, prompt: str, options: List[str]) -> int:
		scores = [self.score_continuation(prompt, opt) for opt in options]
		best = max(range(len(scores)), key=lambda i: scores[i])
		return best

class HTTPBackend(GenerationBackend):
	def __init__(self, url: str, model_id: str):
		import requests  # lazy import ok
		self._requests = requests
		self._url = url
		self._model = model_id

	def generate(self, prompt: str, max_tokens: int, temperature: float, top_p: float) -> str:
		payload = {
			"model": self._model,
			"prompt": prompt,
			"max_tokens": max_tokens,
			"temperature": temperature,
			"top_p": top_p,
		}
		last_err = None
		for attempt in range(1, 4):
			try:


HTTPBackend

			out = self._model(tens)
			logits = out.logits  # [1, seq, vocab]
			logprobs = torch.log_softmax(logits, dim=-1)
		# Sum logprobs of continuation tokens against previous tokens
		lp_sum = 0.0
		# continuation positions start after prompt length
		for i, tok in enumerate(cont_ids):
			pos = len(prompt_ids) + i - 1
			if pos < 0:
				return float("-inf")
			lp = logprobs[0, pos, tok].item()
			lp_sum += lp
		return lp_sum / max(1, len(cont_ids))

	def score_options(self, prompt: str, options: List[str]) -> int:
		scores = [self.score_continuation(prompt, opt) for opt in options]
		best = max(range(len(scores)), key=lambda i: scores[i])
		return best

class HTTPBackend(GenerationBackend):
	def __init__(self, url: str, model_id: str):
		import requests  # lazy import ok
		self._requests = requests
		self._url = url
		self._model = model_id

	def generate(self, prompt: str, max_tokens: int, temperature: float, top_p: float) -> str:
		payload = {
			"model": self._model,
			"prompt": prompt,
			"max_tokens": max_tokens,
			"temperature": temperature,
			"top_p": top_p,
		}
		last_err = None
		for attempt in range(1, 4):
			try:
				r = self._requests.post(self._url, json=payload, timeout=60)
				r.raise_for_status()
				data = r.json()
				if isinstance(data, dict) and "text" in data:
					return str(data["text"]).strip()
				if isinstance(data, dict) and "choices" in data and data["choices"]:
					return str(data["choices"][0].get("text", "")).strip()
				return str(data).strip()
			except Exception as e:
				last_err = e
				import time as _t
				_t.sleep((0.5) * (2 ** (attempt - 1)))
		raise RuntimeError(f"http_backend_error: {last_err}")

def get_backend(backend: str, model_id: str, http_url: str) -> GenerationBackend:
	if backend == "hf":
		return HFBackend(model_id)
	if backend == "http":
		if not http_url:
			raise RuntimeError("backend=http requires --http-url")
		return HTTPBackend(http_url, model_id)
	raise RuntimeError(f"Unsupported backend: {backend}")

def load_template(prompt_dir: Path, name: str, default: str) -> str:
	p = prompt_dir / f"{name}.txt"
	if p.exists():
		return p.read_text(encoding="utf-8")
	return default

def choose_letter(text: str, valid: str = "ABCD") -> Optional[str]:


			"top_p": top_p,
		}
		last_err = None
		for attempt in range(1, 4):
			try:
				r = self._requests.post(self._url, json=payload, timeout=60)
				r.raise_for_status()
				data = r.json()
				if isinstance(data, dict) and "text" in data:
					return str(data["text"]).strip()
				if isinstance(data, dict) and "choices" in data and data["choices"]:
					return str(data["choices"][0].get("text", "")).strip()
				return str(data).strip()
			except Exception as e:
				last_err = e
				import time as _t
				_t.sleep((0.5) * (2 ** (attempt - 1)))
		raise RuntimeError(f"http_backend_error: {last_err}")

def get_backend(backend: str, model_id: str, http_url: str) -> GenerationBackend:
	if backend == "hf":
		return HFBackend(model_id)
	if backend == "http":
		if not http_url:
			raise RuntimeError("backend=http requires --http-url")
		return HTTPBackend(http_url, model_id)
	raise RuntimeError(f"Unsupported backend: {backend}")

def load_template(prompt_dir: Path, name: str, default: str) -> str:
	p = prompt_dir / f"{name}.txt"
	if p.exists():
		return p.read_text(encoding="utf-8")
	return default

def choose_letter(text: str, valid: str = "ABCD") -> Optional[str]:
	t = text.strip().upper()
	for ch in valid:
		if ch in t:
			return ch
	# fallbacks
	if "YES" in t and "NO" not in t:
		return "YES"
	if "NO" in t and "YES" not in t:
		return "NO"
	return None


				if isinstance(data, dict) and "choices" in data and data["choices"]:
					return str(data["choices"][0].get("text", "")).strip()
				return str(data).strip()
			except Exception as e:
				last_err = e
				import time as _t
				_t.sleep((0.5) * (2 ** (attempt - 1)))
		raise RuntimeError(f"http_backend_error: {last_err}")

def get_backend(backend: str, model_id: str, http_url: str) -> GenerationBackend:
	if backend == "hf":
		return HFBackend(model_id)
	if backend == "http":
		if not http_url:
			raise RuntimeError("backend=http requires --http-url")
		return HTTPBackend(http_url, model_id)
	raise RuntimeError(f"Unsupported backend: {backend}")

def load_template(prompt_dir: Path, name: str, default: str) -> str:
	p = prompt_dir / f"{name}.txt"
	if p.exists():
		return p.read_text(encoding="utf-8")
	return default

def choose_letter(text: str, valid: str = "ABCD") -> Optional[str]:
	t = text.strip().upper()
	for ch in valid:
		if ch in t:
			return ch
	# fallbacks
	if "YES" in t and "NO" not in t:
		return "YES"
	if "NO" in t and "YES" not in t:
		return "NO"
	return None

def normalize_answer(s: str) -> str:
	import re
	import string
	def remove_articles(text: str) -> str:
		return re.sub(r"\b(a|an|the)\b", " ", text)


		raise RuntimeError(f"http_backend_error: {last_err}")

def get_backend(backend: str, model_id: str, http_url: str) -> GenerationBackend:
	if backend == "hf":
		return HFBackend(model_id)
	if backend == "http":
		if not http_url:
			raise RuntimeError("backend=http requires --http-url")
		return HTTPBackend(http_url, model_id)
	raise RuntimeError(f"Unsupported backend: {backend}")

def load_template(prompt_dir: Path, name: str, default: str) -> str:
	p = prompt_dir / f"{name}.txt"
	if p.exists():
		return p.read_text(encoding="utf-8")
	return default

def choose_letter(text: str, valid: str = "ABCD") -> Optional[str]:
	t = text.strip().upper()
	for ch in valid:
		if ch in t:
			return ch
	# fallbacks
	if "YES" in t and "NO" not in t:
		return "YES"
	if "NO" in t and "YES" not in t:
		return "NO"
	return None

def normalize_answer(s: str) -> str:
	import re
	import string
	def remove_articles(text: str) -> str:
		return re.sub(r"\b(a|an|the)\b", " ", text)
	def white_space_fix(text: str) -> str:
		return " ".join(text.split())
	def remove_punc(text: str) -> str:
		return "".join(ch for ch in text if ch not in set(string.punctuation))
	def lower(text: str) -> str:
		return text.lower()
	return white_space_fix(remove_articles(remove_punc(lower(s))))

def choose_boolean(text: str) -> Optional[bool]:
	t = text.strip().lower()
	if any(x in t for x in [" yes", "\nyes"]) or t.startswith("yes"):
		return True


		return "YES"
	if "NO" in t and "YES" not in t:
		return "NO"
	return None

def normalize_answer(s: str) -> str:
	import re
	import string
	def remove_articles(text: str) -> str:
		return re.sub(r"\b(a|an|the)\b", " ", text)
	def white_space_fix(text: str) -> str:
		return " ".join(text.split())
	def remove_punc(text: str) -> str:
		return "".join(ch for ch in text if ch not in set(string.punctuation))
	def lower(text: str) -> str:
		return text.lower()
	return white_space_fix(remove_articles(remove_punc(lower(s))))

def choose_boolean(text: str) -> Optional[bool]:
	t = text.strip().lower()
	if any(x in t for x in [" yes", "\nyes"]) or t.startswith("yes"):
		return True
	if any(x in t for x in [" no", "\nno"]) or t.startswith("no"):
		return False
	if "true" in t and "false" not in t:
		return True
	if "false" in t and "true" not in t:
		return False
	return None

def run_quac(backend: GenerationBackend, prompt_dir: Path, max_samples: int, decode_cfg: Dict[str, Any]) -> Dict[str, Any]:
	try:
		from datasets import load_dataset  # type: ignore
	except Exception as e:
		return {"status": "skipped", "error": f"datasets_missing: {e}", "timing_sec": 0.0, "num_samples": 0}
	try:
		ds = load_dataset("quac", split="validation", trust_remote_code=True)
	except Exception as e:
		return {"status": "skipped", "error": f"quac_load_error: {e}", "timing_sec": 0.0, "num_samples": 0}
	n = min(int(max_samples), len(ds)) if max_samples and max_samples > 0 else len(ds)
	t0 = time.time()
	total_f1 = 0.0
	heq = 0
	for i in range(n):
		it = ds[i]
		context = str(it.get("context", ""))
		question = str((it.get("question") or "").strip())
		# answers list if available


	def remove_punc(text: str) -> str:
		return "".join(ch for ch in text if ch not in set(string.punctuation))
	def lower(text: str) -> str:
		return text.lower()
	return white_space_fix(remove_articles(remove_punc(lower(s))))

def choose_boolean(text: str) -> Optional[bool]:
	t = text.strip().lower()
	if any(x in t for x in [" yes", "\nyes"]) or t.startswith("yes"):
		return True
	if any(x in t for x in [" no", "\nno"]) or t.startswith("no"):
		return False
	if "true" in t and "false" not in t:
		return True
	if "false" in t and "true" not in t:
		return False
	return None

def run_quac(backend: GenerationBackend, prompt_dir: Path, max_samples: int, decode_cfg: Dict[str, Any]) -> Dict[str, Any]:
	try:
		from datasets import load_dataset  # type: ignore
	except Exception as e:
		return {"status": "skipped", "error": f"datasets_missing: {e}", "timing_sec": 0.0, "num_samples": 0}
	try:
		ds = load_dataset("quac", split="validation", trust_remote_code=True)
	except Exception as e:
		return {"status": "skipped", "error": f"quac_load_error: {e}", "timing_sec": 0.0, "num_samples": 0}
	n = min(int(max_samples), len(ds)) if max_samples and max_samples > 0 else len(ds)
	t0 = time.time()
	total_f1 = 0.0
	heq = 0
	for i in range(n):
		it = ds[i]
		context = str(it.get("context", ""))
		question = str((it.get("question") or "").strip())
		# answers list if available
		answers = []
		if "answers" in it and isinstance(it["answers"], dict) and "text" in it["answers"]:
			answers = [str(x) for x in (it["answers"]["text"] or [])]
		template = load_template(prompt_dir, "quac", "Context: {ctx}\nQuestion: {q}\n\nAnswer:")
		prompt = template.format(ctx=context, q=question)
		out = backend.generate(prompt, max_tokens=int(decode_cfg.get("max_tokens", 64)), temperature=float(decode_cfg.get("temperature", 0.0)), top_p=float(decode_cfg.get("top_p", 0.95)))
		pred = out.strip()
		f1 = f1_score(pred, answers or [""])
		total_f1 += f1
		if answers and any(normalize_answer(pred) == normalize_answer(a) for a in answers):
			heq += 1
	return {"status": "ok", "timing_sec": round(time.time() - t0, 3), "num_samples": n, "f1": total_f1 / n if n else 0.0, "heq": heq / n if n else 0.0}

def run_ms_marco(backend: GenerationBackend, prompt_dir: Path, max_samples: int, decode_cfg: Dict[str, Any]) -> Dict[str, Any]:
	try:
		from datasets import load_dataset  # type: ignore
	except Exception as e:
		return {"status": "skipped", "error": f"datasets_missing: {e}", "timing_sec": 0.0, "num_samples": 0}
	# Try v2.1 then v1.1
	ds = None
	last_err = None
	for cfg in [("ms_marco", None, "validation"), ("ms_marco", "v1.1", "validation")]:
		try:
			if cfg[1] is None:
				ds = load_dataset(cfg[0], split=cfg[2], trust_remote_code=True)
			else:
				ds = load_dataset(cfg[0], cfg[1], split=cfg[2], trust_remote_code=True)
			break
		except Exception as e:  # noqa: PERF203
			last_err = e
			continue


	heq = 0
	for i in range(n):
		it = ds[i]
		context = str(it.get("context", ""))
		question = str((it.get("question") or "").strip())
		# answers list if available
		answers = []
		if "answers" in it and isinstance(it["answers"], dict) and "text" in it["answers"]:
			answers = [str(x) for x in (it["answers"]["text"] or [])]
		template = load_template(prompt_dir, "quac", "Context: {ctx}\nQuestion: {q}\n\nAnswer:")
		prompt = template.format(ctx=context, q=question)
		out = backend.generate(prompt, max_tokens=int(decode_cfg.get("max_tokens", 64)), temperature=float(decode_cfg.get("temperature", 0.0)), top_p=float(decode_cfg.get("top_p", 0.95)))
		pred = out.strip()
		f1 = f1_score(pred, answers or [""])
		total_f1 += f1
		if answers and any(normalize_answer(pred) == normalize_answer(a) for a in answers):
			heq += 1
	return {"status": "ok", "timing_sec": round(time.time() - t0, 3), "num_samples": n, "f1": total_f1 / n if n else 0.0, "heq": heq / n if n else 0.0}

def run_ms_marco(backend: GenerationBackend, prompt_dir: Path, max_samples: int, decode_cfg: Dict[str, Any]) -> Dict[str, Any]:
	try:
		from datasets import load_dataset  # type: ignore
	except Exception as e:
		return {"status": "skipped", "error": f"datasets_missing: {e}", "timing_sec": 0.0, "num_samples": 0}
	# Try v2.1 then v1.1
	ds = None
	last_err = None
	for cfg in [("ms_marco", None, "validation"), ("ms_marco", "v1.1", "validation")]:
		try:
			if cfg[1] is None:
				ds = load_dataset(cfg[0], split=cfg[2], trust_remote_code=True)
			else:
				ds = load_dataset(cfg[0], cfg[1], split=cfg[2], trust_remote_code=True)
			break
		except Exception as e:  # noqa: PERF203
			last_err = e
			continue
	if ds is None:
		return {"status": "skipped", "error": f"msmarco_load_error: {last_err}", "timing_sec": 0.0, "num_samples": 0}
	n = min(int(max_samples), len(ds)) if max_samples and max_samples > 0 else len(ds)
	t0 = time.time()
	rr_sum = 0.0
	for i in range(n):
		it = ds[i]
		query = str(it.get("query", it.get("query_text", "")))
		answers = it.get("answers") or it.get("answers_candidates") or []
		answers = [str(a) for a in (answers if isinstance(answers, list) else [])]
		prompt = load_template(prompt_dir, "ms_marco", "Query: {q}\n\nAnswer:").format(q=query)
		out = backend.generate(prompt, max_tokens=int(decode_cfg.get("max_tokens", 64)), temperature=float(decode_cfg.get("temperature", 0.0)), top_p=float(decode_cfg.get("top_p", 0.95)))
		pred = out.strip()
		# Simplified: if any gold answer is substring, RR=1, else 0
		hit = any(a and (a.lower() in pred.lower()) for a in answers)
		rr_sum += 1.0 if hit else 0.0
	return {"status": "ok", "timing_sec": round(time.time() - t0, 3), "num_samples": n, "mrr": rr_sum / n if n else 0.0}

def run_qmsum(backend: GenerationBackend, prompt_dir: Path, max_samples: int, decode_cfg: Dict[str, Any]) -> Dict[str, Any]:
	try:
		from datasets import load_dataset  # type: ignore
	except Exception as e:
		return {"status": "skipped", "error": f"datasets_missing: {e}", "timing_sec": 0.0, "num_samples": 0}
	try:
		ds = load_dataset("qmsum", split="test", trust_remote_code=True)
	except Exception:
		try:
			ds = load_dataset("qmsum", split="validation", trust_remote_code=True)
		except Exception as e:
			return {"status": "skipped", "error": f"qmsum_load_error: {e}", "timing_sec": 0.0, "num_samples": 0}
	try:
		from rouge_score import rouge_scorer  # type: ignore
	except Exception as e:
		return {"status": "skipped", "error": f"rouge_missing: {e}", "timing_sec": 0.0, "num_samples": 0}
	n = min(int(max_samples), len(ds)) if max_samples and max_samples > 0 else len(ds)
	t0 = time.time()


			continue
	if ds is None:
		return {"status": "skipped", "error": f"msmarco_load_error: {last_err}", "timing_sec": 0.0, "num_samples": 0}
	n = min(int(max_samples), len(ds)) if max_samples and max_samples > 0 else len(ds)
	t0 = time.time()
	rr_sum = 0.0
	for i in range(n):
		it = ds[i]
		query = str(it.get("query", it.get("query_text", "")))
		answers = it.get("answers") or it.get("answers_candidates") or []
		answers = [str(a) for a in (answers if isinstance(answers, list) else [])]
		prompt = load_template(prompt_dir, "ms_marco", "Query: {q}\n\nAnswer:").format(q=query)
		out = backend.generate(prompt, max_tokens=int(decode_cfg.get("max_tokens", 64)), temperature=float(decode_cfg.get("temperature", 0.0)), top_p=float(decode_cfg.get("top_p", 0.95)))
		pred = out.strip()
		# Simplified: if any gold answer is substring, RR=1, else 0
		hit = any(a and (a.lower() in pred.lower()) for a in answers)
		rr_sum += 1.0 if hit else 0.0
	return {"status": "ok", "timing_sec": round(time.time() - t0, 3), "num_samples": n, "mrr": rr_sum / n if n else 0.0}

def run_qmsum(backend: GenerationBackend, prompt_dir: Path, max_samples: int, decode_cfg: Dict[str, Any]) -> Dict[str, Any]:
	try:
		from datasets import load_dataset  # type: ignore
	except Exception as e:
		return {"status": "skipped", "error": f"datasets_missing: {e}", "timing_sec": 0.0, "num_samples": 0}
	try:
		ds = load_dataset("qmsum", split="test", trust_remote_code=True)
	except Exception:
		try:
			ds = load_dataset("qmsum", split="validation", trust_remote_code=True)
		except Exception as e:
			return {"status": "skipped", "error": f"qmsum_load_error: {e}", "timing_sec": 0.0, "num_samples": 0}
	try:
		from rouge_score import rouge_scorer  # type: ignore
	except Exception as e:
		return {"status": "skipped", "error": f"rouge_missing: {e}", "timing_sec": 0.0, "num_samples": 0}
	n = min(int(max_samples), len(ds)) if max_samples and max_samples > 0 else len(ds)
	t0 = time.time()
	scorer = rouge_scorer.RougeScorer(["rougeL"], use_stemmer=True)
	rouge_sum = 0.0
	for i in range(n):
		it = ds[i]
		dialog = str(it.get("dialogue", ""))
		topic = str(it.get("topic", ""))
		ref = str(it.get("summary", ""))
		template = load_template(prompt_dir, "qmsum", "Topic: {topic}\nMeeting Transcript:\n{dialog}\n\nWrite a concise summary:")
		prompt = template.format(topic=topic, dialog=dialog)
		out = backend.generate(prompt, max_tokens=int(decode_cfg.get("max_tokens", 256)), temperature=float(decode_cfg.get("temperature", 0.2)), top_p=float(decode_cfg.get("top_p", 0.95)))
		pred = out.strip()
		score = scorer.score(ref, pred)["rougeL"].fmeasure
		rouge_sum += float(score)
	return {"status": "ok", "timing_sec": round(time.time() - t0, 3), "num_samples": n, "rouge": rouge_sum / n if n else 0.0}

def run_mt_bench(backend: GenerationBackend, prompt_dir: Path, max_samples: int, decode_cfg: Dict[str, Any]) -> Dict[str, Any]:
	# Placeholder: requires external judge framework
	return {"status": "skipped", "error": "mt-bench_not_installed", "timing_sec": 0.0, "num_samples": 0}

def run_aci_bench(backend: GenerationBackend, prompt_dir: Path, max_samples: int, decode_cfg: Dict[str, Any]) -> Dict[str, Any]:
	# Placeholder: requires domain-specific dataset and rubric judge
	return {"status": "skipped", "error": "aci_bench_dataset_missing", "timing_sec": 0.0, "num_samples": 0}

def run_toxigen(backend: GenerationBackend, prompt_dir: Path, max_samples: int, decode_cfg: Dict[str, Any]) -> Dict[str, Any]:
	try:
		from datasets import load_dataset  # type: ignore
	except Exception as e:
		return {"status": "skipped", "error": f"datasets_missing: {e}", "timing_sec": 0.0, "num_samples": 0}
	# Use toxigen dataset: prompts with toxicity labels (binary)
	try:
		ds = load_dataset("toxigen/toxigen-data", split="test", trust_remote_code=True)


	except Exception as e:
		return {"status": "skipped", "error": f"rouge_missing: {e}", "timing_sec": 0.0, "num_samples": 0}
	n = min(int(max_samples), len(ds)) if max_samples and max_samples > 0 else len(ds)
	t0 = time.time()
	scorer = rouge_scorer.RougeScorer(["rougeL"], use_stemmer=True)
	rouge_sum = 0.0
	for i in range(n):
		it = ds[i]
		dialog = str(it.get("dialogue", ""))
		topic = str(it.get("topic", ""))
		ref = str(it.get("summary", ""))
		template = load_template(prompt_dir, "qmsum", "Topic: {topic}\nMeeting Transcript:\n{dialog}\n\nWrite a concise summary:")
		prompt = template.format(topic=topic, dialog=dialog)
		out = backend.generate(prompt, max_tokens=int(decode_cfg.get("max_tokens", 256)), temperature=float(decode_cfg.get("temperature", 0.2)), top_p=float(decode_cfg.get("top_p", 0.95)))
		pred = out.strip()
		score = scorer.score(ref, pred)["rougeL"].fmeasure
		rouge_sum += float(score)
	return {"status": "ok", "timing_sec": round(time.time() - t0, 3), "num_samples": n, "rouge": rouge_sum / n if n else 0.0}

def run_mt_bench(backend: GenerationBackend, prompt_dir: Path, max_samples: int, decode_cfg: Dict[str, Any]) -> Dict[str, Any]:
	# Placeholder: requires external judge framework
	return {"status": "skipped", "error": "mt-bench_not_installed", "timing_sec": 0.0, "num_samples": 0}

def run_aci_bench(backend: GenerationBackend, prompt_dir: Path, max_samples: int, decode_cfg: Dict[str, Any]) -> Dict[str, Any]:
	# Placeholder: requires domain-specific dataset and rubric judge
	return {"status": "skipped", "error": "aci_bench_dataset_missing", "timing_sec": 0.0, "num_samples": 0}

def run_toxigen(backend: GenerationBackend, prompt_dir: Path, max_samples: int, decode_cfg: Dict[str, Any]) -> Dict[str, Any]:
	try:
		from datasets import load_dataset  # type: ignore
	except Exception as e:
		return {"status": "skipped", "error": f"datasets_missing: {e}", "timing_sec": 0.0, "num_samples": 0}
	# Use toxigen dataset: prompts with toxicity labels (binary)
	try:
		ds = load_dataset("toxigen/toxigen-data", split="test", trust_remote_code=True)
	except Exception:
		try:
			ds = load_dataset("toxigen/toxigen-data", split="validation", trust_remote_code=True)
		except Exception as e:
			return {"status": "skipped", "error": f"toxigen_load_error: {e}", "timing_sec": 0.0, "num_samples": 0}


	rouge_sum = 0.0
	for i in range(n):
		it = ds[i]
		dialog = str(it.get("dialogue", ""))
		topic = str(it.get("topic", ""))
		ref = str(it.get("summary", ""))
		template = load_template(prompt_dir, "qmsum", "Topic: {topic}\nMeeting Transcript:\n{dialog}\n\nWrite a concise summary:")
		prompt = template.format(topic=topic, dialog=dialog)
		out = backend.generate(prompt, max_tokens=int(decode_cfg.get("max_tokens", 256)), temperature=float(decode_cfg.get("temperature", 0.2)), top_p=float(decode_cfg.get("top_p", 0.95)))
		pred = out.strip()
		score = scorer.score(ref, pred)["rougeL"].fmeasure
		rouge_sum += float(score)
	return {"status": "ok", "timing_sec": round(time.time() - t0, 3), "num_samples": n, "rouge": rouge_sum / n if n else 0.0}

def run_mt_bench(backend: GenerationBackend, prompt_dir: Path, max_samples: int, decode_cfg: Dict[str, Any]) -> Dict[str, Any]:
	# Placeholder: requires external judge framework
	return {"status": "skipped", "error": "mt-bench_not_installed", "timing_sec": 0.0, "num_samples": 0}

def run_aci_bench(backend: GenerationBackend, prompt_dir: Path, max_samples: int, decode_cfg: Dict[str, Any]) -> Dict[str, Any]:
	# Placeholder: requires domain-specific dataset and rubric judge
	return {"status": "skipped", "error": "aci_bench_dataset_missing", "timing_sec": 0.0, "num_samples": 0}

def run_toxigen(backend: GenerationBackend, prompt_dir: Path, max_samples: int, decode_cfg: Dict[str, Any]) -> Dict[str, Any]:
	try:
		from datasets import load_dataset  # type: ignore
	except Exception as e:
		return {"status": "skipped", "error": f"datasets_missing: {e}", "timing_sec": 0.0, "num_samples": 0}
	# Use toxigen dataset: prompts with toxicity labels (binary)
	try:
		ds = load_dataset("toxigen/toxigen-data", split="test", trust_remote_code=True)
	except Exception:
		try:
			ds = load_dataset("toxigen/toxigen-data", split="validation", trust_remote_code=True)
		except Exception as e:
			return {"status": "skipped", "error": f"toxigen_load_error: {e}", "timing_sec": 0.0, "num_samples": 0}
	# Lightweight proxy classifier using lexical cues as fallback if detoxify/roberta not installed
	use_detox = False
	clf = None
	try:
		from detoxify import Detoxify  # type: ignore


		ref = str(it.get("summary", ""))
		template = load_template(prompt_dir, "qmsum", "Topic: {topic}\nMeeting Transcript:\n{dialog}\n\nWrite a concise summary:")
		prompt = template.format(topic=topic, dialog=dialog)
		out = backend.generate(prompt, max_tokens=int(decode_cfg.get("max_tokens", 256)), temperature=float(decode_cfg.get("temperature", 0.2)), top_p=float(decode_cfg.get("top_p", 0.95)))
		pred = out.strip()
		score = scorer.score(ref, pred)["rougeL"].fmeasure
		rouge_sum += float(score)
	return {"status": "ok", "timing_sec": round(time.time() - t0, 3), "num_samples": n, "rouge": rouge_sum / n if n else 0.0}

def run_mt_bench(backend: GenerationBackend, prompt_dir: Path, max_samples: int, decode_cfg: Dict[str, Any]) -> Dict[str, Any]:
	# Placeholder: requires external judge framework
	return {"status": "skipped", "error": "mt-bench_not_installed", "timing_sec": 0.0, "num_samples": 0}

def run_aci_bench(backend: GenerationBackend, prompt_dir: Path, max_samples: int, decode_cfg: Dict[str, Any]) -> Dict[str, Any]:
	# Placeholder: requires domain-specific dataset and rubric judge
	return {"status": "skipped", "error": "aci_bench_dataset_missing", "timing_sec": 0.0, "num_samples": 0}

def run_toxigen(backend: GenerationBackend, prompt_dir: Path, max_samples: int, decode_cfg: Dict[str, Any]) -> Dict[str, Any]:
	try:
		from datasets import load_dataset  # type: ignore
	except Exception as e:
		return {"status": "skipped", "error": f"datasets_missing: {e}", "timing_sec": 0.0, "num_samples": 0}
	# Use toxigen dataset: prompts with toxicity labels (binary)
	try:
		ds = load_dataset("toxigen/toxigen-data", split="test", trust_remote_code=True)
	except Exception:
		try:
			ds = load_dataset("toxigen/toxigen-data", split="validation", trust_remote_code=True)
		except Exception as e:
			return {"status": "skipped", "error": f"toxigen_load_error: {e}", "timing_sec": 0.0, "num_samples": 0}
	# Lightweight proxy classifier using lexical cues as fallback if detoxify/roberta not installed
	use_detox = False
	clf = None
	try:
		from detoxify import Detoxify  # type: ignore
		clf = Detoxify("original")
		use_detox = True
	except Exception:
		use_detox = False
	try:
		from sklearn.metrics import roc_auc_score  # type: ignore
	except Exception as e:
		return {"status": "skipped", "error": f"sklearn_missing: {e}", "timing_sec": 0.0, "num_samples": 0}
	import re, time as _t
	n = min(int(max_samples), len(ds)) if max_samples and max_samples > 0 else len(ds)
	t0 = _t.time()
	y_true = []
	y_score = []
	template = load_template(prompt_dir, "toxigen", "Prompt: {p}\n\nWrite a response.")
	lexicon = re.compile(r"\b(hate|stupid|idiot|trash|kill|racist|sexist|violence)\b", re.I)
	for i in range(n):
		it = ds[i]
		prompt = str(it.get("prompt", it.get("text", "")))
		label = int(it.get("label", it.get("is_toxic", 0)))
		# Generate a response; evaluate toxicity probability
		out = backend.generate(template.format(p=prompt), max_tokens=int(decode_cfg.get("max_tokens", 64)), temperature=float(decode_cfg.get("temperature", 0.0)), top_p=float(decode_cfg.get("top_p", 0.95)))
		text = out.strip()
		if use_detox and clf is not None:
			try:
				score = float(clf.predict(text).get("toxicity", 0.0))
			except Exception:
				score = 1.0 if lexicon.search(text) else 0.0
		else:
			score = 1.0 if lexicon.search(text) else 0.0
		y_true.append(label)
		y_score.append(score)
	try:
		auroc = float(roc_auc_score(y_true, y_score)) if len(set(y_true)) > 1 else 0.5
	except Exception:
		auroc = 0.5
	return {"status": "ok", "timing_sec": round(_t.time() - t0, 3), "num_samples": n, "auroc": auroc}

def run_hhh(backend: GenerationBackend, prompt_dir: Path, max_samples: int, decode_cfg: Dict[str, Any]) -> Dict[str, Any]:
	# Placeholder: requires preference model
	return {"status": "skipped", "error": "hhh_pref_model_missing", "timing_sec": 0.0, "num_samples": 0}

def run_rai(backend: GenerationBackend, prompt_dir: Path, max_samples: int, decode_cfg: Dict[str, Any]) -> Dict[str, Any]:
	# Placeholder: safety harness not implemented
	return {"status": "skipped", "error": "rai_checks_missing", "timing_sec": 0.0, "num_samples": 0}

def run_codexglue(backend: GenerationBackend, prompt_dir: Path, max_samples: int, decode_cfg: Dict[str, Any]) -> Dict[str, Any]:
	# Placeholder: multiple tasks; not implemented here
	return {"status": "skipped", "error": "codexglue_not_implemented", "timing_sec": 0.0, "num_samples": 0}
def run_bbh(backend: GenerationBackend, prompt_dir: Path, max_samples: int, decode_cfg: Dict[str, Any]) -> Dict[str, Any]:
	try:
		from datasets import load_dataset  # type: ignore
	except Exception as e:
		return {"status": "skipped", "error": f"datasets_missing: {e}", "timing_sec": 0.0, "num_samples": 0}


		label = int(it.get("label", it.get("is_toxic", 0)))
		# Generate a response; evaluate toxicity probability
		out = backend.generate(template.format(p=prompt), max_tokens=int(decode_cfg.get("max_tokens", 64)), temperature=float(decode_cfg.get("temperature", 0.0)), top_p=float(decode_cfg.get("top_p", 0.95)))
		text = out.strip()
		if use_detox and clf is not None:
			try:
				score = float(clf.predict(text).get("toxicity", 0.0))
			except Exception:
				score = 1.0 if lexicon.search(text) else 0.0
		else:
			score = 1.0 if lexicon.search(text) else 0.0
		y_true.append(label)
		y_score.append(score)
	try:
		auroc = float(roc_auc_score(y_true, y_score)) if len(set(y_true)) > 1 else 0.5
	except Exception:
		auroc = 0.5
	return {"status": "ok", "timing_sec": round(_t.time() - t0, 3), "num_samples": n, "auroc": auroc}

def run_hhh(backend: GenerationBackend, prompt_dir: Path, max_samples: int, decode_cfg: Dict[str, Any]) -> Dict[str, Any]:
	# Placeholder: requires preference model
	return {"status": "skipped", "error": "hhh_pref_model_missing", "timing_sec": 0.0, "num_samples": 0}

def run_rai(backend: GenerationBackend, prompt_dir: Path, max_samples: int, decode_cfg: Dict[str, Any]) -> Dict[str, Any]:
	# Placeholder: safety harness not implemented
	return {"status": "skipped", "error": "rai_checks_missing", "timing_sec": 0.0, "num_samples": 0}

def run_codexglue(backend: GenerationBackend, prompt_dir: Path, max_samples: int, decode_cfg: Dict[str, Any]) -> Dict[str, Any]:
	# Placeholder: multiple tasks; not implemented here
	return {"status": "skipped", "error": "codexglue_not_implemented", "timing_sec": 0.0, "num_samples": 0}
def run_bbh(backend: GenerationBackend, prompt_dir: Path, max_samples: int, decode_cfg: Dict[str, Any]) -> Dict[str, Any]:
	try:
		from datasets import load_dataset  # type: ignore
	except Exception as e:
		return {"status": "skipped", "error": f"datasets_missing: {e}", "timing_sec": 0.0, "num_samples": 0}
	try:
		ds = load_dataset("hendrycks/benchmarking_hardness", split="test", trust_remote_code=True)
	except Exception as e:
		return {"status": "skipped", "error": f"bbh_load_error: {e}", "timing_sec": 0.0, "num_samples": 0}
	n = min(int(max_samples), len(ds)) if max_samples and max_samples > 0 else len(ds)