	results = {
		"ok": bool(ok),
		"aggregate": {"router": router_sr, "t5": t5_sr, "nn": nn_sr, "best": best_sr},
		"max_drop": float(args.max_drop),
	}

	if args.per_task:
		# Build per-task rates from counts
		per_task = summary.get("per_task", {}) if isinstance(summary, dict) else {}
		# Keys like "router:count_lines"
		tasks = set()
		for k in per_task.keys():
			try:
				act, task = k.split(":", 1)
			except Exception:
				continue
			tasks.add(task)
		pt = {}
		per_ok = True
		for task in sorted(tasks):
			def get_counts(act: str) -> tuple[float, float]:
				st = per_task.get(f"{act}:{task}", {}) or {}
				return float(st.get("n", 0.0)), float(st.get("success", 0.0))

			n_r, s_r = get_counts("router")
			n_t, s_t = get_counts("t5")
			n_n, s_n = get_counts("nn")
			r = rate(n_r, s_r)
			best = max(rate(n_t, s_t), rate(n_n, s_n))
			pt[task] = {"router": r, "best": best}
			if (r + 1e-9) < (best - float(args.max_drop)):
				per_ok = False
		results["per_task"] = pt
		ok = ok and per_ok
		results["ok"] = bool(ok)

	print(json.dumps(results))
	return 0 if ok else 1

if __name__ == "__main__":
	raise SystemExit(main())


import argparse
import json
from pathlib import Path
from typing import Dict, Any
import sys

def main() -> int:
	ap = argparse.ArgumentParser(description="Evaluate structured DOM decoding (Outlines/constraints)")
	root = Path(__file__).resolve().parents[1]
	ap.add_argument("--model", default=str(root / "models" / "actuator_dom_t5"))
	ap.add_argument("--data", default=str(root / "data" / "skills" / "actuator_il_dom.jsonl"))
	ap.add_argument("--use-outlines", action="store_true")
	args = ap.parse_args()

	# Robust import: add repo root (containing 'agi_dw') to sys.path if needed
	try:
		import os as _os  # type: ignore
		from agi_dw.core.actuator.service import ActuatorConfig, RouterExtras, select_action  # type: ignore
	except ModuleNotFoundError:
		try:
			cur = Path(__file__).resolve().parent
			repo_root = None
			for _ in range(6):
				if (cur / "agi_dw").is_dir():
					repo_root = cur
					break
				if cur.parent == cur:
					break
				cur = cur.parent
			if repo_root is None:
				repo_root = Path.cwd()
			pp = str(repo_root)
			if pp not in sys.path:
				sys.path.insert(0, pp)
			import os as _os  # type: ignore
			from agi_dw.core.actuator.service import ActuatorConfig, RouterExtras, select_action  # type: ignore
		except Exception as e:
			print(json.dumps({"ok": False, "error": f"import_error: {e}"}))
			return 1

	if bool(args.use_outlines):
		_os.environ["AGI_DOM_OUTLINES"] = "1"

	rows = []
	# Resolve dataset path; try provided file, then common alternates
	ds_path = Path(args.data)
	candidates = [ds_path]
	if not ds_path.is_file():
		# Try common skills dataset fallbacks
		alt1 = Path(__file__).resolve().parents[1] / "data" / "skills" / "actuator_il_dom.jsonl"
		alt2 = Path.cwd() / "data" / "skills" / "actuator_il_dom.jsonl"
		candidates = [p for p in [ds_path, alt1, alt2] if p]
	loaded = False
	for p in candidates:
		try:
			if p.is_file():
				with p.open("r", encoding="utf-8") as f:
					for line in f:
						line = line.strip()
						if not line:
							continue
						try:
							rows.append(json.loads(line))
						except Exception:
							continue
				loaded = True
				break
		except Exception:
			continue
	if not rows:
		print(json.dumps({"ok": False, "error": "no_data", "candidates": [str(p) for p in candidates]}))
		return 0

	# Centralized actuator service config for DOM structured decoding
	cfg = ActuatorConfig(mode="t5", t5_model=str(args.model), dom_structured=True)
	extra = RouterExtras(domain="dom")

	total = 0
	both_acc = 0
	for r in rows:
		inp = r.get("input")
		gold = r.get("output")
		if not isinstance(inp, str) or not isinstance(gold, str):
			continue
		try:
			g = json.loads(gold)
		except Exception:
			g = {}
		g_args = (g.get("args") or {}) if isinstance(g, dict) else {}
		g_url = str(g_args.get("url", ""))
		g_sel = str(g_args.get("selector", ""))
		obs = {"kind": "dom", "meta": {"url": g_url, "selector": g_sel}, "content": inp}
		act, _ = select_action(obs, {"goal": "eval"}, cfg, extra)
		a_args = (act.get("args") or {}) if isinstance(act, dict) else {}
		p_url = str(a_args.get("url", ""))
		p_sel = str(a_args.get("selector", ""))
		ok = (p_url.strip().rstrip('/') == g_url.strip().rstrip('/')) and (" ".join(p_sel.split()) == " ".join(g_sel.split()))
		both_acc += 1 if ok else 0
		total += 1

	out = {"ok": True, "total": int(total), "both_acc": float(both_acc / total) if total else 0.0}
	print(json.dumps(out, ensure_ascii=False))
	return 0

if __name__ == "__main__":
	raise SystemExit(main())


import logging
import argparse
import json
import sys

def main() -> int:
	ap = argparse.ArgumentParser()
	ap.add_argument("--min-router-prob-mean", type=float, default=0.0)
	ap.add_argument("--max-router-prob-mean", type=float, default=1.0)
	ap.add_argument("--min-picks-t5", type=int, default=0)
	ap.add_argument("--min-picks-nn", type=int, default=0)
	args = ap.parse_args()

	data = sys.stdin.read().strip()
	if not data:
		print("No summary on stdin", file=sys.stderr)
		return 2
	try:
		summary = json.loads(data.splitlines()[-1])
	except Exception as e:
		print(f"Failed to parse summary JSON: {e}", file=sys.stderr)
		return 2

	router = summary.get("router", {}) if isinstance(summary, dict) else {}
	prob_mean = float(router.get("router_prob_mean", 0.0))
	picks = router.get("router_picks", {}) if isinstance(router, dict) else {}
	picks_t5 = int(picks.get("t5", 0))
	picks_nn = int(picks.get("nn", 0))

	print(json.dumps({
		"router_prob_mean": prob_mean,
		"min_mean": args.min_router_prob_mean,
		"max_mean": args.max_router_prob_mean,
		"picks_t5": picks_t5,
		"picks_nn": picks_nn,
	}))

	if prob_mean + 1e-9 < args.min_router_prob_mean:
		print("Router prob mean below min", file=sys.stderr)
		return 1
	if prob_mean - 1e-9 > args.max_router_prob_mean:
		print("Router prob mean above max", file=sys.stderr)
		return 1
	if picks_t5 < args.min_picks_t5:
		print("Router picked t5 fewer times than expected", file=sys.stderr)
		return 1
	if picks_nn < args.min_picks_nn:
		print("Router picked nn fewer times than expected", file=sys.stderr)
		return 1
	return 0

if __name__ == "__main__":
	raise SystemExit(main())


import logging
import argparse
import json
from pathlib import Path

def main() -> int:
	ap = argparse.ArgumentParser()
	root = Path(__file__).resolve().parents[1]
	ap.add_argument("--report", default=str(root / "data" / "benchmarks" / "wm_planrank.json"))
	ap.add_argument("--min-delta", type=float, default=0.0, help="Minimum success_rate improvement vs baseline")
	args = ap.parse_args()

	p = Path(args.report)
	if not p.exists():
		print(json.dumps({"ok": False, "error": "report_missing", "path": str(p)}))
		return 1
	obj = json.loads(p.read_text(encoding="utf-8"))
	base = obj.get("baseline", {}) or {}
	wmrk = obj.get("wm_planrank", {}) or {}
	ok = True
	per_task = {}
	for task, b in base.items():
		br = float((b or {}).get("success_rate", 0.0))
		wr = float((wmrk.get(task) or {}).get("success_rate", 0.0))
		delta = wr - br
		per_task[task] = {"baseline": br, "wm": wr, "delta": delta}
		if delta < float(args.min_delta):
			ok = False
	print(json.dumps({"ok": ok, "min_delta": float(args.min_delta), "per_task": per_task}))
	return 0 if ok else 1

if __name__ == "__main__":
	raise SystemExit(main())


import logging
import argparse
import json
from pathlib import Path
from typing import Any, Dict, Optional

def load_first_verified(root: Path) -> Optional[Dict[str, Any]]:
	for rel in [
		root / "data" / "traces" / "seed_os_cli.verified.llm.jsonl",
		root / "data" / "traces" / "web_dom.verified.jsonl",
	]:
		if not rel.exists():
			continue
		try:
			for line in rel.read_text(encoding="utf-8").splitlines():
				line = line.strip()
				if not line:
					continue
				return json.loads(line)
		except Exception:
			continue
	return None

def main() -> int:
	ap = argparse.ArgumentParser()
	root = Path(__file__).resolve().parents[1]
	ap.add_argument("--model", default=str(root / "models" / "wm_mlp" / "wm_mlp.joblib"))
	ap.add_argument("--metrics", default=str(root / "models" / "wm_mlp" / "metrics.json"))
	ap.add_argument("--min-auc", type=float, default=0.6)
	ap.add_argument("--max-ece", type=float, default=0.2)
	ap.add_argument("--rollback", action="store_true", help="If thresholds fail, replace model with best.joblib if present")
	args = ap.parse_args()

	model_path = Path(args.model)
	if not model_path.exists():
		print(json.dumps({"ok": False, "error": "model_missing", "model": str(model_path)}))
		return 1

	# If metrics exist, enforce thresholds
	metrics_path = Path(args.metrics)
	if metrics_path.exists():
		try:
			m = json.loads(metrics_path.read_text(encoding="utf-8"))
			auc = float(m.get("wm_auc", -1))
			ece = float(m.get("wm_ece", 1e9))
			ok = bool(auc >= float(args.min_auc) and ece <= float(args.max_ece))
			if ok:
				print(json.dumps({"ok": True, "metrics": m, "thresholds": {"min_auc": float(args.min_auc), "max_ece": float(args.max_ece)}}))
				return 0
			# Thresholds failed; optionally rollback to best
			if args.rollback:
				best_path = model_path.parent / "best.joblib"
				if best_path.exists():
					# swap best -> current
					model_path.write_bytes(best_path.read_bytes())
					print(json.dumps({"ok": True, "rolled_back": True, "to": str(best_path), "metrics": m}))
					return 0
		except Exception:
			pass

	sample = load_first_verified(root)
	if not isinstance(sample, dict):
		print(json.dumps({"ok": False, "error": "no_verified_sample"}))
		return 2

	try:
		from agi_dw.core.world_model.api import WorldModelPrior  # type: ignore
		wm = WorldModelPrior.load(model_path)
		obs = sample.get("obs", {})
		plan = sample.get("plan", {})
		action = sample.get("action", {})
		prior = wm.predict_prior(obs, plan, action)
		ok = bool(prior and 0.0 <= float(prior.get("success_prob", -1)) <= 1.0 and 0.0 <= float(prior.get("risk", -1)) <= 1.0)
		print(json.dumps({"ok": ok, "prior": prior or {}, "model": str(model_path)}))
		return 0 if ok else 1
	except Exception as e:
		print(json.dumps({"ok": False, "error": str(e)}))
		return 1

if __name__ == "__main__":
	raise SystemExit(main())


import logging
import argparse
import json
from pathlib import Path
from typing import Any, Dict, Optional

def load_first_verified(root: Path) -> Optional[Dict[str, Any]]:
	for rel in [
		root / "data" / "traces" / "seed_os_cli.verified.llm.jsonl",
		root / "data" / "traces" / "web_dom.verified.jsonl",
	]:
		if not rel.exists():
			continue
		try:
			for line in rel.read_text(encoding="utf-8").splitlines():
				line = line.strip()
				if not line:
					continue
				return json.loads(line)
		except Exception:
			continue
	return None

def main() -> int:
	ap = argparse.ArgumentParser()
	root = Path(__file__).resolve().parents[1]
	ap.add_argument("--model", default=str(root / "models" / "wm_mlp" / "wm_mlp.joblib"))
	ap.add_argument("--metrics", default=str(root / "models" / "wm_mlp" / "metrics.json"))
	ap.add_argument("--min-auc", type=float, default=0.6)
	ap.add_argument("--max-ece", type=float, default=0.2)
	ap.add_argument("--rollback", action="store_true", help="If thresholds fail, replace model with best.joblib if present")
	args = ap.parse_args()

	model_path = Path(args.model)
	if not model_path.exists():
		print(json.dumps({"ok": False, "error": "model_missing", "model": str(model_path)}))
		return 1

	# If metrics exist, enforce thresholds
	metrics_path = Path(args.metrics)
	if metrics_path.exists():


main

def load_first_verified(root: Path) -> Optional[Dict[str, Any]]:
	for rel in [
		root / "data" / "traces" / "seed_os_cli.verified.llm.jsonl",
		root / "data" / "traces" / "web_dom.verified.jsonl",
	]:
		if not rel.exists():
			continue
		try:
			for line in rel.read_text(encoding="utf-8").splitlines():
				line = line.strip()
				if not line:
					continue
				return json.loads(line)
		except Exception:
			continue
	return None

def main() -> int:
	ap = argparse.ArgumentParser()
	root = Path(__file__).resolve().parents[1]
	ap.add_argument("--model", default=str(root / "models" / "wm_mlp" / "wm_mlp.joblib"))
	ap.add_argument("--metrics", default=str(root / "models" / "wm_mlp" / "metrics.json"))
	ap.add_argument("--min-auc", type=float, default=0.6)
	ap.add_argument("--max-ece", type=float, default=0.2)
	ap.add_argument("--rollback", action="store_true", help="If thresholds fail, replace model with best.joblib if present")
	args = ap.parse_args()

	model_path = Path(args.model)
	if not model_path.exists():
		print(json.dumps({"ok": False, "error": "model_missing", "model": str(model_path)}))
		return 1

	# If metrics exist, enforce thresholds
	metrics_path = Path(args.metrics)
	if metrics_path.exists():
		try:
			m = json.loads(metrics_path.read_text(encoding="utf-8"))
			auc = float(m.get("wm_auc", -1))
			ece = float(m.get("wm_ece", 1e9))
			ok = bool(auc >= float(args.min_auc) and ece <= float(args.max_ece))
			if ok:
				print(json.dumps({"ok": True, "metrics": m, "thresholds": {"min_auc": float(args.min_auc), "max_ece": float(args.max_ece)}}))
				return 0
			# Thresholds failed; optionally rollback to best
			if args.rollback:
				best_path = model_path.parent / "best.joblib"
				if best_path.exists():
					# swap best -> current
					model_path.write_bytes(best_path.read_bytes())
					print(json.dumps({"ok": True, "rolled_back": True, "to": str(best_path), "metrics": m}))
					return 0
		except Exception:
			pass

	sample = load_first_verified(root)
	if not isinstance(sample, dict):
		print(json.dumps({"ok": False, "error": "no_verified_sample"}))
		return 2

	try:
		from agi_dw.core.world_model.api import WorldModelPrior  # type: ignore
		wm = WorldModelPrior.load(model_path)
		obs = sample.get("obs", {})
		plan = sample.get("plan", {})
		action = sample.get("action", {})
		prior = wm.predict_prior(obs, plan, action)
		ok = bool(prior and 0.0 <= float(prior.get("success_prob", -1)) <= 1.0 and 0.0 <= float(prior.get("risk", -1)) <= 1.0)
		print(json.dumps({"ok": ok, "prior": prior or {}, "model": str(model_path)}))
		return 0 if ok else 1
	except Exception as e:
		print(json.dumps({"ok": False, "error": str(e)}))
		return 1

if __name__ == "__main__":
	raise SystemExit(main())


#!/usr/bin/env python3
import logging
import argparse
import json
import os
from pathlib import Path

def parse_args():
	root = Path(__file__).resolve().parents[1]
	ap = argparse.ArgumentParser(description="CI gate for multi-language code style/type violations")
	ap.add_argument("--style", default=str(root / "data" / "llm_bench" / "code_style_multi.json"))
	ap.add_argument("--max-eslint", type=int, default=int(os.environ.get("MAX_CODE_ESLINT", "0") or 0))
	ap.add_argument("--max-tsc", type=int, default=int(os.environ.get("MAX_CODE_TSC", "0") or 0))
	ap.add_argument("--max-cpplint", type=int, default=int(os.environ.get("MAX_CODE_CPPLINT", "0") or 0))
	ap.add_argument("--max-javac", type=int, default=int(os.environ.get("MAX_CODE_JAVAC", "0") or 0))
	return ap.parse_args()

def main() -> int:
	args = parse_args()
	p = Path(args.style)
	if not p.exists():
		print(json.dumps({"ok": False, "error": "style_summary_missing", "path": str(p)}))
		return 1
	try:
		obj = json.loads(p.read_text(encoding="utf-8"))
	except Exception as e:
		print(json.dumps({"ok": False, "error": "style_summary_parse_error", "detail": str(e)}))
		return 1

	f = obj.get("violations", {})
	failures = []
	if args.max_eslint >= 0 and int(f.get("eslint", 0)) > args.max_eslint:
		failures.append({"tool": "eslint", "count": int(f.get("eslint", 0)), "max": args.max_eslint})
	if args.max_tsc >= 0 and int(f.get("tsc", 0)) > args.max_tsc:
		failures.append({"tool": "tsc", "count": int(f.get("tsc", 0)), "max": args.max_tsc})
	if args.max_cpplint >= 0 and int(f.get("cpplint", 0)) > args.max_cpplint:
		failures.append({"tool": "cpplint", "count": int(f.get("cpplint", 0)), "max": args.max_cpplint})
	if args.max_javac >= 0 and int(f.get("javac", 0)) > args.max_javac:
		failures.append({"tool": "javac", "count": int(f.get("javac", 0)), "max": args.max_javac})

	ok = len(failures) == 0
	print(json.dumps({"ok": ok, "failures": failures}))
	return 0 if ok else 1

if __name__ == "__main__":
	raise SystemExit(main())


#!/usr/bin/env python3
import logging
import argparse
import json
import os
from pathlib import Path

def parse_args():
	root = Path(__file__).resolve().parents[1]
	ap = argparse.ArgumentParser(description="CI gate for multi-language code style/type violations")
	ap.add_argument("--style", default=str(root / "data" / "llm_bench" / "code_style_multi.json"))
	ap.add_argument("--max-eslint", type=int, default=int(os.environ.get("MAX_CODE_ESLINT", "0") or 0))
	ap.add_argument("--max-tsc", type=int, default=int(os.environ.get("MAX_CODE_TSC", "0") or 0))
	ap.add_argument("--max-cpplint", type=int, default=int(os.environ.get("MAX_CODE_CPPLINT", "0") or 0))
	ap.add_argument("--max-javac", type=int, default=int(os.environ.get("MAX_CODE_JAVAC", "0") or 0))
	return ap.parse_args()

def main() -> int:
	args = parse_args()
	p = Path(args.style)
	if not p.exists():
		print(json.dumps({"ok": False, "error": "style_summary_missing", "path": str(p)}))
		return 1
	try:
		obj = json.loads(p.read_text(encoding="utf-8"))
	except Exception as e:
		print(json.dumps({"ok": False, "error": "style_summary_parse_error", "detail": str(e)}))
		return 1

	f = obj.get("violations", {})
	failures = []
	if args.max_eslint >= 0 and int(f.get("eslint", 0)) > args.max_eslint:
		failures.append({"tool": "eslint", "count": int(f.get("eslint", 0)), "max": args.max_eslint})
	if args.max_tsc >= 0 and int(f.get("tsc", 0)) > args.max_tsc:
		failures.append({"tool": "tsc", "count": int(f.get("tsc", 0)), "max": args.max_tsc})


import logging
import argparse
import json
import re
from pathlib import Path
from typing import List, Dict, Any
from concurrent.futures import ThreadPoolExecutor, TimeoutError

import torch
from datasets import load_dataset
from transformers import AutoTokenizer, AutoModelForSeq2SeqLM
try:
	# Prefer normal import when PYTHONPATH includes repo root
	from agi_dw.bench.web_dom.runner import fetch_text
except ModuleNotFoundError:
	# Robust fallback: add repo root (containing 'agi_dw') to sys.path
	import sys  # type: ignore
	cur = Path(__file__).resolve().parent
	repo_root = None
	for _ in range(6):
		if (cur / "agi_dw").is_dir():
			repo_root = cur
			break
		if cur.parent == cur:
			break
		cur = cur.parent
	if repo_root is None:
		repo_root = Path.cwd()
	pp = str(repo_root)
	if pp not in sys.path:
		sys.path.insert(0, pp)
	from agi_dw.bench.web_dom.runner import fetch_text  # type: ignore

DOM_INSTRUCTION = (
	'DOM task: Return ONLY two tokens: the URL and the CSS selector, separated by a single space. '
	'Example: https://example.com h1. No quotes, no explanations, no extra text. Input follows:\n'
)

def main() -> int:
	parser = argparse.ArgumentParser()
	root = Path(__file__).resolve().parents[1]
	parser.add_argument("--data", default=str(root / "data" / "skills" / "actuator_il_dom.jsonl"))
	parser.add_argument("--model", default=str(root / "models" / "actuator_dom_t5"))
	parser.add_argument("--max-new", type=int, default=32)
	parser.add_argument("--batch", type=int, default=8)
	parser.add_argument("--num-beams", type=int, default=1, help="Beam search width for generation")
	parser.add_argument("--normalize", action="store_true", help="Normalize URLs/selectors before comparing")
	parser.add_argument("--exec-validate", action="store_true", help="Treat selector as correct if predicted selector fetches non-empty text")
	parser.add_argument("--exec-timeout", type=int, default=6, help="Timeout (seconds) per execution validation fetch")
	parser.add_argument("--progress-every", type=int, default=25, help="Log progress every N examples")
	parser.add_argument("--report", default="", help="Optional path to write a JSONL report of per-example results")
	parser.add_argument("--show-failures", action="store_true", help="Print failing examples to stdout")
	args = parser.parse_args()

	ds = load_dataset("json", data_files={"test": args.data})
	tok = AutoTokenizer.from_pretrained(args.model)
	model = AutoModelForSeq2SeqLM.from_pretrained(args.model)
	device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
	if device.type == "cuda":
		model.half()
	model.to(device)
	model.eval()

	total = 0
	url_acc = 0
	sel_acc = 0
	both_acc = 0
	invalid_parse = 0
	exec_fail = 0
	report_rows: List[Dict[str, Any]] = []

	test_ds = ds["test"]
	n = len(test_ds)
	for i in range(0, n, args.batch):
		rows = test_ds[i : i + args.batch]
		if isinstance(rows, dict):
			inputs_raw = [DOM_INSTRUCTION + x for x in rows["input"]]
			gold_list = list(rows["output"])
		else:
			inputs_raw = [DOM_INSTRUCTION + row["input"] for row in rows]
			gold_list = [row["output"] for row in rows]

		enc = tok(inputs_raw, return_tensors="pt", truncation=True, padding=True)
		enc = {k: v.to(device) for k, v in enc.items()}

		with torch.inference_mode():
			outputs = model.generate(**enc, max_new_tokens=args.max_new, do_sample=False, num_beams=max(1, args.num_beams))
		pred_texts = [t.strip() for t in tok.batch_decode(outputs, skip_special_tokens=True)]

		for pred_text, gold in zip(pred_texts, gold_list):
			parts: List[str] = [t for t in pred_text.split() if t]
			p_url = parts[0] if len(parts) > 0 else ""
			p_sel = parts[1] if len(parts) > 1 else ""
			if not p_url or not p_sel:
				invalid_parse += 1
			try:
				g: Dict = json.loads(gold)
			except Exception:
				g = {}
			g_args = (g.get("args") or {})
			g_url = g_args.get("url") or ""
			# Multi-gold support: allow a list of selectors under 'selectors', else fallback to single 'selector'
			gold_selectors: List[str] = []
			if isinstance(g_args.get("selectors"), list):
				gold_selectors = [str(s) for s in g_args.get("selectors") if isinstance(s, str)]
			else:
				one = g_args.get("selector") or ""
				if isinstance(one, str) and one:
					gold_selectors = [one]
			if args.normalize:
				def norm_url(u: str) -> str:
					return u.strip().rstrip('/')
				def norm_sel(s: str) -> str:
					return re.sub(r"\s+", " ", s.strip())
				p_url_cmp, g_url_cmp = norm_url(p_url), norm_url(g_url)
				p_sel_cmp = norm_sel(p_sel)
				gold_sel_cmp = [norm_sel(s) for s in gold_selectors]
			else:
				p_url_cmp, g_url_cmp = p_url, g_url
				p_sel_cmp = p_sel
				gold_sel_cmp = gold_selectors

			ok_url = (p_url_cmp == g_url_cmp)
			ok_sel = (p_sel_cmp in gold_sel_cmp) if gold_sel_cmp else False

			if args.exec_validate and p_url and p_sel:
				try:
					# Guard exec validation with a per-example timeout to prevent hangs
					with ThreadPoolExecutor(max_workers=1) as ex:
						fut = ex.submit(fetch_text, p_url, p_sel)
						fetched = fut.result(timeout=max(1, args.exec_timeout))
					if isinstance(fetched, dict) and bool(fetched.get("text")):
						ok_sel = True
					else:
						exec_fail += 1
				except TimeoutError:
					exec_fail += 1
				except Exception:
					exec_fail += 1
			url_acc += 1 if ok_url else 0
			sel_acc += 1 if ok_sel else 0
			both_acc += 1 if (ok_url and ok_sel) else 0
			total += 1
			row = {
				"pred_url": p_url,
				"pred_selector": p_sel,
				"gold_url": g_url,
				"gold_selectors": gold_selectors,
				"ok_url": ok_url,
				"ok_selector": ok_sel,
			}
			report_rows.append(row)
			if args.progress_every > 0 and (total % args.progress_every == 0):
				print(f"progress: evaluated {total}/{n} examples...")

	def ratio(x: int) -> float:
		return (x / total) if total else 0.0

	print(f"dom_url_acc={ratio(url_acc):.3f} ({url_acc}/{total})")
	print(f"dom_selector_acc={ratio(sel_acc):.3f} ({sel_acc}/{total})")
	print(f"dom_both_acc={ratio(both_acc):.3f} ({both_acc}/{total})")
	print(f"dom_invalid_parse_rate={(invalid_parse/total) if total else 0.0:.3f} ({invalid_parse}/{total})")
	if args.exec_validate:
		print(f"dom_exec_fail_rate={(exec_fail/total) if total else 0.0:.3f} ({exec_fail}/{total})")
	if args.show_failures:
		for idx, r in enumerate(report_rows):
			if not (r.get("ok_url") and r.get("ok_selector")):
				print(f"FAIL[{idx}]: pred=({r['pred_url']} | {r['pred_selector']}) gold=({r['gold_url']} | {r['gold_selectors']}) ok_url={r['ok_url']} ok_sel={r['ok_selector']}")
	if args.report:
		try:
			outp = Path(args.report)
			outp.parent.mkdir(parents=True, exist_ok=True)
			with outp.open("w", encoding="utf-8") as fout:
				for r in report_rows:
					fout.write(json.dumps(r, ensure_ascii=False) + "\n")
			print(f"wrote report -> {outp}")
		except Exception:
			pass
	return 0

if __name__ == "__main__":
	raise SystemExit(main())


		if (cur / "agi_dw").is_dir():
			repo_root = cur
			break
		if cur.parent == cur:
			break
		cur = cur.parent
	if repo_root is None:
		repo_root = Path.cwd()
	pp = str(repo_root)
	if pp not in sys.path:
		sys.path.insert(0, pp)
	from agi_dw.bench.web_dom.runner import fetch_text  # type: ignore

DOM_INSTRUCTION = (
	'DOM task: Return ONLY two tokens: the URL and the CSS selector, separated by a single space. '
	'Example: https://example.com h1. No quotes, no explanations, no extra text. Input follows:\n'
)

def main() -> int:
	parser = argparse.ArgumentParser()
	root = Path(__file__).resolve().parents[1]
	parser.add_argument("--data", default=str(root / "data" / "skills" / "actuator_il_dom.jsonl"))
	parser.add_argument("--model", default=str(root / "models" / "actuator_dom_t5"))
	parser.add_argument("--max-new", type=int, default=32)
	parser.add_argument("--batch", type=int, default=8)
	parser.add_argument("--num-beams", type=int, default=1, help="Beam search width for generation")
	parser.add_argument("--normalize", action="store_true", help="Normalize URLs/selectors before comparing")
	parser.add_argument("--exec-validate", action="store_true", help="Treat selector as correct if predicted selector fetches non-empty text")
	parser.add_argument("--exec-timeout", type=int, default=6, help="Timeout (seconds) per execution validation fetch")
	parser.add_argument("--progress-every", type=int, default=25, help="Log progress every N examples")
	parser.add_argument("--report", default="", help="Optional path to write a JSONL report of per-example results")
	parser.add_argument("--show-failures", action="store_true", help="Print failing examples to stdout")
	args = parser.parse_args()

	ds = load_dataset("json", data_files={"test": args.data})
	tok = AutoTokenizer.from_pretrained(args.model)
	model = AutoModelForSeq2SeqLM.from_pretrained(args.model)
	device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
	if device.type == "cuda":
		model.half()
	model.to(device)
	model.eval()

	total = 0
	url_acc = 0
	sel_acc = 0
	both_acc = 0
	invalid_parse = 0
	exec_fail = 0
	report_rows: List[Dict[str, Any]] = []

	test_ds = ds["test"]
	n = len(test_ds)
	for i in range(0, n, args.batch):
		rows = test_ds[i : i + args.batch]
		if isinstance(rows, dict):
			inputs_raw = [DOM_INSTRUCTION + x for x in rows["input"]]
			gold_list = list(rows["output"])
		else:
			inputs_raw = [DOM_INSTRUCTION + row["input"] for row in rows]
			gold_list = [row["output"] for row in rows]

		enc = tok(inputs_raw, return_tensors="pt", truncation=True, padding=True)
		enc = {k: v.to(device) for k, v in enc.items()}

		with torch.inference_mode():
			outputs = model.generate(**enc, max_new_tokens=args.max_new, do_sample=False, num_beams=max(1, args.num_beams))
		pred_texts = [t.strip() for t in tok.batch_decode(outputs, skip_special_tokens=True)]

		for pred_text, gold in zip(pred_texts, gold_list):
			parts: List[str] = [t for t in pred_text.split() if t]
			p_url = parts[0] if len(parts) > 0 else ""
			p_sel = parts[1] if len(parts) > 1 else ""
			if not p_url or not p_sel:
				invalid_parse += 1
			try:
				g: Dict = json.loads(gold)
			except Exception:
				g = {}
			g_args = (g.get("args") or {})
			g_url = g_args.get("url") or ""
			# Multi-gold support: allow a list of selectors under 'selectors', else fallback to single 'selector'
			gold_selectors: List[str] = []
			if isinstance(g_args.get("selectors"), list):
				gold_selectors = [str(s) for s in g_args.get("selectors") if isinstance(s, str)]
			else:
				one = g_args.get("selector") or ""
				if isinstance(one, str) and one:
					gold_selectors = [one]
			if args.normalize:
				def norm_url(u: str) -> str:
					return u.strip().rstrip('/')
				def norm_sel(s: str) -> str:
					return re.sub(r"\s+", " ", s.strip())
				p_url_cmp, g_url_cmp = norm_url(p_url), norm_url(g_url)
				p_sel_cmp = norm_sel(p_sel)
				gold_sel_cmp = [norm_sel(s) for s in gold_selectors]
			else:
				p_url_cmp, g_url_cmp = p_url, g_url
				p_sel_cmp = p_sel
				gold_sel_cmp = gold_selectors

			ok_url = (p_url_cmp == g_url_cmp)
			ok_sel = (p_sel_cmp in gold_sel_cmp) if gold_sel_cmp else False

			if args.exec_validate and p_url and p_sel:
				try:
					# Guard exec validation with a per-example timeout to prevent hangs
					with ThreadPoolExecutor(max_workers=1) as ex:
						fut = ex.submit(fetch_text, p_url, p_sel)
						fetched = fut.result(timeout=max(1, args.exec_timeout))
					if isinstance(fetched, dict) and bool(fetched.get("text")):
						ok_sel = True
					else:
						exec_fail += 1
				except TimeoutError:
					exec_fail += 1
				except Exception:
					exec_fail += 1
			url_acc += 1 if ok_url else 0
			sel_acc += 1 if ok_sel else 0
			both_acc += 1 if (ok_url and ok_sel) else 0
			total += 1
			row = {
				"pred_url": p_url,
				"pred_selector": p_sel,
				"gold_url": g_url,
				"gold_selectors": gold_selectors,
				"ok_url": ok_url,
				"ok_selector": ok_sel,
			}
			report_rows.append(row)
			if args.progress_every > 0 and (total % args.progress_every == 0):
				print(f"progress: evaluated {total}/{n} examples...")

	def ratio(x: int) -> float:
		return (x / total) if total else 0.0

	print(f"dom_url_acc={ratio(url_acc):.3f} ({url_acc}/{total})")
	print(f"dom_selector_acc={ratio(sel_acc):.3f} ({sel_acc}/{total})")
	print(f"dom_both_acc={ratio(both_acc):.3f} ({both_acc}/{total})")
	print(f"dom_invalid_parse_rate={(invalid_parse/total) if total else 0.0:.3f} ({invalid_parse}/{total})")
	if args.exec_validate:
		print(f"dom_exec_fail_rate={(exec_fail/total) if total else 0.0:.3f} ({exec_fail}/{total})")
	if args.show_failures:
		for idx, r in enumerate(report_rows):
			if not (r.get("ok_url") and r.get("ok_selector")):
				print(f"FAIL[{idx}]: pred=({r['pred_url']} | {r['pred_selector']}) gold=({r['gold_url']} | {r['gold_selectors']}) ok_url={r['ok_url']} ok_sel={r['ok_selector']}")
	if args.report:
		try:
			outp = Path(args.report)
			outp.parent.mkdir(parents=True, exist_ok=True)
			with outp.open("w", encoding="utf-8") as fout:
				for r in report_rows:
					fout.write(json.dumps(r, ensure_ascii=False) + "\n")
			print(f"wrote report -> {outp}")
		except Exception:
			pass
	return 0

if __name__ == "__main__":
	raise SystemExit(main())


ratio

				except TimeoutError:
					exec_fail += 1
				except Exception:
					exec_fail += 1
			url_acc += 1 if ok_url else 0
			sel_acc += 1 if ok_sel else 0
			both_acc += 1 if (ok_url and ok_sel) else 0
			total += 1
			row = {
				"pred_url": p_url,
				"pred_selector": p_sel,
				"gold_url": g_url,
				"gold_selectors": gold_selectors,
				"ok_url": ok_url,
				"ok_selector": ok_sel,
			}
			report_rows.append(row)
			if args.progress_every > 0 and (total % args.progress_every == 0):
				print(f"progress: evaluated {total}/{n} examples...")

	def ratio(x: int) -> float:
		return (x / total) if total else 0.0

	print(f"dom_url_acc={ratio(url_acc):.3f} ({url_acc}/{total})")
	print(f"dom_selector_acc={ratio(sel_acc):.3f} ({sel_acc}/{total})")
	print(f"dom_both_acc={ratio(both_acc):.3f} ({both_acc}/{total})")
	print(f"dom_invalid_parse_rate={(invalid_parse/total) if total else 0.0:.3f} ({invalid_parse}/{total})")
	if args.exec_validate:
		print(f"dom_exec_fail_rate={(exec_fail/total) if total else 0.0:.3f} ({exec_fail}/{total})")
	if args.show_failures:
		for idx, r in enumerate(report_rows):
			if not (r.get("ok_url") and r.get("ok_selector")):
				print(f"FAIL[{idx}]: pred=({r['pred_url']} | {r['pred_selector']}) gold=({r['gold_url']} | {r['gold_selectors']}) ok_url={r['ok_url']} ok_sel={r['ok_selector']}")
	if args.report:
		try:
			outp = Path(args.report)
			outp.parent.mkdir(parents=True, exist_ok=True)
			with outp.open("w", encoding="utf-8") as fout:
				for r in report_rows:
					fout.write(json.dumps(r, ensure_ascii=False) + "\n")
			print(f"wrote report -> {outp}")
		except Exception:


			parts: List[str] = [t for t in pred_text.split() if t]
			p_url = parts[0] if len(parts) > 0 else ""
			p_sel = parts[1] if len(parts) > 1 else ""
			if not p_url or not p_sel:
				invalid_parse += 1
			try:
				g: Dict = json.loads(gold)
			except Exception:
				g = {}
			g_args = (g.get("args") or {})
			g_url = g_args.get("url") or ""
			# Multi-gold support: allow a list of selectors under 'selectors', else fallback to single 'selector'
			gold_selectors: List[str] = []
			if isinstance(g_args.get("selectors"), list):
				gold_selectors = [str(s) for s in g_args.get("selectors") if isinstance(s, str)]
			else:
				one = g_args.get("selector") or ""
				if isinstance(one, str) and one:
					gold_selectors = [one]
			if args.normalize:
				def norm_url(u: str) -> str:
					return u.strip().rstrip('/')
				def norm_sel(s: str) -> str:
					return re.sub(r"\s+", " ", s.strip())
				p_url_cmp, g_url_cmp = norm_url(p_url), norm_url(g_url)
				p_sel_cmp = norm_sel(p_sel)
				gold_sel_cmp = [norm_sel(s) for s in gold_selectors]
			else:
				p_url_cmp, g_url_cmp = p_url, g_url
				p_sel_cmp = p_sel
				gold_sel_cmp = gold_selectors

			ok_url = (p_url_cmp == g_url_cmp)
			ok_sel = (p_sel_cmp in gold_sel_cmp) if gold_sel_cmp else False

			if args.exec_validate and p_url and p_sel:
				try:
					# Guard exec validation with a per-example timeout to prevent hangs
					with ThreadPoolExecutor(max_workers=1) as ex:
						fut = ex.submit(fetch_text, p_url, p_sel)
						fetched = fut.result(timeout=max(1, args.exec_timeout))
					if isinstance(fetched, dict) and bool(fetched.get("text")):


			p_sel = parts[1] if len(parts) > 1 else ""
			if not p_url or not p_sel:
				invalid_parse += 1
			try:
				g: Dict = json.loads(gold)
			except Exception:
				g = {}
			g_args = (g.get("args") or {})
			g_url = g_args.get("url") or ""
			# Multi-gold support: allow a list of selectors under 'selectors', else fallback to single 'selector'
			gold_selectors: List[str] = []
			if isinstance(g_args.get("selectors"), list):
				gold_selectors = [str(s) for s in g_args.get("selectors") if isinstance(s, str)]
			else:
				one = g_args.get("selector") or ""
				if isinstance(one, str) and one:
					gold_selectors = [one]
			if args.normalize:
				def norm_url(u: str) -> str:
					return u.strip().rstrip('/')
				def norm_sel(s: str) -> str:
					return re.sub(r"\s+", " ", s.strip())
				p_url_cmp, g_url_cmp = norm_url(p_url), norm_url(g_url)
				p_sel_cmp = norm_sel(p_sel)
				gold_sel_cmp = [norm_sel(s) for s in gold_selectors]
			else:
				p_url_cmp, g_url_cmp = p_url, g_url
				p_sel_cmp = p_sel
				gold_sel_cmp = gold_selectors

			ok_url = (p_url_cmp == g_url_cmp)
			ok_sel = (p_sel_cmp in gold_sel_cmp) if gold_sel_cmp else False

			if args.exec_validate and p_url and p_sel:
				try:
					# Guard exec validation with a per-example timeout to prevent hangs
					with ThreadPoolExecutor(max_workers=1) as ex:
						fut = ex.submit(fetch_text, p_url, p_sel)
						fetched = fut.result(timeout=max(1, args.exec_timeout))
					if isinstance(fetched, dict) and bool(fetched.get("text")):
						ok_sel = True
					else:


import logging
import argparse
import json
import re
from pathlib import Path
from typing import Any, Dict, List

from datasets import load_dataset
from agi_dw.core.actuator.service import ActuatorConfig, RouterExtras, select_action

try:
	import yaml  # type: ignore
except Exception:
	yaml = None

INSTRUCTION = (
	'Actuator task: Return ONLY the CLI argv as a single space-separated string. '
	'Example: wc -l docs/a.txt. No quotes, no explanations, no extra text. Input follows:\n'
)

def robust_parse_yaml_first(text: str) -> Dict[str, Any]:
	if yaml is not None:
		try:
			y = yaml.safe_load(text)
			if isinstance(y, dict):
				return y
		except Exception:
			pass
	return robust_parse_json(text)

def robust_parse_json(text: str) -> Dict[str, Any]:
	text = text.strip()
	if text.startswith("{") and text.endswith("}"):
		try:
			return json.loads(text)
		except Exception:
			pass
	start = text.find("{")
	end = text.rfind("}")
	if start != -1 and end != -1 and end > start:
		try:
			return json.loads(text[start : end + 1])
		except Exception:
			return {}
	return {}

def coerce_flat_yaml(pred_text: str) -> Dict[str, Any]:
	# Normalize by inserting newlines before known keys
	s = pred_text
	for k in ["tool:", "args:", "argv:", "cwd:"]:
		s = s.replace(k, f"\n{k}")
	# Ensure list items use leading dashes on separate lines
	s = s.replace(" - ", "\n  - ")
	parsed = robust_parse_yaml_first(s)
	if not isinstance(parsed, dict):
		parsed = {}
	# If YAML parse produced a usable dict, normalize argv and return
	if "tool" in parsed and "args" in parsed and isinstance(parsed["args"], dict):
		argv = parsed["args"].get("argv")
		if isinstance(argv, list):
			parsed["args"]["argv"] = [str(x) for x in argv]
		return parsed
	# Regex fallback on the raw prediction
	tool_match = re.search(r"tool:\s*([^\s\n]+)", pred_text)
	tool = tool_match.group(1).strip() if tool_match else ""
	argv: List[str] = re.findall(r"-\s*([^\s\n]+)", pred_text)
	cwd_match = re.search(r"cwd:\s*([^\s\n]+)", pred_text)
	cwd = cwd_match.group(1).strip() if cwd_match else ""
	if tool and argv and cwd:
		return {"tool": tool, "args": {"argv": argv, "cwd": cwd}}
	return {}

def strict_pairs_equal(argv_pred: List[str], argv_gold: List[str]) -> bool:
	def to_pairs(xs: List[str]):
		out = []
		i = 0
		while i < len(xs):
			if i + 1 < len(xs) and xs[i].startswith('-') and not xs[i + 1].startswith('-'):
				out.append((xs[i], xs[i + 1]))
				i += 2
			else:
				out.append((xs[i], None))
				i += 1
		return set(out)
	return to_pairs(argv_pred) == to_pairs(argv_gold)

def main() -> int:
	parser = argparse.ArgumentParser()
	root = Path(__file__).resolve().parents[1]
	parser.add_argument("--data", default=str(root / "data" / "skills" / "actuator_il_combined.jsonl"))
	parser.add_argument("--model", default=str(root / "models" / "actuator_il_t5"))
	parser.add_argument("--max-new", type=int, default=64)
	parser.add_argument("--batch", type=int, default=8)
	parser.add_argument("--no-constraints", action="store_true")
	args = parser.parse_args()

	ds = load_dataset("json", data_files={"test": args.data})
	# Centralized actuator service config (CLI T5)
	cfg = ActuatorConfig(mode="t5", t5_model=str(args.model))
	extra = RouterExtras(domain="cli")

	total = 0
	exact = 0
	tool_match = 0
	argv_match = 0
	argv_strict_match = 0

	test_ds = ds["test"]
	n = len(test_ds)
	for i in range(0, n, args.batch):
		# Slice can return a dict-of-lists in datasets; normalize
		rows = test_ds[i : i + args.batch]
		if isinstance(rows, dict):
			inp_list = list(rows["input"])
			gold_list = list(rows["output"])
		else:
			inp_list = [row["input"] for row in rows]
			gold_list = [row["output"] for row in rows]

		for inp_text, gold in zip(inp_list, gold_list):
			# Parse obs/plan from dataset input JSON
			try:
				inp_obj = json.loads(inp_text)
			except Exception:
				inp_obj = {}
			obs = inp_obj.get("obs", {}) if isinstance(inp_obj, dict) else {}
			plan = inp_obj.get("plan", {}) if isinstance(inp_obj, dict) else {}
			# Predict action via centralized actuator service (T5)
			pred_action, _ = select_action(obs, plan, cfg, extra)
			args_d = pred_action.get("args") if isinstance(pred_action, dict) else {}
			p_argv = list(args_d.get("argv") or []) if isinstance(args_d, dict) else []
			p_argv = [str(x) for x in p_argv]
			# Gold argv from JSON
			g_argv: List[str] = []
			try:
				gold_json = json.loads(gold)
				g_argv = (gold_json.get("args") or {}).get("argv") or []
			except Exception:
				g_argv = []
			# Score argv
			if isinstance(p_argv, list) and isinstance(g_argv, list):
				if set(p_argv) == set(g_argv):
					argv_match += 1
				if strict_pairs_equal(p_argv, g_argv):
					argv_strict_match += 1
			total += 1
	exact_acc = exact / total if total else 0.0
	tool_acc = tool_match / total if total else 0.0
	argv_acc = argv_match / total if total else 0.0
	print(f"t5_argv_acc={argv_match/total if total else 0.0:.3f} ({argv_match}/{total})")
	strict_acc = (argv_strict_match / total) if total else 0.0
	print(f"t5_argv_strict_acc={strict_acc:.3f} ({argv_strict_match}/{total})")
	return 0

if __name__ == "__main__":
	raise SystemExit(main())


import argparse
import json
import re
from pathlib import Path
from typing import Any, Dict, List

from datasets import load_dataset
from agi_dw.core.actuator.service import ActuatorConfig, RouterExtras, select_action

try:
	import yaml  # type: ignore
except Exception:
	yaml = None

INSTRUCTION = (
	'Actuator task: Return ONLY the CLI argv as a single space-separated string. '
	'Example: wc -l docs/a.txt. No quotes, no explanations, no extra text. Input follows:\n'
)

def robust_parse_yaml_first(text: str) -> Dict[str, Any]:
	if yaml is not None:
		try:
			y = yaml.safe_load(text)
			if isinstance(y, dict):
				return y
		except Exception:
			pass
	return robust_parse_json(text)

def robust_parse_json(text: str) -> Dict[str, Any]:
	text = text.strip()
	if text.startswith("{") and text.endswith("}"):
		try:
			return json.loads(text)
		except Exception:
			pass
	start = text.find("{")
	end = text.rfind("}")
	if start != -1 and end != -1 and end > start:
		try:
			return json.loads(text[start : end + 1])
		except Exception:
			return {}
	return {}

def coerce_flat_yaml(pred_text: str) -> Dict[str, Any]:


except Exception:
	yaml = None

INSTRUCTION = (
	'Actuator task: Return ONLY the CLI argv as a single space-separated string. '
	'Example: wc -l docs/a.txt. No quotes, no explanations, no extra text. Input follows:\n'
)

def robust_parse_yaml_first(text: str) -> Dict[str, Any]:
	if yaml is not None:
		try:
			y = yaml.safe_load(text)
			if isinstance(y, dict):
				return y
		except Exception:
			pass
	return robust_parse_json(text)

def robust_parse_json(text: str) -> Dict[str, Any]:
	text = text.strip()
	if text.startswith("{") and text.endswith("}"):
		try:
			return json.loads(text)
		except Exception:
			pass
	start = text.find("{")
	end = text.rfind("}")
	if start != -1 and end != -1 and end > start:
		try:
			return json.loads(text[start : end + 1])
		except Exception:
			return {}
	return {}

def coerce_flat_yaml(pred_text: str) -> Dict[str, Any]:
	# Normalize by inserting newlines before known keys
	s = pred_text
	for k in ["tool:", "args:", "argv:", "cwd:"]:
		s = s.replace(k, f"\n{k}")
	# Ensure list items use leading dashes on separate lines
	s = s.replace(" - ", "\n  - ")
	parsed = robust_parse_yaml_first(s)
	if not isinstance(parsed, dict):
		parsed = {}
	# If YAML parse produced a usable dict, normalize argv and return
	if "tool" in parsed and "args" in parsed and isinstance(parsed["args"], dict):
		argv = parsed["args"].get("argv")
		if isinstance(argv, list):
			parsed["args"]["argv"] = [str(x) for x in argv]
		return parsed
	# Regex fallback on the raw prediction
	tool_match = re.search(r"tool:\s*([^\s\n]+)", pred_text)


	return robust_parse_json(text)

def robust_parse_json(text: str) -> Dict[str, Any]:
	text = text.strip()
	if text.startswith("{") and text.endswith("}"):
		try:
			return json.loads(text)
		except Exception:
			pass
	start = text.find("{")
	end = text.rfind("}")
	if start != -1 and end != -1 and end > start:
		try:
			return json.loads(text[start : end + 1])
		except Exception:
			return {}
	return {}

def coerce_flat_yaml(pred_text: str) -> Dict[str, Any]:
	# Normalize by inserting newlines before known keys
	s = pred_text
	for k in ["tool:", "args:", "argv:", "cwd:"]:
		s = s.replace(k, f"\n{k}")
	# Ensure list items use leading dashes on separate lines
	s = s.replace(" - ", "\n  - ")
	parsed = robust_parse_yaml_first(s)
	if not isinstance(parsed, dict):
		parsed = {}
	# If YAML parse produced a usable dict, normalize argv and return
	if "tool" in parsed and "args" in parsed and isinstance(parsed["args"], dict):
		argv = parsed["args"].get("argv")
		if isinstance(argv, list):
			parsed["args"]["argv"] = [str(x) for x in argv]
		return parsed
	# Regex fallback on the raw prediction
	tool_match = re.search(r"tool:\s*([^\s\n]+)", pred_text)
	tool = tool_match.group(1).strip() if tool_match else ""
	argv: List[str] = re.findall(r"-\s*([^\s\n]+)", pred_text)
	cwd_match = re.search(r"cwd:\s*([^\s\n]+)", pred_text)
	cwd = cwd_match.group(1).strip() if cwd_match else ""
	if tool and argv and cwd:
		return {"tool": tool, "args": {"argv": argv, "cwd": cwd}}
	return {}

def strict_pairs_equal(argv_pred: List[str], argv_gold: List[str]) -> bool:
	def to_pairs(xs: List[str]):
		out = []
		i = 0
		while i < len(xs):
			if i + 1 < len(xs) and xs[i].startswith('-') and not xs[i + 1].startswith('-'):
				out.append((xs[i], xs[i + 1]))
				i += 2
			else:
				out.append((xs[i], None))
				i += 1
		return set(out)
	return to_pairs(argv_pred) == to_pairs(argv_gold)

def main() -> int:
	parser = argparse.ArgumentParser()
	root = Path(__file__).resolve().parents[1]


	parsed = robust_parse_yaml_first(s)
	if not isinstance(parsed, dict):
		parsed = {}
	# If YAML parse produced a usable dict, normalize argv and return
	if "tool" in parsed and "args" in parsed and isinstance(parsed["args"], dict):
		argv = parsed["args"].get("argv")
		if isinstance(argv, list):
			parsed["args"]["argv"] = [str(x) for x in argv]
		return parsed
	# Regex fallback on the raw prediction
	tool_match = re.search(r"tool:\s*([^\s\n]+)", pred_text)
	tool = tool_match.group(1).strip() if tool_match else ""
	argv: List[str] = re.findall(r"-\s*([^\s\n]+)", pred_text)
	cwd_match = re.search(r"cwd:\s*([^\s\n]+)", pred_text)
	cwd = cwd_match.group(1).strip() if cwd_match else ""
	if tool and argv and cwd:
		return {"tool": tool, "args": {"argv": argv, "cwd": cwd}}
	return {}

def strict_pairs_equal(argv_pred: List[str], argv_gold: List[str]) -> bool:
	def to_pairs(xs: List[str]):
		out = []
		i = 0
		while i < len(xs):
			if i + 1 < len(xs) and xs[i].startswith('-') and not xs[i + 1].startswith('-'):
				out.append((xs[i], xs[i + 1]))
				i += 2
			else:
				out.append((xs[i], None))
				i += 1
		return set(out)
	return to_pairs(argv_pred) == to_pairs(argv_gold)

def main() -> int:
	parser = argparse.ArgumentParser()
	root = Path(__file__).resolve().parents[1]
	parser.add_argument("--data", default=str(root / "data" / "skills" / "actuator_il_combined.jsonl"))
	parser.add_argument("--model", default=str(root / "models" / "actuator_il_t5"))
	parser.add_argument("--max-new", type=int, default=64)
	parser.add_argument("--batch", type=int, default=8)
	parser.add_argument("--no-constraints", action="store_true")
	args = parser.parse_args()

	ds = load_dataset("json", data_files={"test": args.data})
	# Centralized actuator service config (CLI T5)
	cfg = ActuatorConfig(mode="t5", t5_model=str(args.model))
	extra = RouterExtras(domain="cli")

	total = 0
	exact = 0
	tool_match = 0


main

	if tool and argv and cwd:
		return {"tool": tool, "args": {"argv": argv, "cwd": cwd}}
	return {}

def strict_pairs_equal(argv_pred: List[str], argv_gold: List[str]) -> bool:
	def to_pairs(xs: List[str]):
		out = []
		i = 0
		while i < len(xs):
			if i + 1 < len(xs) and xs[i].startswith('-') and not xs[i + 1].startswith('-'):
				out.append((xs[i], xs[i + 1]))
				i += 2
			else:
				out.append((xs[i], None))
				i += 1
		return set(out)
	return to_pairs(argv_pred) == to_pairs(argv_gold)

def main() -> int:
	parser = argparse.ArgumentParser()
	root = Path(__file__).resolve().parents[1]
	parser.add_argument("--data", default=str(root / "data" / "skills" / "actuator_il_combined.jsonl"))
	parser.add_argument("--model", default=str(root / "models" / "actuator_il_t5"))
	parser.add_argument("--max-new", type=int, default=64)
	parser.add_argument("--batch", type=int, default=8)
	parser.add_argument("--no-constraints", action="store_true")
	args = parser.parse_args()

	ds = load_dataset("json", data_files={"test": args.data})
	# Centralized actuator service config (CLI T5)
	cfg = ActuatorConfig(mode="t5", t5_model=str(args.model))
	extra = RouterExtras(domain="cli")

	total = 0
	exact = 0
	tool_match = 0
	argv_match = 0
	argv_strict_match = 0

	test_ds = ds["test"]
	n = len(test_ds)
	for i in range(0, n, args.batch):
		# Slice can return a dict-of-lists in datasets; normalize
		rows = test_ds[i : i + args.batch]
		if isinstance(rows, dict):
			inp_list = list(rows["input"])
			gold_list = list(rows["output"])
		else:
			inp_list = [row["input"] for row in rows]
			gold_list = [row["output"] for row in rows]

		for inp_text, gold in zip(inp_list, gold_list):
			# Parse obs/plan from dataset input JSON
			try:
				inp_obj = json.loads(inp_text)
			except Exception:
				inp_obj = {}
			obs = inp_obj.get("obs", {}) if isinstance(inp_obj, dict) else {}
			plan = inp_obj.get("plan", {}) if isinstance(inp_obj, dict) else {}
			# Predict action via centralized actuator service (T5)
			pred_action, _ = select_action(obs, plan, cfg, extra)
			args_d = pred_action.get("args") if isinstance(pred_action, dict) else {}
			p_argv = list(args_d.get("argv") or []) if isinstance(args_d, dict) else []
			p_argv = [str(x) for x in p_argv]
			# Gold argv from JSON
			g_argv: List[str] = []
			try:
				gold_json = json.loads(gold)
				g_argv = (gold_json.get("args") or {}).get("argv") or []
			except Exception:
				g_argv = []
			# Score argv
			if isinstance(p_argv, list) and isinstance(g_argv, list):
				if set(p_argv) == set(g_argv):
					argv_match += 1
				if strict_pairs_equal(p_argv, g_argv):
					argv_strict_match += 1
			total += 1
	exact_acc = exact / total if total else 0.0
	tool_acc = tool_match / total if total else 0.0
	argv_acc = argv_match / total if total else 0.0
	print(f"t5_argv_acc={argv_match/total if total else 0.0:.3f} ({argv_match}/{total})")
	strict_acc = (argv_strict_match / total) if total else 0.0
	print(f"t5_argv_strict_acc={strict_acc:.3f} ({argv_strict_match}/{total})")
	return 0

if __name__ == "__main__":
	raise SystemExit(main())


	if not isinstance(parsed, dict):
		parsed = {}
	# If YAML parse produced a usable dict, normalize argv and return
	if "tool" in parsed and "args" in parsed and isinstance(parsed["args"], dict):
		argv = parsed["args"].get("argv")
		if isinstance(argv, list):
			parsed["args"]["argv"] = [str(x) for x in argv]
		return parsed
	# Regex fallback on the raw prediction
	tool_match = re.search(r"tool:\s*([^\s\n]+)", pred_text)
	tool = tool_match.group(1).strip() if tool_match else ""
	argv: List[str] = re.findall(r"-\s*([^\s\n]+)", pred_text)
	cwd_match = re.search(r"cwd:\s*([^\s\n]+)", pred_text)
	cwd = cwd_match.group(1).strip() if cwd_match else ""
	if tool and argv and cwd:
		return {"tool": tool, "args": {"argv": argv, "cwd": cwd}}
	return {}

def strict_pairs_equal(argv_pred: List[str], argv_gold: List[str]) -> bool:
	def to_pairs(xs: List[str]):
		out = []
		i = 0
		while i < len(xs):
			if i + 1 < len(xs) and xs[i].startswith('-') and not xs[i + 1].startswith('-'):
				out.append((xs[i], xs[i + 1]))
				i += 2
			else:
				out.append((xs[i], None))
				i += 1
		return set(out)
	return to_pairs(argv_pred) == to_pairs(argv_gold)

def main() -> int:
	parser = argparse.ArgumentParser()
	root = Path(__file__).resolve().parents[1]
	parser.add_argument("--data", default=str(root / "data" / "skills" / "actuator_il_combined.jsonl"))
	parser.add_argument("--model", default=str(root / "models" / "actuator_il_t5"))
	parser.add_argument("--max-new", type=int, default=64)
	parser.add_argument("--batch", type=int, default=8)
	parser.add_argument("--no-constraints", action="store_true")
	args = parser.parse_args()

	ds = load_dataset("json", data_files={"test": args.data})
	# Centralized actuator service config (CLI T5)
	cfg = ActuatorConfig(mode="t5", t5_model=str(args.model))
	extra = RouterExtras(domain="cli")

	total = 0
	exact = 0


import logging
import argparse
import json
from pathlib import Path

def main() -> int:
	ap = argparse.ArgumentParser()
	root = Path(__file__).resolve().parents[1]
	ap.add_argument("--summary", default=str(root / "data" / "dashboards" / "summary.json"))
	ap.add_argument("--min-cli-match", type=float, default=0.98)
	ap.add_argument("--min-dom-match", type=float, default=0.95)
	args = ap.parse_args()

	obj = json.loads(Path(args.summary).read_text(encoding="utf-8"))
	cli_sr = float(obj.get("bench", {}).get("cli_summary", {}).get("success_rate", 0.0))
	dom_sr = float(obj.get("bench", {}).get("dom_summary", {}).get("success_rate", 0.0))
	ok = bool(cli_sr >= float(args.min_cli_match) and dom_sr >= float(args.min_dom_match))
	print(json.dumps({"ok": ok, "cli": cli_sr, "dom": dom_sr, "min_cli": float(args.min_cli_match), "min_dom": float(args.min_dom_match)}))
	return 0 if ok else 1

if __name__ == "__main__":
	raise SystemExit(main())


import logging
import argparse
import json
from pathlib import Path

def main() -> int:
	ap = argparse.ArgumentParser()
	root = Path(__file__).resolve().parents[1]
	ap.add_argument("--results", default=str(root / "data" / "benchmarks" / "docs_results.jsonl"))
	args = ap.parse_args()

	p = Path(args.results)
	if not p.exists():
		print(json.dumps({"ok": False, "error": "docs_results_missing", "path": str(p)}))
		return 1
	total = 0
	ok = 0
	with p.open("r", encoding="utf-8") as f:
		for line in f:
			line = line.strip()
			if not line:
				continue
			try:
				rec = json.loads(line)
			except Exception:
				continue
			total += 1
			ok += 1 if bool(rec.get("ok", False)) else 0
	print(json.dumps({"ok": bool(ok == total and total > 0), "total": int(total), "ok_n": int(ok)}))
	return 0 if (ok == total and total > 0) else 1

if __name__ == "__main__":
	import sys
	sys.exit(main())


import logging
import argparse
import json
import time
from pathlib import Path
from subprocess import run, PIPE  # type: ignore
from typing import Dict, Any, List

def run_cmd(cmd: List[str]) -> Dict[str, Any]:
	start = time.time()
	p = run(cmd, stdout=PIPE, stderr=PIPE, encoding="utf-8")
	dur = time.time() - start
	ok = False
	try:
		last = p.stdout.strip().splitlines()[-1] if p.stdout else ""
		obj = json.loads(last) if last.startswith("{") else {}
		ok = str(obj.get("status", "")).lower() == "ok"
	except Exception:
		ok = False
	return {"ok": bool(ok), "dur": float(dur), "rc": int(p.returncode)}

def eval_one(root: Path, runs: int, with_wm: bool, planner_candidates: int, use_tot: bool, wm_horizon: int, pref_weights: str | None) -> Dict[str, Any]:
	tasks = ["count_lines", "grep_error"]
	res: Dict[str, Any] = {}
	for task in tasks:
		succ = 0
		dur_sum = 0.0
		for _ in range(max(1, runs)):
			cmd = [
				"python3", str(root / "scripts" / "run_loop_oscli.py"),
				"--planner-backend", "hf",
				"--verifier-backend", "hf",
				"--planner-model", "meta-llama/Llama-3.2-3B",
				"--verifier-model", "meta-llama/Llama-3.2-3B",
				"--timeout", "20",
				"--task", task,
			]
			# Always use the same number of candidates for fair comparison
			if int(planner_candidates) > 1:
				cmd.extend(["--planner-candidates", str(int(planner_candidates))])
				if bool(use_tot):
					cmd.append("--planner-tot")
				if pref_weights:
					cmd.extend(["--planner-pref-weights", str(pref_weights)])
			if with_wm:
				cmd.extend(["--wm-prior", "--wm-plan-rank", "--wm-horizon", str(int(max(1, wm_horizon)))])
			out = run_cmd(cmd)
			succ += 1 if out["ok"] else 0
			dur_sum += float(out["dur"])
		res[task] = {"success": int(succ), "runs": int(runs), "success_rate": (succ / max(1, runs)), "avg_latency_sec": (dur_sum / max(1, runs))}
	return res

def main() -> int:
	ap = argparse.ArgumentParser()
	root = Path(__file__).resolve().parents[1]
	ap.add_argument("--runs", type=int, default=5)
	ap.add_argument("--planner-candidates", type=int, default=3)
	ap.add_argument("--tot", action="store_true", help="Use ToT-style candidate generation in loops")
	# Compatibility flags (ignored by this script other than passing through intent)
	ap.add_argument("--planner-tot", action="store_true", help="Alias for --tot")
	ap.add_argument("--wm-prior", action="store_true", help="Enable WM prior in loops (passed through)")
	ap.add_argument("--wm-plan-rank", action="store_true", help="Enable WM plan ranking in loops (passed through)")
	ap.add_argument("--wm-horizon", type=int, default=1, help="WM rollout horizon to pass to loops")
	ap.add_argument("--out", default=str(root / "data" / "benchmarks" / "wm_planrank.json"))
	ap.add_argument("--planner-pref-weights", default=None, help="Path to JSON weights for reranking candidates")
	args = ap.parse_args()

	root.mkdir(parents=True, exist_ok=True)
	p_out = Path(args.out)
	p_out.parent.mkdir(parents=True, exist_ok=True)

	use_tot_flag = bool(args.tot or args.planner_tot)
	baseline = eval_one(root, max(1, int(args.runs)), with_wm=False, planner_candidates=int(args.planner_candidates), use_tot=use_tot_flag, wm_horizon=int(args.wm_horizon), pref_weights=args.planner_pref_weights)
	wmrank = eval_one(root, max(1, int(args.runs)), with_wm=True, planner_candidates=int(args.planner_candidates), use_tot=use_tot_flag, wm_horizon=int(args.wm_horizon), pref_weights=args.planner_pref_weights)

	# Compute average success rates for convenience
	def avg_sr(block: Dict[str, Any]) -> float:
		vals = []
		for k, v in block.items():
			try:
				vals.append(float(v.get("success_rate", 0.0)))
			except Exception:
				pass
		return (sum(vals) / float(len(vals))) if vals else 0.0

	out = {
		"baseline": baseline,
		"wm_planrank": wmrank,
		"meta": {
			"runs": int(args.runs),
			"planner_candidates": int(args.planner_candidates),
			"tot": bool(use_tot_flag),
			"wm_horizon": int(args.wm_horizon),
			"avg_success_rate_baseline": avg_sr(baseline),
			"avg_success_rate_wm": avg_sr(wmrank),
			"delta_success_rate": (avg_sr(wmrank) - avg_sr(baseline)),
		},
	}
	p_out.write_text(json.dumps(out, ensure_ascii=False, indent=2), encoding="utf-8")
	print(json.dumps({"ok": True, "out": str(p_out)}))
	return 0

if __name__ == "__main__":
	raise SystemExit(main())


import logging
import argparse
import json
import time
from pathlib import Path
from subprocess import run, PIPE  # type: ignore
from typing import Dict, Any, List

def run_cmd(cmd: List[str]) -> Dict[str, Any]:
	start = time.time()
	p = run(cmd, stdout=PIPE, stderr=PIPE, encoding="utf-8")
	dur = time.time() - start
	ok = False
	try:
		last = p.stdout.strip().splitlines()[-1] if p.stdout else ""
		obj = json.loads(last) if last.startswith("{") else {}
		ok = str(obj.get("status", "")).lower() == "ok"
	except Exception:
		ok = False
	return {"ok": bool(ok), "dur": float(dur), "rc": int(p.returncode)}

def eval_one(root: Path, runs: int, with_wm: bool, planner_candidates: int, use_tot: bool, wm_horizon: int, pref_weights: str | None) -> Dict[str, Any]:
	tasks = ["count_lines", "grep_error"]
	res: Dict[str, Any] = {}
	for task in tasks:
		succ = 0
		dur_sum = 0.0
		for _ in range(max(1, runs)):
			cmd = [
				"python3", str(root / "scripts" / "run_loop_oscli.py"),
				"--planner-backend", "hf",
				"--verifier-backend", "hf",
				"--planner-model", "meta-llama/Llama-3.2-3B",
				"--verifier-model", "meta-llama/Llama-3.2-3B",
				"--timeout", "20",
				"--task", task,
			]
			# Always use the same number of candidates for fair comparison
			if int(planner_candidates) > 1:


import time
from pathlib import Path
from subprocess import run, PIPE  # type: ignore
from typing import Dict, Any, List

def run_cmd(cmd: List[str]) -> Dict[str, Any]:
	start = time.time()
	p = run(cmd, stdout=PIPE, stderr=PIPE, encoding="utf-8")
	dur = time.time() - start
	ok = False
	try:
		last = p.stdout.strip().splitlines()[-1] if p.stdout else ""
		obj = json.loads(last) if last.startswith("{") else {}
		ok = str(obj.get("status", "")).lower() == "ok"
	except Exception:
		ok = False
	return {"ok": bool(ok), "dur": float(dur), "rc": int(p.returncode)}

def eval_one(root: Path, runs: int, with_wm: bool, planner_candidates: int, use_tot: bool, wm_horizon: int, pref_weights: str | None) -> Dict[str, Any]:
	tasks = ["count_lines", "grep_error"]
	res: Dict[str, Any] = {}
	for task in tasks:
		succ = 0
		dur_sum = 0.0
		for _ in range(max(1, runs)):
			cmd = [
				"python3", str(root / "scripts" / "run_loop_oscli.py"),
				"--planner-backend", "hf",
				"--verifier-backend", "hf",
				"--planner-model", "meta-llama/Llama-3.2-3B",
				"--verifier-model", "meta-llama/Llama-3.2-3B",
				"--timeout", "20",
				"--task", task,
			]
			# Always use the same number of candidates for fair comparison
			if int(planner_candidates) > 1:
				cmd.extend(["--planner-candidates", str(int(planner_candidates))])
				if bool(use_tot):
					cmd.append("--planner-tot")
				if pref_weights:
					cmd.extend(["--planner-pref-weights", str(pref_weights)])
			if with_wm:
				cmd.extend(["--wm-prior", "--wm-plan-rank", "--wm-horizon", str(int(max(1, wm_horizon)))])
			out = run_cmd(cmd)
			succ += 1 if out["ok"] else 0
			dur_sum += float(out["dur"])
		res[task] = {"success": int(succ), "runs": int(runs), "success_rate": (succ / max(1, runs)), "avg_latency_sec": (dur_sum / max(1, runs))}
	return res

def main() -> int:
	ap = argparse.ArgumentParser()
	root = Path(__file__).resolve().parents[1]
	ap.add_argument("--runs", type=int, default=5)
	ap.add_argument("--planner-candidates", type=int, default=3)
	ap.add_argument("--tot", action="store_true", help="Use ToT-style candidate generation in loops")
	# Compatibility flags (ignored by this script other than passing through intent)
	ap.add_argument("--planner-tot", action="store_true", help="Alias for --tot")
	ap.add_argument("--wm-prior", action="store_true", help="Enable WM prior in loops (passed through)")
	ap.add_argument("--wm-plan-rank", action="store_true", help="Enable WM plan ranking in loops (passed through)")
	ap.add_argument("--wm-horizon", type=int, default=1, help="WM rollout horizon to pass to loops")
	ap.add_argument("--out", default=str(root / "data" / "benchmarks" / "wm_planrank.json"))
	ap.add_argument("--planner-pref-weights", default=None, help="Path to JSON weights for reranking candidates")
	args = ap.parse_args()

	root.mkdir(parents=True, exist_ok=True)
	p_out = Path(args.out)
	p_out.parent.mkdir(parents=True, exist_ok=True)


				"--verifier-model", "meta-llama/Llama-3.2-3B",
				"--timeout", "20",
				"--task", task,
			]
			# Always use the same number of candidates for fair comparison
			if int(planner_candidates) > 1:
				cmd.extend(["--planner-candidates", str(int(planner_candidates))])
				if bool(use_tot):
					cmd.append("--planner-tot")
				if pref_weights:
					cmd.extend(["--planner-pref-weights", str(pref_weights)])
			if with_wm:
				cmd.extend(["--wm-prior", "--wm-plan-rank", "--wm-horizon", str(int(max(1, wm_horizon)))])
			out = run_cmd(cmd)
			succ += 1 if out["ok"] else 0
			dur_sum += float(out["dur"])
		res[task] = {"success": int(succ), "runs": int(runs), "success_rate": (succ / max(1, runs)), "avg_latency_sec": (dur_sum / max(1, runs))}
	return res

def main() -> int:
	ap = argparse.ArgumentParser()
	root = Path(__file__).resolve().parents[1]
	ap.add_argument("--runs", type=int, default=5)
	ap.add_argument("--planner-candidates", type=int, default=3)
	ap.add_argument("--tot", action="store_true", help="Use ToT-style candidate generation in loops")
	# Compatibility flags (ignored by this script other than passing through intent)
	ap.add_argument("--planner-tot", action="store_true", help="Alias for --tot")
	ap.add_argument("--wm-prior", action="store_true", help="Enable WM prior in loops (passed through)")
	ap.add_argument("--wm-plan-rank", action="store_true", help="Enable WM plan ranking in loops (passed through)")
	ap.add_argument("--wm-horizon", type=int, default=1, help="WM rollout horizon to pass to loops")
	ap.add_argument("--out", default=str(root / "data" / "benchmarks" / "wm_planrank.json"))
	ap.add_argument("--planner-pref-weights", default=None, help="Path to JSON weights for reranking candidates")
	args = ap.parse_args()

	root.mkdir(parents=True, exist_ok=True)
	p_out = Path(args.out)
	p_out.parent.mkdir(parents=True, exist_ok=True)

	use_tot_flag = bool(args.tot or args.planner_tot)
	baseline = eval_one(root, max(1, int(args.runs)), with_wm=False, planner_candidates=int(args.planner_candidates), use_tot=use_tot_flag, wm_horizon=int(args.wm_horizon), pref_weights=args.planner_pref_weights)
	wmrank = eval_one(root, max(1, int(args.runs)), with_wm=True, planner_candidates=int(args.planner_candidates), use_tot=use_tot_flag, wm_horizon=int(args.wm_horizon), pref_weights=args.planner_pref_weights)

	# Compute average success rates for convenience
	def avg_sr(block: Dict[str, Any]) -> float:
		vals = []
		for k, v in block.items():
			try:
				vals.append(float(v.get("success_rate", 0.0)))
			except Exception:
				pass
		return (sum(vals) / float(len(vals))) if vals else 0.0

	out = {
		"baseline": baseline,
		"wm_planrank": wmrank,
		"meta": {
			"runs": int(args.runs),
			"planner_candidates": int(args.planner_candidates),
			"tot": bool(use_tot_flag),
			"wm_horizon": int(args.wm_horizon),
			"avg_success_rate_baseline": avg_sr(baseline),
			"avg_success_rate_wm": avg_sr(wmrank),
			"delta_success_rate": (avg_sr(wmrank) - avg_sr(baseline)),
		},
	}
	p_out.write_text(json.dumps(out, ensure_ascii=False, indent=2), encoding="utf-8")
	print(json.dumps({"ok": True, "out": str(p_out)}))
	return 0

if __name__ == "__main__":
	raise SystemExit(main())


	ap.add_argument("--planner-candidates", type=int, default=3)
	ap.add_argument("--tot", action="store_true", help="Use ToT-style candidate generation in loops")
	# Compatibility flags (ignored by this script other than passing through intent)
	ap.add_argument("--planner-tot", action="store_true", help="Alias for --tot")
	ap.add_argument("--wm-prior", action="store_true", help="Enable WM prior in loops (passed through)")
	ap.add_argument("--wm-plan-rank", action="store_true", help="Enable WM plan ranking in loops (passed through)")
	ap.add_argument("--wm-horizon", type=int, default=1, help="WM rollout horizon to pass to loops")
	ap.add_argument("--out", default=str(root / "data" / "benchmarks" / "wm_planrank.json"))
	ap.add_argument("--planner-pref-weights", default=None, help="Path to JSON weights for reranking candidates")
	args = ap.parse_args()

	root.mkdir(parents=True, exist_ok=True)
	p_out = Path(args.out)
	p_out.parent.mkdir(parents=True, exist_ok=True)

	use_tot_flag = bool(args.tot or args.planner_tot)
	baseline = eval_one(root, max(1, int(args.runs)), with_wm=False, planner_candidates=int(args.planner_candidates), use_tot=use_tot_flag, wm_horizon=int(args.wm_horizon), pref_weights=args.planner_pref_weights)
	wmrank = eval_one(root, max(1, int(args.runs)), with_wm=True, planner_candidates=int(args.planner_candidates), use_tot=use_tot_flag, wm_horizon=int(args.wm_horizon), pref_weights=args.planner_pref_weights)

	# Compute average success rates for convenience
	def avg_sr(block: Dict[str, Any]) -> float:
		vals = []
		for k, v in block.items():
			try:
				vals.append(float(v.get("success_rate", 0.0)))
			except Exception:
				pass
		return (sum(vals) / float(len(vals))) if vals else 0.0

	out = {
		"baseline": baseline,
		"wm_planrank": wmrank,
		"meta": {
			"runs": int(args.runs),
			"planner_candidates": int(args.planner_candidates),
			"tot": bool(use_tot_flag),
			"wm_horizon": int(args.wm_horizon),
			"avg_success_rate_baseline": avg_sr(baseline),
			"avg_success_rate_wm": avg_sr(wmrank),
			"delta_success_rate": (avg_sr(wmrank) - avg_sr(baseline)),
		},
	}
	p_out.write_text(json.dumps(out, ensure_ascii=False, indent=2), encoding="utf-8")
	print(json.dumps({"ok": True, "out": str(p_out)}))
	return 0

if __name__ == "__main__":


import logging
import argparse
import json
from pathlib import Path

def _get_rates(summary: dict, use_budgeted: bool) -> tuple[float, float]:
	bench = summary.get("bench", {}) if isinstance(summary, dict) else {}
	cli = bench.get("cli_summary", {}) if isinstance(bench, dict) else {}
	dom = bench.get("dom_summary", {}) if isinstance(bench, dict) else {}
	if use_budgeted:
		cli_rate = float(cli.get("success_rate_effective", cli.get("success_rate_budgeted", cli.get("success_rate", 0.0))))
		dom_rate = float(dom.get("success_rate_effective", dom.get("success_rate_budgeted", dom.get("success_rate", 0.0))))
	else:
		cli_rate = float(cli.get("success_rate", 0.0))
		dom_rate = float(dom.get("success_rate", 0.0))
	return cli_rate, dom_rate

def _get_costs(summary: dict) -> tuple[float | None, float | None]:
	bench = summary.get("bench", {}) if isinstance(summary, dict) else {}
	cli = bench.get("cli_summary", {}) if isinstance(bench, dict) else {}
	dom = bench.get("dom_summary", {}) if isinstance(bench, dict) else {}
	cli_cost = cli.get("cost_per_success")
	dom_cost = dom.get("cost_per_success")
	try:
		cli_cost = float(cli_cost) if cli_cost is not None else None
	except Exception:
		cli_cost = None
	try:
		dom_cost = float(dom_cost) if dom_cost is not None else None
	except Exception:
		dom_cost = None
	return cli_cost, dom_cost

def main() -> int:
	ap = argparse.ArgumentParser()
	root = Path(__file__).resolve().parents[1]
	ap.add_argument("--current", default=str(root / "data" / "dashboards" / "summary.json"))
	ap.add_argument("--baseline", required=True)
	ap.add_argument("--max-drop", type=float, default=0.0, help="Allow up to this drop in success rate (absolute)")
	ap.add_argument("--use-budgeted", action="store_true")
	ap.add_argument("--max-cost-inc-cli", type=float, default=None, help="Max allowed absolute increase in CLI cost_per_success")
	ap.add_argument("--max-cost-inc-dom", type=float, default=None, help="Max allowed absolute increase in DOM cost_per_success")
	args = ap.parse_args()

	curp = Path(args.current)
	basep = Path(args.baseline)
	if not curp.exists() or not basep.exists():
		print(json.dumps({"ok": False, "error": "missing_files", "current": str(curp), "baseline": str(basep)}))
		return 1
	cur = json.loads(curp.read_text(encoding="utf-8"))
	base = json.loads(basep.read_text(encoding="utf-8"))

	cur_cli, cur_dom = _get_rates(cur, bool(args.use_budgeted))
	base_cli, base_dom = _get_rates(base, bool(args.use_budgeted))
	cur_cli_cost, cur_dom_cost = _get_costs(cur)
	base_cli_cost, base_dom_cost = _get_costs(base)

	drop_cli = base_cli - cur_cli
	drop_dom = base_dom - cur_dom
	ok = (drop_cli <= float(args.max_drop)) and (drop_dom <= float(args.max_drop))
	# Optional cost-per-success regression gates
	if ok and args.max_cost_inc_cli is not None and (base_cli_cost is not None and cur_cli_cost is not None):
		inc_cli_cost = float(cur_cli_cost) - float(base_cli_cost)
		ok = ok and (inc_cli_cost <= float(args.max_cost_inc_cli))
	else:
		inc_cli_cost = None
	if ok and args.max_cost_inc_dom is not None and (base_dom_cost is not None and cur_dom_cost is not None):
		inc_dom_cost = float(cur_dom_cost) - float(base_dom_cost)
		ok = ok and (inc_dom_cost <= float(args.max_cost_inc_dom))
	else:
		inc_dom_cost = None
	print(json.dumps({
		"ok": bool(ok),
		"current": {"cli": cur_cli, "dom": cur_dom},
		"baseline": {"cli": base_cli, "dom": base_dom},
		"drops": {"cli": drop_cli, "dom": drop_dom},
		"max_drop": float(args.max_drop),
		"budgeted": bool(args.use_budgeted),
		"costs": {
			"current": {"cli": cur_cli_cost, "dom": cur_dom_cost},
			"baseline": {"cli": base_cli_cost, "dom": base_dom_cost},
			"inc": {"cli": inc_cli_cost, "dom": inc_dom_cost},
			"max_inc": {"cli": args.max_cost_inc_cli, "dom": args.max_cost_inc_dom},
		},
	}))
	return 0 if ok else 1

if __name__ == "__main__":
	raise SystemExit(main())


import logging
import argparse
import json
from pathlib import Path

def _get_rates(summary: dict, use_budgeted: bool) -> tuple[float, float]:
	bench = summary.get("bench", {}) if isinstance(summary, dict) else {}
	cli = bench.get("cli_summary", {}) if isinstance(bench, dict) else {}
	dom = bench.get("dom_summary", {}) if isinstance(bench, dict) else {}
	if use_budgeted:
		cli_rate = float(cli.get("success_rate_effective", cli.get("success_rate_budgeted", cli.get("success_rate", 0.0))))
		dom_rate = float(dom.get("success_rate_effective", dom.get("success_rate_budgeted", dom.get("success_rate", 0.0))))
	else:
		cli_rate = float(cli.get("success_rate", 0.0))
		dom_rate = float(dom.get("success_rate", 0.0))
	return cli_rate, dom_rate

def _get_costs(summary: dict) -> tuple[float | None, float | None]:
	bench = summary.get("bench", {}) if isinstance(summary, dict) else {}
	cli = bench.get("cli_summary", {}) if isinstance(bench, dict) else {}
	dom = bench.get("dom_summary", {}) if isinstance(bench, dict) else {}
	cli_cost = cli.get("cost_per_success")
	dom_cost = dom.get("cost_per_success")
	try:
		cli_cost = float(cli_cost) if cli_cost is not None else None
	except Exception:
		cli_cost = None
	try:
		dom_cost = float(dom_cost) if dom_cost is not None else None
	except Exception:
		dom_cost = None
	return cli_cost, dom_cost

def main() -> int:


import logging
import argparse
import json
from pathlib import Path

def _get_rates(summary: dict, use_budgeted: bool) -> tuple[float, float]:
	bench = summary.get("bench", {}) if isinstance(summary, dict) else {}
	cli = bench.get("cli_summary", {}) if isinstance(bench, dict) else {}
	dom = bench.get("dom_summary", {}) if isinstance(bench, dict) else {}
	if use_budgeted:
		cli_rate = float(cli.get("success_rate_effective", cli.get("success_rate_budgeted", cli.get("success_rate", 0.0))))
		dom_rate = float(dom.get("success_rate_effective", dom.get("success_rate_budgeted", dom.get("success_rate", 0.0))))
	else:
		cli_rate = float(cli.get("success_rate", 0.0))
		dom_rate = float(dom.get("success_rate", 0.0))
	return cli_rate, dom_rate

def _get_costs(summary: dict) -> tuple[float | None, float | None]:
	bench = summary.get("bench", {}) if isinstance(summary, dict) else {}
	cli = bench.get("cli_summary", {}) if isinstance(bench, dict) else {}
	dom = bench.get("dom_summary", {}) if isinstance(bench, dict) else {}
	cli_cost = cli.get("cost_per_success")
	dom_cost = dom.get("cost_per_success")
	try:
		cli_cost = float(cli_cost) if cli_cost is not None else None
	except Exception:
		cli_cost = None
	try:
		dom_cost = float(dom_cost) if dom_cost is not None else None
	except Exception:
		dom_cost = None
	return cli_cost, dom_cost

def main() -> int:
	ap = argparse.ArgumentParser()
	root = Path(__file__).resolve().parents[1]
	ap.add_argument("--current", default=str(root / "data" / "dashboards" / "summary.json"))
	ap.add_argument("--baseline", required=True)
	ap.add_argument("--max-drop", type=float, default=0.0, help="Allow up to this drop in success rate (absolute)")
	ap.add_argument("--use-budgeted", action="store_true")
	ap.add_argument("--max-cost-inc-cli", type=float, default=None, help="Max allowed absolute increase in CLI cost_per_success")
	ap.add_argument("--max-cost-inc-dom", type=float, default=None, help="Max allowed absolute increase in DOM cost_per_success")
	args = ap.parse_args()

	curp = Path(args.current)
	basep = Path(args.baseline)
	if not curp.exists() or not basep.exists():
		print(json.dumps({"ok": False, "error": "missing_files", "current": str(curp), "baseline": str(basep)}))
		return 1
	cur = json.loads(curp.read_text(encoding="utf-8"))
	base = json.loads(basep.read_text(encoding="utf-8"))


	return cli_rate, dom_rate

def _get_costs(summary: dict) -> tuple[float | None, float | None]:
	bench = summary.get("bench", {}) if isinstance(summary, dict) else {}
	cli = bench.get("cli_summary", {}) if isinstance(bench, dict) else {}
	dom = bench.get("dom_summary", {}) if isinstance(bench, dict) else {}
	cli_cost = cli.get("cost_per_success")
	dom_cost = dom.get("cost_per_success")
	try:
		cli_cost = float(cli_cost) if cli_cost is not None else None
	except Exception:
		cli_cost = None
	try:
		dom_cost = float(dom_cost) if dom_cost is not None else None
	except Exception:
		dom_cost = None
	return cli_cost, dom_cost

def main() -> int:
	ap = argparse.ArgumentParser()
	root = Path(__file__).resolve().parents[1]
	ap.add_argument("--current", default=str(root / "data" / "dashboards" / "summary.json"))
	ap.add_argument("--baseline", required=True)
	ap.add_argument("--max-drop", type=float, default=0.0, help="Allow up to this drop in success rate (absolute)")
	ap.add_argument("--use-budgeted", action="store_true")
	ap.add_argument("--max-cost-inc-cli", type=float, default=None, help="Max allowed absolute increase in CLI cost_per_success")
	ap.add_argument("--max-cost-inc-dom", type=float, default=None, help="Max allowed absolute increase in DOM cost_per_success")
	args = ap.parse_args()

	curp = Path(args.current)
	basep = Path(args.baseline)
	if not curp.exists() or not basep.exists():
		print(json.dumps({"ok": False, "error": "missing_files", "current": str(curp), "baseline": str(basep)}))
		return 1
	cur = json.loads(curp.read_text(encoding="utf-8"))
	base = json.loads(basep.read_text(encoding="utf-8"))

	cur_cli, cur_dom = _get_rates(cur, bool(args.use_budgeted))
	base_cli, base_dom = _get_rates(base, bool(args.use_budgeted))
	cur_cli_cost, cur_dom_cost = _get_costs(cur)
	base_cli_cost, base_dom_cost = _get_costs(base)

	drop_cli = base_cli - cur_cli
	drop_dom = base_dom - cur_dom
	ok = (drop_cli <= float(args.max_drop)) and (drop_dom <= float(args.max_drop))
	# Optional cost-per-success regression gates
	if ok and args.max_cost_inc_cli is not None and (base_cli_cost is not None and cur_cli_cost is not None):
		inc_cli_cost = float(cur_cli_cost) - float(base_cli_cost)
		ok = ok and (inc_cli_cost <= float(args.max_cost_inc_cli))
	else:
		inc_cli_cost = None
	if ok and args.max_cost_inc_dom is not None and (base_dom_cost is not None and cur_dom_cost is not None):
		inc_dom_cost = float(cur_dom_cost) - float(base_dom_cost)
		ok = ok and (inc_dom_cost <= float(args.max_cost_inc_dom))
	else:
		inc_dom_cost = None
	print(json.dumps({
		"ok": bool(ok),
		"current": {"cli": cur_cli, "dom": cur_dom},
		"baseline": {"cli": base_cli, "dom": base_dom},
		"drops": {"cli": drop_cli, "dom": drop_dom},
		"max_drop": float(args.max_drop),
		"budgeted": bool(args.use_budgeted),
		"costs": {
			"current": {"cli": cur_cli_cost, "dom": cur_dom_cost},
			"baseline": {"cli": base_cli_cost, "dom": base_dom_cost},
			"inc": {"cli": inc_cli_cost, "dom": inc_dom_cost},
			"max_inc": {"cli": args.max_cost_inc_cli, "dom": args.max_cost_inc_dom},
		},
	}))
	return 0 if ok else 1

if __name__ == "__main__":
	raise SystemExit(main())


import logging
import argparse
import json
import os
import sys

def main() -> int:
	ap = argparse.ArgumentParser()
	ap.add_argument("--min", type=float, default=float(os.environ.get("MIN_ROUTER_SR", 0.5)))
	ap.add_argument(
		"--per-task",
		action="append",
		default=[],
		help="Optional per-task gates formatted as 'actuator:task:threshold' (e.g., router:count_lines:0.5)",
	)
	args = ap.parse_args()

	data = sys.stdin.read().strip()
	if not data:
		print("No summary on stdin", file=sys.stderr)
		return 2
	try:
		summary = json.loads(data.splitlines()[-1])
	except Exception as e:
		print(f"Failed to parse summary JSON: {e}", file=sys.stderr)
		return 2

	router = summary.get("router", {}) if isinstance(summary, dict) else {}
	sr = float(router.get("success_rate", -1.0)) if isinstance(router, dict) else -1.0
	if sr < 0:
		print("Router success_rate not found in summary", file=sys.stderr)
		return 2
	print(json.dumps({"router_success_rate": sr, "threshold": args.__dict__["min"]}))
	if sr + 1e-9 < args.__dict__["min"]:
		print("Router success rate below threshold", file=sys.stderr)
		return 1

	# Optional per-task gating
	per_task = summary.get("per_task", {}) if isinstance(summary, dict) else {}
	if args.per_task:
		for spec in args.per_task:
			try:
				act, task, thr_s = spec.split(":", 2)
				thr = float(thr_s)
			except Exception:
				print(f"Invalid --per-task spec: {spec}", file=sys.stderr)
				return 2
			key = f"{act}:{task}"
			stats = per_task.get(key, {})
			n = float(stats.get("n", 0))
			s = float(stats.get("success", 0))
			rate = (s / n) if n else 0.0
			print(json.dumps({"per_task": key, "rate": rate, "n": n, "threshold": thr}))
			if rate + 1e-9 < thr:
				print(f"Per-task gate failed for {key}: {rate} < {thr}", file=sys.stderr)
				return 1
	return 0

if __name__ == "__main__":
	raise SystemExit(main())


#!/usr/bin/env python3
import logging
import argparse
import json
from pathlib import Path

def parse_args() -> argparse.Namespace:
	root = Path(__file__).resolve().parents[1]
	ap = argparse.ArgumentParser(description="CI assertion for LLM benchmark results")
	ap.add_argument("--results", default=str(root / "data" / "llm_bench" / "results.json"))
	ap.add_argument("--thresholds", default=str(root / "data" / "llm_bench" / "thresholds.json"))
	ap.add_argument("--allow-missing", action="store_true")
	ap.add_argument("--max-elapsed-sec", type=float, default=None, help="Optional max total elapsed seconds budget")
	return ap.parse_args()

def main() -> int:
	args = parse_args()
	results_path = Path(args.results)
	thr_path = Path(args.thresholds)

	if not results_path.exists():
		print(json.dumps({"ok": False, "error": "results_missing", "path": str(results_path)}))
		return 2
	try:
		res = json.loads(results_path.read_text(encoding="utf-8"))
	except Exception as e:
		print(json.dumps({"ok": False, "error": "results_parse_error", "detail": str(e)}))
		return 2

	thresholds = {}
	if thr_path.exists():
		try:
			thresholds = json.loads(thr_path.read_text(encoding="utf-8"))
		except Exception as e:
			print(json.dumps({"ok": False, "error": "thresholds_parse_error", "detail": str(e)}))
			return 2

	failures = []
	for name, data in res.get("benchmarks", {}).items():
		target = thresholds.get(name)
		score = data.get("score")
		status = data.get("status")
		if status == "not_implemented":
			if args.allow_missing:
				continue
			failures.append({"name": name, "reason": "not_implemented"})
			continue
		if target is None:
			# No threshold defined; skip
			continue
		if score is None or score < target:
			failures.append({"name": name, "score": score, "min": target})

	# Optional budget check
	if args.max_elapsed_sec is not None:
		elapsed = None
		try:
			elapsed = float(res.get("meta", {}).get("elapsed_sec"))
		except Exception:
			elapsed = None
		if elapsed is None or elapsed > args.max_elapsed_sec:
			failures.append({"name": "budget", "elapsed": elapsed, "max": args.max_elapsed_sec})

	ok = len(failures) == 0
	print(json.dumps({"ok": ok, "failures": failures}))
	return 0 if ok else 1

if __name__ == "__main__":
	raise SystemExit(main())


#!/usr/bin/env python3
import logging
import argparse
import json
from pathlib import Path

def parse_args() -> argparse.Namespace:
	root = Path(__file__).resolve().parents[1]
	ap = argparse.ArgumentParser(description="CI assertion for LLM benchmark results")
	ap.add_argument("--results", default=str(root / "data" / "llm_bench" / "results.json"))
	ap.add_argument("--thresholds", default=str(root / "data" / "llm_bench" / "thresholds.json"))
	ap.add_argument("--allow-missing", action="store_true")
	ap.add_argument("--max-elapsed-sec", type=float, default=None, help="Optional max total elapsed seconds budget")
	return ap.parse_args()

def main() -> int:
	args = parse_args()
	results_path = Path(args.results)
	thr_path = Path(args.thresholds)

	if not results_path.exists():
		print(json.dumps({"ok": False, "error": "results_missing", "path": str(results_path)}))
		return 2
	try:
		res = json.loads(results_path.read_text(encoding="utf-8"))
	except Exception as e:
		print(json.dumps({"ok": False, "error": "results_parse_error", "detail": str(e)}))
		return 2

	thresholds = {}
	if thr_path.exists():
		try:
			thresholds = json.loads(thr_path.read_text(encoding="utf-8"))


import logging
import argparse
import json
import random
from pathlib import Path

from agi_dw.bench.web_dom.runner import fetch_text

def main() -> int:
	ap = argparse.ArgumentParser()
	root = Path(__file__).resolve().parents[1]
	ap.add_argument("--out", default=str(root / "data" / "traces" / "web_dom.adversarial.jsonl"))
	ap.add_argument("--num", type=int, default=20)
	args = ap.parse_args()

	# Generate trivial adversarial selectors by perturbing valid ones
	base = [
		("https://example.com", "h1"),
		("https://en.wikipedia.org/wiki/Alan_Turing", "#firstHeading"),
		("https://docs.python.org/3/", "h1"),
	]
	mods = [" ", ":not(*)", ":nth-child(1)", ", h9", "#does-not-exist"]
	out = Path(args.out)
	out.parent.mkdir(parents=True, exist_ok=True)

	def _write(obj: dict) -> None:
		with out.open("a", encoding="utf-8") as f:
			f.write(json.dumps(obj, ensure_ascii=False) + "\n")

	count = 0
	while count < int(args.num):
		url, sel = random.choice(base)
		m = random.choice(mods)
		bad_sel = f"{sel}{m}"
		r = fetch_text(url, bad_sel)
		obj = {"url": url, "selector": bad_sel, "text": r.get("text", "")}
		obj["success"] = bool(obj["text"]) and bad_sel != "#does-not-exist"
		_write(obj)
		count += 1
	print(str(out))
	return 0

if __name__ == "__main__":
	import sys
	sys.exit(main())


from agi_dw.bench.web_dom.runner import fetch_text

def main() -> int:
	ap = argparse.ArgumentParser()
	root = Path(__file__).resolve().parents[1]
	ap.add_argument("--out", default=str(root / "data" / "traces" / "web_dom.adversarial.jsonl"))
	ap.add_argument("--num", type=int, default=20)
	args = ap.parse_args()

	# Generate trivial adversarial selectors by perturbing valid ones
	base = [
		("https://example.com", "h1"),
		("https://en.wikipedia.org/wiki/Alan_Turing", "#firstHeading"),
		("https://docs.python.org/3/", "h1"),
	]
	mods = [" ", ":not(*)", ":nth-child(1)", ", h9", "#does-not-exist"]
	out = Path(args.out)
	out.parent.mkdir(parents=True, exist_ok=True)

	def _write(obj: dict) -> None:
		with out.open("a", encoding="utf-8") as f:
			f.write(json.dumps(obj, ensure_ascii=False) + "\n")

	count = 0
	while count < int(args.num):
		url, sel = random.choice(base)
		m = random.choice(mods)
		bad_sel = f"{sel}{m}"
		r = fetch_text(url, bad_sel)
		obj = {"url": url, "selector": bad_sel, "text": r.get("text", "")}
		obj["success"] = bool(obj["text"]) and bad_sel != "#does-not-exist"
		_write(obj)
		count += 1
	print(str(out))
	return 0

if __name__ == "__main__":
	import sys
	sys.exit(main())


import logging
import argparse
import json
from pathlib import Path

def main() -> int:
	ap = argparse.ArgumentParser()
	root = Path(__file__).resolve().parents[1]
	ap.add_argument("--metrics", default=str(root / "models" / "verifier_calib" / "metrics.json"))
	ap.add_argument("--max-ece", type=float, default=0.15)
	args = ap.parse_args()

	p = Path(args.metrics)
	if not p.exists():
		print(json.dumps({"ok": False, "error": "metrics_missing", "path": str(p)}))
		return 1
	try:
		obj = json.loads(p.read_text(encoding="utf-8"))
	except Exception:
		print(json.dumps({"ok": False, "error": "metrics_invalid", "path": str(p)}))
		return 1
	ece = float((obj or {}).get("ece", (obj or {}).get("calibration_ece", 1.0)))
	ok = (ece <= float(args.max_ece))
	print(json.dumps({"ok": bool(ok), "ece": ece, "max_ece": float(args.max_ece)}))
	return 0 if ok else 1

if __name__ == "__main__":
	import sys
	sys.exit(main())


import logging
import argparse
import json
from pathlib import Path

def main() -> int:
	ap = argparse.ArgumentParser()
	root = Path(__file__).resolve().parents[1]
	ap.add_argument("--summary", default=str(root / "data" / "dashboards" / "summary.json"))
	ap.add_argument("--max-p90", type=float, default=10.0)
	args = ap.parse_args()

	p = Path(args.summary)
	if not p.exists():
		print(json.dumps({"ok": False, "reason": "summary_missing", "path": str(p)}))
		return 2
	obj = json.loads(p.read_text(encoding="utf-8"))
	dom = obj.get("bench", {}).get("dom_summary", {})
	p90 = float(dom.get("avg_latency_sec", 0.0))  # proxy for p90 in current summary
	ok = bool(p90 <= float(args.max_p90))
	print(json.dumps({"ok": ok, "p90_proxy": p90, "max": float(args.max_p90)}))
	return 0 if ok else 1

if __name__ == "__main__":
	raise SystemExit(main())


import logging
import argparse
import json
from pathlib import Path

def _run() -> int:
	ap = argparse.ArgumentParser(description="Assert zero unsafe edits in dev-loop logs (ADR Patch Safety)")
	root = Path(__file__).resolve().parents[1]
	ap.add_argument("--logs", default=str(root / "data" / "logs" / "scheduler_runs.jsonl"))
	args = ap.parse_args()

	unsafe = 0
	total = 0
	log_path = Path(args.logs)
	if log_path.exists():
		with log_path.open("r", encoding="utf-8") as f:
			for line in f:
				line = line.strip()
				if not line:
					continue
				try:
					obj = json.loads(line)
				except Exception:
					continue
				# Detect unsafe diff validation signals in stdout tail
				stderr_tail = str(obj.get("stderr_tail", ""))
				stdout_tail = str(obj.get("stdout_tail", ""))
				total += 1
				if ("diff_validation_failed" in stderr_tail) or ("diff_validation_failed" in stdout_tail):
					unsafe += 1
	out = {"ok": bool(unsafe == 0), "total": int(total), "unsafe": int(unsafe)}
	print(json.dumps(out))
	return 0 if unsafe == 0 else 1

def main() -> None:
	import sys
	sys.exit(_run())

if __name__ == "__main__":
	main()

# Backward-compatible alias for tests that import main as ci_main
ci_main = main


		with log_path.open("r", encoding="utf-8") as f:
			for line in f:
				line = line.strip()
				if not line:
					continue
				try:
					obj = json.loads(line)
				except Exception:
					continue
				# Detect unsafe diff validation signals in stdout tail
				stderr_tail = str(obj.get("stderr_tail", ""))
				stdout_tail = str(obj.get("stdout_tail", ""))
				total += 1
				if ("diff_validation_failed" in stderr_tail) or ("diff_validation_failed" in stdout_tail):
					unsafe += 1
	out = {"ok": bool(unsafe == 0), "total": int(total), "unsafe": int(unsafe)}
	print(json.dumps(out))
	return 0 if unsafe == 0 else 1

def main() -> None:
	import sys
	sys.exit(_run())

if __name__ == "__main__":
	main()

# Backward-compatible alias for tests that import main as ci_main
ci_main = main


#!/usr/bin/env python3
import logging
import argparse
import json
import os
from pathlib import Path

def parse_args():
	root = Path(__file__).resolve().parents[1]
	ap = argparse.ArgumentParser(description="CI gate for code style/type violations from code-style task")
	ap.add_argument("--style", default=str(root / "data" / "llm_bench" / "code_style.json"))
	ap.add_argument("--max-ruff", type=int, default=int(os.environ.get("MAX_CODE_RUFF", "0") or 0))
	ap.add_argument("--max-flake8", type=int, default=int(os.environ.get("MAX_CODE_FLAKE8", "0") or 0))
	ap.add_argument("--max-mypy", type=int, default=int(os.environ.get("MAX_CODE_MYPY", "0") or 0))
	return ap.parse_args()

def main() -> int:
	args = parse_args()
	p = Path(args.style)
	if not p.exists():
		print(json.dumps({"ok": False, "error": "style_summary_missing", "path": str(p)}))
		return 1
	try:
		obj = json.loads(p.read_text(encoding="utf-8"))
	except Exception as e:
		print(json.dumps({"ok": False, "error": "style_summary_parse_error", "detail": str(e)}))
		return 1

	vio = obj.get("violations", {})
	failures = []
	if args.max_ruff >= 0 and int(vio.get("ruff", 0)) > args.max_ruff:
		failures.append({"tool": "ruff", "count": int(vio.get("ruff", 0)), "max": args.max_ruff})
	if args.max_flake8 >= 0 and int(vio.get("flake8", 0)) > args.max_flake8:
		failures.append({"tool": "flake8", "count": int(vio.get("flake8", 0)), "max": args.max_flake8})
	if args.max_mypy >= 0 and int(vio.get("mypy", 0)) > args.max_mypy:
		failures.append({"tool": "mypy", "count": int(vio.get("mypy", 0)), "max": args.max_mypy})

	ok = len(failures) == 0
	print(json.dumps({"ok": ok, "failures": failures}))
	return 0 if ok else 1

if __name__ == "__main__":
	raise SystemExit(main())


#!/usr/bin/env python3
import logging
import argparse
import json
import os
from pathlib import Path

def parse_args():
	root = Path(__file__).resolve().parents[1]
	ap = argparse.ArgumentParser(description="CI gate for code style/type violations from code-style task")
	ap.add_argument("--style", default=str(root / "data" / "llm_bench" / "code_style.json"))
	ap.add_argument("--max-ruff", type=int, default=int(os.environ.get("MAX_CODE_RUFF", "0") or 0))
	ap.add_argument("--max-flake8", type=int, default=int(os.environ.get("MAX_CODE_FLAKE8", "0") or 0))
	ap.add_argument("--max-mypy", type=int, default=int(os.environ.get("MAX_CODE_MYPY", "0") or 0))
	return ap.parse_args()

def main() -> int:
	args = parse_args()
	p = Path(args.style)
	if not p.exists():
		print(json.dumps({"ok": False, "error": "style_summary_missing", "path": str(p)}))
		return 1
	try:
		obj = json.loads(p.read_text(encoding="utf-8"))
	except Exception as e:
		print(json.dumps({"ok": False, "error": "style_summary_parse_error", "detail": str(e)}))
		return 1

	vio = obj.get("violations", {})
	failures = []
	if args.max_ruff >= 0 and int(vio.get("ruff", 0)) > args.max_ruff:
		failures.append({"tool": "ruff", "count": int(vio.get("ruff", 0)), "max": args.max_ruff})
	if args.max_flake8 >= 0 and int(vio.get("flake8", 0)) > args.max_flake8:
		failures.append({"tool": "flake8", "count": int(vio.get("flake8", 0)), "max": args.max_flake8})


import logging
import argparse
import json
import subprocess
from pathlib import Path
from typing import Dict, List

def run_once(root: Path, actuator: str, task: str, timeout: int, hf_model: str, extra_args: List[str]) -> Dict:
	cmd = [
		"python3",
		str(root / "scripts" / "run_loop_oscli.py"),
		"--planner-backend",
		"hf",
		"--verifier-backend",
		"hf",
		"--planner-model",
		hf_model,
		"--verifier-model",
		hf_model,
		"--timeout",
		str(timeout),
		"--task",
		task,
		"--actuator",
		actuator,
	]
	# Enable learned router when evaluating router mode
	if actuator == "router":
		cmd.append("--learned-router")
		# Use tuned threshold if available in the packed router model
		cmd.append("--router-use-packed-threshold")
	cmd.extend(extra_args)
	res = subprocess.run(cmd, capture_output=True, text=True)
	out_lines = res.stdout.strip().splitlines()
	# Find last valid JSON object line with a status field
	for line in reversed(out_lines):
		line = line.strip()
		if not (line.startswith("{") and line.endswith("}")):
			continue
		try:
			obj = json.loads(line)
			if isinstance(obj, dict) and "status" in obj:
				return obj
		except Exception:
			continue
	# Fallback: try parsing last line
	last = out_lines[-1] if out_lines else "{}"
	try:
		return json.loads(last)
	except Exception:
		return {"status": "error", "raw": last}

def main() -> int:
	ap = argparse.ArgumentParser()
	root = Path(__file__).resolve().parents[1]
	ap.add_argument("--hf-model", default="meta-llama/Llama-3.2-3B")
	ap.add_argument("--timeout", type=int, default=60)
	ap.add_argument("--runs", type=int, default=1)
	ap.add_argument("--wm", action="store_true")
	ap.add_argument("--calib", action="store_true")
	ap.add_argument("--only-router", action="store_true")
	args = ap.parse_args()

	extra: List[str] = []
	if args.wm:
		extra.append("--wm-prior")
	if args.calib:
		extra.append("--calibrate-verifier")

	actuators = ["two_head", "t5", "nn", "router", "template"]
	if args.only_router:
		actuators = ["router"]
	tasks = ["count_lines", "grep_error"]

	summary: Dict[str, Dict[str, float]] = {}
	per_task: Dict[str, Dict[str, float]] = {}
	per_expert: Dict[str, Dict[str, float]] = {}
	router_pick_counts: Dict[str, int] = {"t5": 0, "nn": 0, "two_head": 0}
	router_probs: List[float] = []
	for act in actuators:
		total = 0
		success = 0
		for _ in range(args.runs):
			for t in tasks:
				info = run_once(root, act, t, args.timeout, args.hf_model, extra)
				total += 1
				if info.get("status") == "ok":
					success += 1
				key = f"{act}:{t}"
				per_task[key] = per_task.get(key, {"n": 0, "success": 0.0})
				per_task[key]["n"] += 1
				per_task[key]["success"] += 1 if info.get("status") == "ok" else 0
				# Track per-expert aggregates
				per_expert[act] = per_expert.get(act, {"n": 0, "success": 0.0})
				per_expert[act]["n"] += 1
				per_expert[act]["success"] += 1 if info.get("status") == "ok" else 0
				# If router mode, aggregate pick distribution and probabilities
				if act == "router":
					picked = str(info.get("router_picked", ""))
					if picked in router_pick_counts:
						router_pick_counts[picked] += 1
					prob = info.get("router_prob")
					if isinstance(prob, (int, float)):
						router_probs.append(float(prob))
		summary[act] = {"success_rate": (success / total) if total else 0.0, "n": total}

	# Convenience: include router entry as top-level if present
	# Convert counts to rates for per_expert
	per_expert_rates: Dict[str, Dict[str, float]] = {}
	for k, v in per_expert.items():
		n = float(v.get("n", 0.0))
		s = float(v.get("success", 0.0))
		per_expert_rates[k] = {"success_rate": (s / n) if n else 0.0, "n": n}

	# Router metrics
	router_metrics = summary.get("router", {}).copy() if isinstance(summary.get("router", {}), dict) else {}
	if router_probs:
		try:
			import statistics as _stats
			router_metrics.update({
				"router_prob_mean": float(_stats.fmean(router_probs)),
				"router_prob_median": float(_stats.median(router_probs)),
				"router_prob_count": int(len(router_probs)),
			})
			# Entropy over Bernoulli prob p: -[p log p + (1-p) log (1-p)] base 2
			import math as _m
			def _bern_entropy(p: float) -> float:
				p = max(1e-9, min(1 - 1e-9, p))
				return float(- (p * _m.log2(p) + (1 - p) * _m.log2(1 - p)))
			ents = [_bern_entropy(p) for p in router_probs]
			router_metrics.update({
				"router_entropy_mean": float(_stats.fmean(ents)),
				"router_entropy_median": float(_stats.median(ents)),
			})
		except Exception:
			router_metrics["router_prob_count"] = int(len(router_probs))
	router_metrics.update({
		"router_picks": router_pick_counts,
	})
	out = {"router": router_metrics, "all": summary, "per_task": per_task, "per_expert": per_expert_rates}
	print(json.dumps(out))
	return 0

if __name__ == "__main__":
	raise SystemExit(main())