import logging
import argparse
import json
from pathlib import Path
from typing import Any, Dict, Tuple, Optional, List

from agi_dw.bench.common.safe_shell import SafeShellRunner
from datetime import datetime
from agi_dw.bench.common.trace import build_trace, write_jsonl
from agi_dw.bench.os_cli.tasks import setup_count_lines, setup_grep_word
from agi_dw.core.planner.service import PlannerConfig, VerifierConfig, WMConfig, ContextAugment, plan_with_context
from agi_dw.core.ops.tracing import trace_span, meter_cost  # type: ignore
from agi_dw.core.actuator.service import ActuatorConfig, RouterExtras, RouterVerifierConfig, WMPriorConfig, WMScreenConfig, RepairConfig, select_action
from agi_dw.core.memory.service import match_skill_action

def prepare_task(sandbox: Path, task: str) -> Tuple[Dict[str, Any], str]:
	if task == "count_lines":
		setup_count_lines(sandbox)
		obs = {"kind": "cli", "content": "Count file lines", "meta": {"cwd": str(sandbox)}}
		return obs, "count_lines"
	elif task == "grep_error":
		setup_grep_word(sandbox)
		obs = {"kind": "cli", "content": "Find ERROR lines", "meta": {"cwd": str(sandbox)}}
		return obs, "grep_error"
	else:
		raise ValueError("Unknown task: " + task)

def main() -> int:
	parser = argparse.ArgumentParser()
	root = Path(__file__).resolve().parents[1]
	parser.add_argument("--sandbox", default=str(root / "data" / "sandbox"))
	parser.add_argument("--out", default=str(root / "data" / "traces" / "loop_run.jsonl"))
	parser.add_argument("--task", choices=["count_lines", "grep_error"], default="count_lines")
	parser.add_argument("--planner-backend", choices=["hf"], default="hf")
	parser.add_argument("--planner-model", default="Qwen/Qwen2.5-7B-Instruct")
	parser.add_argument("--planner-adapter", default=None)
	parser.add_argument("--planner-adapter-bank", default=None, help="Name of adapter bank to use for planner (skills registry)")
	parser.add_argument("--planner-structured", choices=["none", "json"], default="none")
	parser.add_argument("--verifier-backend", choices=["hf"], default="hf")
	parser.add_argument("--verifier-model", default="Qwen/Qwen2.5-7B-Instruct")
	parser.add_argument("--verifier-adapter", default=None)
	parser.add_argument("--verifier-adapter-bank", default=None, help="Name of adapter bank to use for verifier (skills registry)")
	parser.add_argument("--verifier-structured", choices=["none", "json"], default="json")
	parser.add_argument("--timeout", type=int, default=30)
	parser.add_argument("--il", default=str(root / "data" / "skills" / "actuator_il.jsonl"))
	parser.add_argument("--actuator", choices=["template", "two_head", "router", "t5", "nn"], default="template")
	parser.add_argument("--t5-model", default=str(root / "models" / "actuator_il_t5"))
	parser.add_argument("--auto-t5", action="store_true")
	parser.add_argument("--strict-verify", action="store_true")
	parser.add_argument("--log-prompts", action="store_true")
	parser.add_argument("--log-repair", action="store_true")
	parser.add_argument("--log-router", action="store_true")
	parser.add_argument("--wm-prior", action="store_true", help="Compute and log WM prior scores")
	parser.add_argument("--wm-model", default=str(root / "models" / "wm_mlp" / "wm_mlp.joblib"))
	parser.add_argument("--wm-screen", action="store_true", help="Use WM prior to screen and pick lower-risk action between two actuators")
	parser.add_argument("--wm-threshold", type=float, default=0.7, help="Consider alternative if WM risk >= threshold")
	parser.add_argument("--wm-safety", action="store_true", help="Enforce WM safety budget: block actions with risk above max")
	parser.add_argument("--wm-max-risk", type=float, default=0.85, help="Max allowed WM risk before blocking [0..1]")
	parser.add_argument("--wm-min-success", type=float, default=0.0, help="If >0 and WM prior enabled, block action if success_prob < threshold")
	parser.add_argument("--wm-plan-rank", action="store_true", help="Rank planner candidates with WM rollout/prior and pick lowest-risk")
	parser.add_argument("--wm-horizon", type=int, default=1, help="Short-horizon rollout steps for plan ranking")
	parser.add_argument("--planner-candidates", type=int, default=1, help="If >1, generate multiple plans and pick by lowest verifier risk")
	parser.add_argument("--calibrate-verifier", action="store_true", help="Apply isotonic calibration to verifier risk")
	parser.add_argument("--calib-model", default=str(root / "models" / "verifier_calib" / "calib.joblib"))
	parser.add_argument("--learned-router", action="store_true", help="Enable learned router for actuator selection")
	parser.add_argument("--router-model", default=str(root / "models" / "router" / "router.joblib"))
	parser.add_argument("--router-threshold", type=float, default=0.5, help="Threshold on P(success) to pick T5 over NN")
	parser.add_argument("--router-use-packed-threshold", action="store_true", help="Use threshold saved in router model pack if available")
	parser.add_argument("--router-thresholds-json", default=None, help="Optional JSON file mapping task->threshold override, e.g., {\"count_lines\":0.4}")
	parser.add_argument("--use-memory", action="store_true")
	parser.add_argument("--mem-path", default=str(root / "models" / "memory"))
	parser.add_argument("--mem-topk", type=int, default=3)
	parser.add_argument("--mem-recency", type=float, default=0.0, help="Recency weighting [0..1] when ranking memory hits")
	parser.add_argument("--mem-query", default=None, help="Optional explicit memory query text to override default obs-based query")
	parser.add_argument("--planner-tot", action="store_true", help="Use ToT-style candidate generator for plans")
	parser.add_argument("--planner-pref-weights", default=None, help="Path to JSON weights for reranking candidates: {w_verifier, w_wm, w_self}")
	parser.add_argument("--require-approval", action="store_true", help="Require human approval for high-risk actions (verifier-gated)")
	parser.add_argument("--approval-threshold", type=float, default=0.8, help="Risk threshold to trigger approval checkpoint [0..1]")
	parser.add_argument("--approval-dir", default=str(root / "data" / "approvals"))
	parser.add_argument("--planner-seeded", action="store_true", help="Use seeded candidate plans for known CLI tasks (no LLM)")
	# Optional: include top-k code index candidates in planner observation
	parser.add_argument("--planner-index-k", type=int, default=0, help="If >0, include top-k code index candidates in planner obs")
	parser.add_argument("--planner-index-path", default=str(root / "data" / "sandbox" / "tmp" / "code_index.json"), help="Optional prebuilt code index JSON path")
	# Meta-controller integration
	parser.add_argument("--meta-state", default=str(Path(__file__).resolve().parents[2] / "data" / "sandbox" / "tmp" / "meta_state.json"))
	parser.add_argument("--apply-meta", action="store_true")
	args = parser.parse_args()

	sandbox = Path(args.sandbox)
	# Apply meta-state if requested
	if bool(getattr(args, "apply_meta", False)):
		try:
			import os as _os, json as _json  # type: ignore
			m = _json.loads(Path(str(args.meta_state)).read_text(encoding="utf-8"))
			_os.environ["PLANNER_TOT"] = "1" if bool(m.get("enable_tot", False)) else "0"
			_os.environ["SAMPLING_TEMPERATURE"] = str(float(m.get("sampling_temperature", 0.2)))
			_os.environ["PLANNER_STEP_BUDGET"] = str(int(m.get("planner_step_budget", 1)))
		except Exception:
			pass
	traces_out = Path(args.out)

	obs, task_name = prepare_task(sandbox, args.task)

	# Defaults favor structured planner output and WM usage for better training signal
	try:
		args.planner_structured = "json"
		args.wm_prior = True
		# If generating multiple candidates, also enable WM plan ranking by default
		if int(getattr(args, "planner_candidates", 1) or 1) > 1:
			args.wm_plan_rank = True
	except Exception:
		pass

	# Consolidated planning via planner service
	mem_snippets: List[Dict[str, Any]] = []
	mem_query_ms: Optional[float] = None
	plan = None
	planner_info = None
	obs_aug = dict(obs)
	with trace_span("plan", {"backend": str(args.planner_backend), "candidates": int(getattr(args, "planner_candidates", 1) or 1)}):
		pl_cfg = PlannerConfig(
							model=args.planner_model,
			backend=args.planner_backend,
							timeout_sec=args.timeout,
							adapter_dir=args.planner_adapter,
			adapter_bank=getattr(args, "planner_adapter_bank", None),
							structured_mode=str(args.planner_structured),
			candidates=int(getattr(args, "planner_candidates", 1) or 1),
			use_tot=bool(getattr(args, "planner_tot", False)),
			seeded=bool(getattr(args, "planner_seeded", False)),
			pref_weights_path=getattr(args, "planner_pref_weights", None),
		)
		vf_cfg = VerifierConfig(
			model=args.verifier_model,
			backend=args.verifier_backend,
			adapter_dir=args.verifier_adapter,
			adapter_bank=getattr(args, "verifier_adapter_bank", None),
			structured_mode=str(args.verifier_structured),
		)
		wm_cfg = WMConfig(
			enabled=bool(getattr(args, "wm_prior", False)),
			model_path=str(args.wm_model),
			horizon=int(getattr(args, "wm_horizon", 1) or 1),
			plan_rank=bool(getattr(args, "wm_plan_rank", False)),
		)
		ctx_cfg = ContextAugment(
			use_memory=bool(getattr(args, "use_memory", False)),
			mem_path=str(args.mem_path),
			mem_topk=int(getattr(args, "mem_topk", 3) or 3),
			mem_recency=float(getattr(args, "mem_recency", 0.0) or 0.0),
			mem_query=getattr(args, "mem_query", None),
			index_k=int(getattr(args, "planner_index_k", 0) or 0),
			index_path=str(getattr(args, "planner_index_path", "")),
			inject_dom_policy=False,
			inject_cli_policy=True,
			inject_caps=True,
		)
		critic_thr = float(getattr(args, "approval_threshold", 0.8) or 0.8) if int(getattr(args, "planner_candidates", 1) or 1) <= 1 else None
		plan, planner_info, obs_aug, mem_snippets, mem_query_ms = plan_with_context(
			obs,
			"cli",
			pl_cfg,
			vf_cfg,
			wm_cfg,
			ctx_cfg,
			critic_fallback_threshold=critic_thr,
			log_prompts=bool(getattr(args, "log_prompts", False)),
			task_name=task_name,
		)

	# Optionally auto-select T5 actuator if a trained model exists
	if args.auto_t5 and args.actuator in ("template", "two_head", "router", "nn"):
		from pathlib import Path as _P
		try:
			m_dir = _P(args.t5_model)
			if m_dir.exists() and any(m_dir.iterdir()):
				args.actuator = "t5"
		except Exception:
			pass

	# Compute WM prior early (before routing) to inform router features
	wm_prior: Optional[Dict[str, float]] = None
	if args.wm_prior:
		try:
			from agi_dw.core.world_model.service import WorldModelService  # type: ignore
			model_path = Path(args.wm_model)
			if model_path.exists():
				wm = WorldModelService.load_if_exists(model_path)
				if wm:
					wm_prior = wm.predict_prior(obs_aug if 'obs_aug' in locals() else obs, plan, action={})
		except Exception:
			wm_prior = None

	# Centralized action selection via actuator service
	act_cfg = ActuatorConfig(
		mode=str(args.actuator),
		t5_model=str(args.t5_model),
		il_path=str(args.il),
		learned_router=bool(getattr(args, "learned_router", False)),
		router_model_path=str(args.router_model),
		router_threshold=float(getattr(args, "router_threshold", 0.5) or 0.5),
		router_use_packed_threshold=bool(getattr(args, "router_use_packed_threshold", False)),
		router_thresholds_json=getattr(args, "router_thresholds_json", None),
	)
	vf_cfg = RouterVerifierConfig(
		model=str(args.verifier_model),
		backend=str(args.verifier_backend),
		adapter_dir=getattr(args, "verifier_adapter", None),
		adapter_bank=getattr(args, "verifier_adapter_bank", None),
		timeout_sec=int(getattr(args, "timeout", 30) or 30),
		structured_mode=str(getattr(args, "verifier_structured", "json")),
	)
	wm_cfg = WMPriorConfig(enabled=bool(getattr(args, "wm_prior", False)), model_path=str(args.wm_model))
	wm_scr = WMScreenConfig(enabled=bool(getattr(args, "wm_screen", False)), threshold=float(getattr(args, "wm_threshold", 0.7) or 0.7))
	repair = RepairConfig(domain="cli")
	extra = RouterExtras(domain="cli", wm_prior=wm_prior, task_name=str(getattr(args, "task", "") or ""), log_router=bool(getattr(args, "log_router", False)))
	action, router_decision = select_action(obs, plan, act_cfg, extra, verifier_cfg=vf_cfg, wm_prior_cfg=wm_cfg, wm_screen_cfg=wm_scr, repair_cfg=repair)

	# Optional WM screen: compare with an alternative actuator and pick lower-risk by WM
	if args.wm_screen and args.wm_prior:
		try:
			from agi_dw.core.world_model.api import WorldModelPrior  # type: ignore
			wm_path = Path(args.wm_model)
			if wm_path.exists():
				wm = WorldModelPrior.load(wm_path)
				# Score primary
				prior_main = wm.predict_prior(obs, plan, action or {}) or {"risk": 0.5}
				risk_main = float(prior_main.get("risk", 0.5))
				# Build alternative via actuator service for consistency
				try:
					from agi_dw.core.actuator.service import ActuatorConfig, RouterExtras, select_action
					alt_cfg = ActuatorConfig(mode=("t5" if args.actuator in ("template", "two_head", "nn") else "nn"), t5_model=str(args.t5_model), il_path=str(args.il))
					alt_extra = RouterExtras(domain="cli")
					alt_action, _ = select_action(obs, plan, alt_cfg, alt_extra)
				except Exception:
					alt_action = {}
				prior_alt = wm.predict_prior(obs, plan, alt_action or {}) or {"risk": 0.5}
				risk_alt = float(prior_alt.get("risk", 0.5))
				# Pick lower risk if main is high risk
				if risk_main >= float(args.wm_threshold) and (risk_alt < risk_main):
					action = alt_action or action
					# annotate selection
					if isinstance(action, dict):
						action["wm_screen"] = {"risk_main": risk_main, "risk_alt": risk_alt, "success_entropy_main": float(prior_main.get("success_entropy", 0.0)), "success_entropy_alt": float(prior_alt.get("success_entropy", 0.0))}
		except Exception:
			pass

	# Try Skill Library match to reuse a promoted skill
	try:
		action, adapters = match_skill_action("cli", Path(__file__).resolve().parents[1], obs, action)
		if adapters.get("verifier"):
			args.verifier_adapter = adapters["verifier"]
		if adapters.get("planner"):
			args.planner_adapter = adapters["planner"]
	except Exception:
		pass

	pred_action = json.loads(json.dumps(action, ensure_ascii=False))  # shallow copy for diff

	# Ensure action is a dict so repair can synthesize if needed
	if not isinstance(action, dict):
		action = {}

	# Action already repaired by actuator service when configured; keep as-is

	# WM safety budget: block execution if WM risk exceeds max
	if args.wm_safety and args.wm_prior:
		try:
			from agi_dw.core.world_model.service import WorldModelService  # type: ignore
			wm_path = Path(args.wm_model)
			if wm_path.exists():
				wm = WorldModelService.load_if_exists(wm_path)
				prior_now = wm.predict_prior(obs, plan, action or {}) if wm else {"risk": 0.5}
				r_now = float(prior_now.get("risk", 0.5))
				if r_now >= float(args.wm_max_risk):
					trace = build_trace(
						task_id="loop-" + args.task,
						obs=obs,
						plan=plan,
						action=action,
						result={"stdout": "", "stderr": "", "status": "blocked"},
						reward={"scalar": 0.0, "components": {"success": 0, "latency": 0, "side_effect": 1}},
						critique={"issues": [], "risk": r_now, "proposal": "wm-safety-budget"},
					)
					try:
						import os as _os
						trace["meta_state"] = {
							"enable_tot": True if _os.environ.get("PLANNER_TOT", "0") == "1" else False,
							"temperature": float(_os.environ.get("SAMPLING_TEMPERATURE", "0.2") or 0.2),
							"step_budget": int(_os.environ.get("PLANNER_STEP_BUDGET", "1") or 1),
						}
					except Exception:
						pass
					write_jsonl(str(traces_out), trace)
					print(json.dumps({"status": "blocked", "wm_risk": r_now, "wm_max_risk": float(args.wm_max_risk)}, ensure_ascii=False))
					return 0
		except Exception:
			pass

	# WM success budget: block execution if WM success_prob below min
	if args.wm_prior and float(getattr(args, "wm_min_success", 0.0) or 0.0) > 0.0:
		try:
			from agi_dw.core.world_model.service import WorldModelService  # type: ignore
			wm_path = Path(args.wm_model)
			if wm_path.exists():
				wm = WorldModelService.load_if_exists(wm_path)
				prior_now = wm.predict_prior(obs, plan, action or {}) if wm else {"success_prob": 0.5}
				p_now = float(prior_now.get("success_prob", 0.5))
				if p_now < float(args.wm_min_success):
					trace = build_trace(
						task_id="loop-" + args.task,
						obs=obs,
						plan=plan,
						action=action,
						result={"stdout": "", "stderr": "", "status": "blocked"},
						reward={"scalar": 0.0, "components": {"success": 0, "latency": 0, "side_effect": 1}},
						critique={"issues": [], "risk": float((prior_now or {}).get("risk", 0.5)), "proposal": "wm-success-budget"},
					)
					try:
						import os as _os
						trace["meta_state"] = {
							"enable_tot": True if _os.environ.get("PLANNER_TOT", "0") == "1" else False,
							"temperature": float(_os.environ.get("SAMPLING_TEMPERATURE", "0.2") or 0.2),
							"step_budget": int(_os.environ.get("PLANNER_STEP_BUDGET", "1") or 1),
						}
					except Exception:
						pass
					write_jsonl(str(traces_out), trace)
					print(json.dumps({"status": "blocked", "wm_success_prob": p_now, "wm_min_success": float(args.wm_min_success)}, ensure_ascii=False))
					return 0
		except Exception:
			pass

	# Verifier-gated human approval (pre-execution)
	if args.require_approval:
		try:
			from agi_dw.core.verifier.service import VerifierServiceConfig, verify as verifier_run  # type: ignore
			pre_check_trace = {"obs": obs, "plan": plan, "action": action, "result": {"status": "pending"}}
			v_cfg = VerifierServiceConfig(
				model=str(args.verifier_model),
				backend=str(args.verifier_backend),
				adapter_dir=getattr(args, "verifier_adapter", None),
				adapter_bank=getattr(args, "verifier_adapter_bank", None),
				structured_mode=str(args.verifier_structured),
				timeout_sec=max(2, int(args.timeout)),
				strict=False,
				calibrate=False,
				log_prompts=bool(getattr(args, "log_prompts", False)),
			)
			vpre = verifier_run(pre_check_trace, v_cfg)
			risk_val = float(vpre.get("risk", 0.5))
			if risk_val >= float(args.approval_threshold):
				ap_dir = Path(args.approval_dir)
				ap_dir.mkdir(parents=True, exist_ok=True)
				ap_id = datetime.utcnow().strftime("%Y%m%dT%H%M%SZ")
				ap_path = ap_dir / f"approval_oscli_{ap_id}.json"
				ap_path.write_text(json.dumps({"obs": obs, "plan": plan, "action": action, "verifier": vpre, "threshold": float(args.approval_threshold)}, ensure_ascii=False, indent=2), encoding="utf-8")
				print(json.dumps({"stage": "approval_block", "risk": risk_val, "threshold": float(args.approval_threshold), "request": str(ap_path)}, ensure_ascii=False))
				# Log a blocked trace and exit without executing
				trace = build_trace(
					task_id="loop-" + args.task,
					obs=obs,
					plan=plan,
					action=action,
					result={"stdout": "", "stderr": "", "status": "blocked"},
					reward={"scalar": 0.0, "components": {"success": 0, "latency": 0, "side_effect": 1}},
					critique={"issues": [], "risk": risk_val, "proposal": "approval-required"},
				)
				try:
					import os as _os
					trace["meta_state"] = {
						"enable_tot": True if _os.environ.get("PLANNER_TOT", "0") == "1" else False,
						"temperature": float(_os.environ.get("SAMPLING_TEMPERATURE", "0.2") or 0.2),
						"step_budget": int(_os.environ.get("PLANNER_STEP_BUDGET", "1") or 1),
					}
				except Exception:
					pass
				write_jsonl(str(traces_out), trace)
				return 0
		except Exception:
			pass

	# Optionally log repair as IL example if changed
	if args.log_repair and action != pred_action:
		inp = json.dumps({"obs": obs, "plan": plan}, ensure_ascii=False)
		out = json.dumps(action, ensure_ascii=False)
		repair_path = root / "data" / "skills" / "actuator_il_repairs.jsonl"
		repair_path.parent.mkdir(parents=True, exist_ok=True)
		with open(repair_path, "a", encoding="utf-8") as f:
			f.write(json.dumps({"input": inp, "output": out}, ensure_ascii=False) + "\n")

	runner = SafeShellRunner(str(sandbox))
	argv = action.get("args", {}).get("ar
# ... truncated ...


import logging
import argparse
import json
from pathlib import Path
from typing import Any, Dict, Tuple, Optional, List

from agi_dw.bench.common.safe_shell import SafeShellRunner
from datetime import datetime
from agi_dw.bench.common.trace import build_trace, write_jsonl
from agi_dw.bench.os_cli.tasks import setup_count_lines, setup_grep_word
from agi_dw.core.planner.service import PlannerConfig, VerifierConfig, WMConfig, ContextAugment, plan_with_context
from agi_dw.core.ops.tracing import trace_span, meter_cost  # type: ignore
from agi_dw.core.actuator.service import ActuatorConfig, RouterExtras, RouterVerifierConfig, WMPriorConfig, WMScreenConfig, RepairConfig, select_action
from agi_dw.core.memory.service import match_skill_action

def prepare_task(sandbox: Path, task: str) -> Tuple[Dict[str, Any], str]:
	if task == "count_lines":
		setup_count_lines(sandbox)
		obs = {"kind": "cli", "content": "Count file lines", "meta": {"cwd": str(sandbox)}}
		return obs, "count_lines"
	elif task == "grep_error":
		setup_grep_word(sandbox)
		obs = {"kind": "cli", "content": "Find ERROR lines", "meta": {"cwd": str(sandbox)}}
		return obs, "grep_error"
	else:
		raise ValueError("Unknown task: " + task)

def main() -> int:
	parser = argparse.ArgumentParser()
	root = Path(__file__).resolve().parents[1]
	parser.add_argument("--sandbox", default=str(root / "data" / "sandbox"))
	parser.add_argument("--out", default=str(root / "data" / "traces" / "loop_run.jsonl"))
	parser.add_argument("--task", choices=["count_lines", "grep_error"], default="count_lines")
	parser.add_argument("--planner-backend", choices=["hf"], default="hf")
	parser.add_argument("--planner-model", default="Qwen/Qwen2.5-7B-Instruct")
	parser.add_argument("--planner-adapter", default=None)
	parser.add_argument("--planner-adapter-bank", default=None, help="Name of adapter bank to use for planner (skills registry)")
	parser.add_argument("--planner-structured", choices=["none", "json"], default="none")
	parser.add_argument("--verifier-backend", choices=["hf"], default="hf")
	parser.add_argument("--verifier-model", default="Qwen/Qwen2.5-7B-Instruct")
	parser.add_argument("--verifier-adapter", default=None)
	parser.add_argument("--verifier-adapter-bank", default=None, help="Name of adapter bank to use for verifier (skills registry)")
	parser.add_argument("--verifier-structured", choices=["none", "json"], default="json")
	parser.add_argument("--timeout", type=int, default=30)
	parser.add_argument("--il", default=str(root / "data" / "skills" / "actuator_il.jsonl"))


main

from agi_dw.bench.os_cli.tasks import setup_count_lines, setup_grep_word
from agi_dw.core.planner.service import PlannerConfig, VerifierConfig, WMConfig, ContextAugment, plan_with_context
from agi_dw.core.ops.tracing import trace_span, meter_cost  # type: ignore
from agi_dw.core.actuator.service import ActuatorConfig, RouterExtras, RouterVerifierConfig, WMPriorConfig, WMScreenConfig, RepairConfig, select_action
from agi_dw.core.memory.service import match_skill_action

def prepare_task(sandbox: Path, task: str) -> Tuple[Dict[str, Any], str]:
	if task == "count_lines":
		setup_count_lines(sandbox)
		obs = {"kind": "cli", "content": "Count file lines", "meta": {"cwd": str(sandbox)}}
		return obs, "count_lines"
	elif task == "grep_error":
		setup_grep_word(sandbox)
		obs = {"kind": "cli", "content": "Find ERROR lines", "meta": {"cwd": str(sandbox)}}
		return obs, "grep_error"
	else:
		raise ValueError("Unknown task: " + task)

def main() -> int:
	parser = argparse.ArgumentParser()
	root = Path(__file__).resolve().parents[1]
	parser.add_argument("--sandbox", default=str(root / "data" / "sandbox"))
	parser.add_argument("--out", default=str(root / "data" / "traces" / "loop_run.jsonl"))
	parser.add_argument("--task", choices=["count_lines", "grep_error"], default="count_lines")
	parser.add_argument("--planner-backend", choices=["hf"], default="hf")
	parser.add_argument("--planner-model", default="Qwen/Qwen2.5-7B-Instruct")
	parser.add_argument("--planner-adapter", default=None)
	parser.add_argument("--planner-adapter-bank", default=None, help="Name of adapter bank to use for planner (skills registry)")
	parser.add_argument("--planner-structured", choices=["none", "json"], default="none")
	parser.add_argument("--verifier-backend", choices=["hf"], default="hf")
	parser.add_argument("--verifier-model", default="Qwen/Qwen2.5-7B-Instruct")
	parser.add_argument("--verifier-adapter", default=None)
	parser.add_argument("--verifier-adapter-bank", default=None, help="Name of adapter bank to use for verifier (skills registry)")
	parser.add_argument("--verifier-structured", choices=["none", "json"], default="json")
	parser.add_argument("--timeout", type=int, default=30)
	parser.add_argument("--il", default=str(root / "data" / "skills" / "actuator_il.jsonl"))
	parser.add_argument("--actuator", choices=["template", "two_head", "router", "t5", "nn"], default="template")
	parser.add_argument("--t5-model", default=str(root / "models" / "actuator_il_t5"))
	parser.add_argument("--auto-t5", action="store_true")
	parser.add_argument("--strict-verify", action="store_true")
	parser.add_argument("--log-prompts", action="store_true")
	parser.add_argument("--log-repair", action="store_true")
	parser.add_argument("--log-router", action="store_true")
	parser.add_argument("--wm-prior", action="store_true", help="Compute and log WM prior scores")
	parser.add_argument("--wm-model", default=str(root / "models" / "wm_mlp" / "wm_mlp.joblib"))
	parser.add_argument("--wm-screen", action="store_true", help="Use WM prior to screen and pick lower-risk action between two actuators")
	parser.add_argument("--wm-threshold", type=float, default=0.7, help="Consider alternative if WM risk >= threshold")
	parser.add_argument("--wm-safety", action="store_true", help="Enforce WM safety budget: block actions with risk above max")
	parser.add_argument("--wm-max-risk", type=float, default=0.85, help="Max allowed WM risk before blocking [0..1]")
	parser.add_argument("--wm-min-success", type=float, default=0.0, help="If >0 and WM prior enabled, block action if success_prob < threshold")
	parser.add_argument("--wm-plan-rank", action="store_true", help="Rank planner candidates with WM rollout/prior and pick lowest-risk")
	parser.add_argument("--wm-horizon", type=int, default=1, help="Short-horizon rollout steps for plan ranking")
	parser.add_argument("--planner-candidates", type=int, default=1, help="If >1, generate multiple plans and pick by lowest verifier risk")
	parser.add_argument("--calibrate-verifier", action="store_true", help="Apply isotonic calibration to verifier risk")
	parser.add_argument("--calib-model", default=str(root / "models" / "verifier_calib" / "calib.joblib"))
	parser.add_argument("--learned-router", action="store_true", help="Enable learned router for actuator selection")
	parser.add_argument("--router-model", default=str(root / "models" / "router" / "router.joblib"))
	parser.add_argument("--router-threshold", type=float, default=0.5, help="Threshold on P(success) to pick T5 over NN")
	parser.add_argument("--router-use-packed-threshold", action="store_true", help="Use threshold saved in router model pack if available")
	parser.add_argument("--router-thresholds-json", default=None, help="Optional JSON file mapping task->threshold override, e.g., {\"count_lines\":0.4}")
	parser.add_argument("--use-memory", action="store_true")
	parser.add_argument("--mem-path", default=str(root / "models" / "memory"))
	parser.add_argument("--mem-topk", type=int, default=3)
	parser.add_argument("--mem-recency", type=float, default=0.0, help="Recency weighting [0..1] when ranking memory hits")
	parser.add_argument("--mem-query", default=None, help="Optional explicit memory query text to override default obs-based query")
	parser.add_argument("--planner-tot", action="store_true", help="Use ToT-style candidate generator for plans")
	parser.add_argument("--planner-pref-weights", default=None, help="Path to JSON weights for reranking candidates: {w_verifier, w_wm, w_self}")
	parser.add_argument("--require-approval", action="store_true", help="Require human approval for high-risk actions (verifier-gated)")
	parser.add_argument("--approval-threshold", type=float, default=0.8, help="Risk threshold to trigger approval checkpoint [0..1]")
	parser.add_argument("--approval-dir", default=str(root / "data" / "approvals"))
	parser.add_argument("--planner-seeded", action="store_true", help="Use seeded candidate plans for known CLI tasks (no LLM)")
	# Optional: include top-k code index candidates in planner observation
	parser.add_argument("--planner-index-k", type=int, default=0, help="If >0, include top-k code index candidates in planner obs")
	parser.add_argument("--planner-index-path", default=str(root / "data" / "sandbox" / "tmp" / "code_index.json"), help="Optional prebuilt code index JSON path")
	# Meta-controller integration
	parser.add_argument("--meta-state", default=str(Path(__file__).resolve().parents[2] / "data" / "sandbox" / "tmp" / "meta_state.json"))
	parser.add_argument("--apply-meta", action="store_true")
	args = parser.parse_args()

	sandbox = Path(args.sandbox)
	# Apply meta-state if requested
	if bool(getattr(args, "apply_meta", False)):
		try:
			import os as _os, json as _json  # type: ignore
			m = _json.loads(Path(str(args.meta_state)).read_text(encoding="utf-8"))
			_os.environ["PLANNER_TOT"] = "1" if bool(m.get("enable_tot", False)) else "0"
			_os.environ["SAMPLING_TEMPERATURE"] = str(float(m.get("sampling_temperature", 0.2)))
			_os.environ["PLANNER_STEP_BUDGET"] = str(int(m.get("planner_step_budget", 1)))
		except Exception:
			pass
	traces_out = Path(args.out)

	obs, task_name = prepare_task(sandbox, args.task)

	# Defaults favor structured planner output and WM usage for better training signal
	try:
		args.planner_structured = "json"
		args.wm_prior = True
		# If generating multiple candidates, also enable WM plan ranking by default
		if int(getattr(args, "planner_candidates", 1) or 1) > 1:
			args.wm_plan_rank = True
	except Exception:
		pass

	# Consolidated planning via planner service
	mem_snippets: List[Dict[str, Any]] = []
	mem_query_ms: Optional[float] = None
	plan = None
	planner_info = None
	obs_aug = dict(obs)
	with trace_span("plan", {"backend": str(args.planner_backend), "candidates": int(getattr(args, "planner_candidates", 1) or 1)}):
		pl_cfg = PlannerConfig(
							model=args.planner_model,
			backend=args.planner_backend,
							timeout_sec=args.timeout,
							adapter_dir=args.planner_adapter,
			adapter_bank=getattr(args, "planner_adapter_bank", None),
							structured_mode=str(args.planner_structured),
			candidates=int(getattr(args, "planner_candidates", 1) or 1),
			use_tot=bool(getattr(args, "planner_tot", False)),
			seeded=bool(getattr(args, "planner_seeded", False)),
			pref_weights_path=getattr(args, "planner_pref_weights", None),
		)
		vf_cfg = VerifierConfig(
			model=args.verifier_model,
			backend=args.verifier_backend,
			adapter_dir=args.verifier_adapter,
			adapter_bank=getattr(args, "verifier_adapter_bank", None),
			structured_mode=str(args.verifier_structured),
		)
		wm_cfg = WMConfig(
			enabled=bool(getattr(args, "wm_prior", False)),
			model_path=str(args.wm_model),
			horizon=int(getattr(args, "wm_horizon", 1) or 1),
			plan_rank=bool(getattr(args, "wm_plan_rank", False)),
		)
		ctx_cfg = ContextAugment(
			use_memory=bool(getattr(args, "use_memory", False)),
			mem_path=str(args.mem_path),
			mem_topk=int(getattr(args, "mem_topk", 3) or 3),
			mem_recency=float(getattr(args, "mem_recency", 0.0) or 0.0),
			mem_query=getattr(args, "mem_query", None),
			index_k=int(getattr(args, "planner_index_k", 0) or 0),
			index_path=str(getattr(args, "planner_index_path", "")),
			inject_dom_policy=False,
			inject_cli_policy=True,
			inject_caps=True,
		)
		critic_thr = float(getattr(args, "approval_threshold", 0.8) or 0.8) if int(getattr(args, "planner_candidates", 1) or 1) <= 1 else None
		plan, planner_info, obs_aug, mem_snippets, mem_query_ms = plan_with_context(
			obs,
			"cli",
			pl_cfg,
			vf_cfg,
			wm_cfg,
			ctx_cfg,
			critic_fallback_threshold=critic_thr,
			log_prompts=bool(getattr(args, "log_prompts", False)),
			task_name=task_name,
		)

	# Optionally auto-select T5 actuator if a trained model exists
	if args.auto_t5 and args.actuator in ("template", "two_head", "router", "nn"):
		from pathlib import Path as _P
		try:
			m_dir = _P(args.t5_model)
			if m_dir.exists() and any(m_dir.iterdir()):
				args.actuator = "t5"
		except Exception:
			pass

	# Compute WM prior early (before routing) to inform router features
	wm_prior: Optional[Dict[str, float]] = None
	if args.wm_prior:
		try:
			from agi_dw.core.world_model.service import WorldModelService  # type: ignore
			model_path = Path(args.wm_model)
			if model_path.exists():
				wm = WorldModelService.load_if_exists(model_path)
				if wm:
					wm_prior = wm.predict_prior(obs_aug if 'obs_aug' in locals() else obs, plan, action={})
		except Exception:
			wm_prior = None

	# Centralized action selection via actuator service
	act_cfg = ActuatorConfig(
		mode=str(args.actuator),
		t5_model=str(args.t5_model),
		il_path=str(args.il),
		learned_router=bool(getattr(args, "learned_router", False)),
		router_model_path=str(args.router_model),
		router_threshold=float(getattr(args, "router_threshold", 0.5) or 0.5),
		router_use_packed_threshold=bool(getattr(args, "router_use_packed_threshold", False)),
		router_thresholds_json=getattr(args, "router_thresholds_json", None),
	)
	vf_cfg = RouterVerifierConfig(
		model=str(args.verifier_model),
		backend=str(args.verifier_backend),
		adapter_dir=getattr(args, "verifier_adapter", None),
		adapter_bank=getattr(args, "verifier_adapter_bank", None),
		timeout_sec=int(getattr(args, "timeout", 30) or 30),
		structured_mode=str(getattr(args, "verifier_structured", "json")),
	)
	wm_cfg = WMPriorConfig(enabled=bool(getattr(args, "wm_prior", False)), model_path=str(args.wm_model))
	wm_scr = WMScreenConfig(enabled=bool(getattr(args, "wm_screen", False)), threshold=float(getattr(args, "wm_threshold", 0.7) or 0.7))
	repair = RepairConfig(domain="cli")
	extra = RouterExtras(domain="cli", wm_prior=wm_prior, task_name=str(getattr(args, "task", "") or ""), log_router=bool(getattr(args, "log_router", False)))
	action, router_decision = select_action(obs, plan, act_cfg, extra, verifier_cfg=vf_cfg, wm_prior_cfg=wm_cfg, wm_screen_cfg=wm_scr, repair_cfg=repair)

	# Optional WM screen: compare with an alternative actuator and pick lower-risk by WM
	if args.wm_screen and args.wm_prior:
		try:
			from agi_dw.core.world_model.api import WorldModelPrior  # type: ignore
			wm_path = Path(args.wm_model)
			if wm_path.exists():
				wm = WorldModelPrior.load(wm_path)
				# Score primary
				prior_main = wm.predict_prior(obs, plan, action or {}) or {"risk": 0.5}
				risk_main = float(prior_main.get("risk", 0.5))
				# Build alternative via actuator service for consistency
				try:
					from agi_dw.core.actuator.service import ActuatorConfig, RouterExtras, select_action
					alt_cfg = ActuatorConfig(mode=("t5" if args.actuator in ("template", "two_head", "nn") else "nn"), t5_model=str(args.t5_model), il_path=str(args.il))
					alt_extra = RouterExtras(domain="cli")
					alt_action, _ = select_action(obs, plan, alt_cfg, alt_extra)
				except Exception:
					alt_action = {}
				prior_alt = wm.predict_prior(obs, plan, alt_action or {}) or {"risk": 0.5}
				risk_alt = float(prior_alt.get("risk", 0.5))
				# Pick lower risk if main is high risk
				if risk_main >= float(args.wm_threshold) and (risk_alt < risk_main):
					action = alt_action or action
					# annotate selection
					if isinstance(action, dict):
						action["wm_screen"] = {"risk_main": risk_main, "risk_alt": risk_alt, "success_entropy_main": float(prior_main.get("success_entropy", 0.0)), "success_entropy_alt": float(prior_alt.get("success_entropy", 0.0))}
		except Exception:
			pass

	# Try Skill Library match to reuse a promoted skill
	try:
		action, adapters = match_skill_action("cli", Path(__file__).resolve().parents[1], obs, action)
		if adapters.get("verifier"):
			args.verifier_adapter = adapters["verifier"]
		if adapters.get("planner"):
			args.planner_adapter = adapters["planner"]
	except Exception:
		pass

	pred_action = json.loads(json.dumps(action, ensure_ascii=False))  # shallow copy for diff

	# Ensure action is a dict so repair can synthesize if needed
	if not isinstance(action, dict):
		action = {}

	# Action already repaired by actuator service when configured; keep as-is

	# WM safety budget: block execution if WM risk exceeds max
	if args.wm_safety and args.wm_prior:
		try:
			from agi_dw.core.world_model.service import WorldModelService  # type: ignore
			wm_path = Path(args.wm_model)
			if wm_path.exists():
				wm = WorldModelService.load_if_exists(wm_path)
				prior_now = wm.predict_prior(obs, plan, action or {}) if wm else {"risk": 0.5}
				r_now = float(prior_now.get("risk", 0.5))
				if r_now >= float(args.wm_max_risk):
					trace = build_trace(
						task_id="loop-" + args.task,
						obs=obs,
						plan=plan,
						action=action,
						result={"stdout": "", "stderr": "", "status": "blocked"},
						reward={"scalar": 0.0, "components": {"success": 0, "latency": 0, "side_effect": 1}},
						critique={"issues": [], "risk": r_now, "proposal": "wm-safety-budget"},
					)
					try:
						import os as _os
						trace["meta_state"] = {
							"enable_tot": True if _os.environ.get("PLANNER_TOT", "0") == "1" else False,
							"temperature": float(_os.environ.get("SAMPLING_TEMPERATURE", "0.2") or 0.2),
							"step_budget": int(_os.environ.get("PLANNER_STEP_BUDGET", "1") or 1),
						}
					except Exception:
						pass
					write_jsonl(str(traces_out), trace)
					print(json.dumps({"status": "blocked", "wm_risk": r_now, "wm_max_risk": float(args.wm_max_risk)}, ensure_ascii=False))
					return 0
		except Exception:
			pass

	# WM success budget: block execution if WM success_prob below min
	if args.wm_prior and float(getattr(args, "wm_min_success", 0.0) or 0.0) > 0.0:
		try:
			from agi_dw.core.world_model.service import WorldModelService  # type: ignore
			wm_path = Path(args.wm_model)
			if wm_path.exists():
				wm = WorldModelService.load_if_exists(wm_path)
				prior_now = wm.predict_prior(obs, plan, action or {}) if wm else {"success_prob": 0.5}
				p_now = float(prior_now.get("success_prob", 0.5))
				if p_now < float(args.wm_min_success):
					trace = build_trace(
						task_id="loop-" + args.task,
						obs=obs,
						plan=plan,
						action=action,
						result={"stdout": "", "stderr": "", "status": "blocked"},
						reward={"scalar": 0.0, "components": {"success": 0, "latency": 0, "side_effect": 1}},
						critique={"issues": [], "risk": float((prior_now or {}).get("risk", 0.5)), "proposal": "wm-success-budget"},
					)
					try:
						import os as _os
						trace["meta_state"] = {
							"enable_tot": True if _os.environ.get("PLANNER_TOT", "0") == "1" else False,
							"temperature": float(_os.environ.get("SAMPLING_TEMPERATURE", "0.2") or 0.2),
							"step_budget": int(_os.environ.get("PLANNER_STEP_BUDGET", "1") or 1),
						}
					except Exception:
						pass
					write_jsonl(str(traces_out), trace)
					print(json.dumps({"status": "blocked", "wm_success_prob": p_now, "wm_min_success": float(args.wm_min_success)}, ensure_ascii=False))
					return 0
		except Exception:
			pass

	# Verifier-gated human approval (pre-execution)
	if args.require_approval:
		try:
			from agi_dw.core.verifier.service import VerifierServiceConfig, verify as verifier_run  # type: ignore
			pre_check_trace = {"obs": obs, "plan": plan, "action": action, "result": {"status": "pending"}}
			v_cfg = VerifierServiceConfig(
				model=str(args.verifier_model),
				backend=str(args.verifier_backend),
				adapter_dir=getattr(args, "verifier_adapter", None),
				adapter_bank=getattr(args, "verifier_adapter_bank", None),
				structured_mode=str(args.verifier_structured),
				timeout_sec=max(2, int(args.timeout)),
				strict=False,
				calibrate=False,
				log_prompts=bool(getattr(args, "log_prompts", False)),
			)
			vpre = verifier_run(pre_check_trace, v_cfg)
			risk_val = float(vpre.get("risk", 0.5))
			if risk_val >= float(args.approval_threshold):
				ap_dir = Path(args.approval_dir)
				ap_dir.mkdir(parents=True, exist_ok=True)
				ap_id = datetime.utcnow().strftime("%Y%m%dT%H%M%SZ")
				ap_path = ap_dir / f"approval_oscli_{ap_id}.json"
				ap_path.write_text(json.dumps({"obs": obs, "plan": plan, "action": action, "verifier": vpre, "threshold": float(args.approval_threshold)}, ensure_ascii=False, indent=2), encoding="utf-8")
				print(json.dumps({"stage": "approval_block", "risk": risk_val, "threshold": float(args.approval_threshold), "request": str(ap_path)}, ensure_ascii=False))
				# Log a blocked trace and exit without executing
				trace = build_trace(
					task_id="loop-" + args.task,
					obs=obs,
					plan=plan,
					action=action,
					result={"stdout": "", "stderr": "", "status": "blocked"},
					reward={"scalar": 0.0, "components": {"success": 0, "latency": 0, "side_effect": 1}},
					critique={"issues": [], "risk": risk_val, "proposal": "approval-required"},
				)
				try:
					import os as _os
					trace["meta_state"] = {
						"enable_tot": True if _os.environ.get("PLANNER_TOT", "0") == "1" else False,
						"temperature": float(_os.environ.get("SAMPLING_TEMPERATURE", "0.2") or 0.2),
						"step_budget": int(_os.environ.get("PLANNER_STEP_BUDGET", "1") or 1),
					}
				except Exception:
					pass
				write_jsonl(str(traces_out), trace)
				return 0
		except Exception:
			pass

	# Optionally log repair as IL example if changed
	if args.log_repair and action != pred_action:
		inp = json.dumps({"obs": obs, "plan": plan}, ensure_ascii=False)
		out = json.dumps(action, ensure_ascii=False)
		repair_path = root / "data" / "skills" / "actuator_il_repairs.jsonl"
		repair_path.parent.mkdir(parents=True, exist_ok=True)
		with open(repair_path, "a", encoding="utf-8") as f:
			f.write(json.dumps({"input": inp, "output": out}, ensure_ascii=False) + "\n")

	runner = SafeShellRunner(str(sandbox))
	argv = action.get("args", {}).get("argv") if isinstance(action, dict) else None
	if not isinstance(argv, list):
		raise RuntimeError("Predicted action missing argv")
	result_obj = runner.run(argv)
	status = "ok" if result_obj.returncode == 0 or (argv and argv[0] == "grep" and result_obj.returncode in (0, 1))
# ... truncated ...


from __future__ import annotations
import logging

import argparse
import json
from pathlib import Path
from typing import Any, Dict, List

def iter_jsonl(path: Path):
	if not path.exists():
		return
	with path.open("r", encoding="utf-8") as f:
		for line in f:
			s = line.strip()
			if not s:
				continue
			try:
				yield json.loads(s)
			except Exception:
				continue

def main() -> int:
	ap = argparse.ArgumentParser()
	root = Path(__file__).resolve().parents[2]
	ap.add_argument("--traces", default=str(root / "data" / "traces" / "dev_loop.jsonl"))
	ap.add_argument("--out", default=str(root / "data" / "dashboards" / "devloop.json"))
	args = ap.parse_args()

	p = Path(args.traces)
	total = 0
	ok = 0
	elapsed: List[float] = []
	sizes: List[int] = []
	files: List[int] = []
	added: List[int] = []
	deleted: List[int] = []
	reverts = 0

	flake_runs = 0
	covered: List[float] = []
	policy_rejects = 0
	for rec in iter_jsonl(p) or []:
		total += 1
		try:
			status = str((rec.get("result") or {}).get("status", "")).lower()
			if status == "ok":
				ok += 1
		except Exception:
			pass
		try:
			elapsed.append(float((rec.get("elapsed_sec") or 0.0)))
		except Exception:
			pass
		try:
			argsd = (rec.get("action") or {}).get("args", {})
			sizes.append(int(argsd.get("size", 0) or 0))
			files.append(int(argsd.get("files", 0) or 0))
		except Exception:
			pass
		try:
			ch = (rec.get("churn") or {})
			added.append(int(ch.get("added", 0) or 0))
			deleted.append(int(ch.get("deleted", 0) or 0))
		except Exception:
			pass
		try:
			if bool((rec.get("result") or {}).get("revert_applied")):
				reverts += 1
		except Exception:
			pass
		try:
			meta = rec.get("meta") or {}
			if isinstance(meta, dict) and bool(meta.get("flake_hint")):
				flake_runs += 1
		except Exception:
			pass
		try:
			res = rec.get("result") or {}
			if str(res.get("status", "")).lower() == "rejected":
				policy_rejects += 1
		except Exception:
			pass
		try:
			cov = (rec.get("result") or {}).get("coverage") or {}
			pct = float(cov.get("percent", 0.0)) if isinstance(cov, dict) else 0.0
			if pct > 0.0:
				covered.append(pct)
		except Exception:
			pass

	def p90(xs: List[float]) -> float:
		if not xs:
			return 0.0
		xs2 = sorted(xs)
		idx = min(len(xs2) - 1, max(0, int(round(0.9 * (len(xs2) - 1)))))
		return float(xs2[idx])

	summary: Dict[str, Any] = {
		"runs": int(total),
		"ok": int(ok),
		"success_rate": float(round(ok / max(1, total), 4)),
		"p90_elapsed_sec": float(round(p90(elapsed), 3)),
		"avg_candidate_size": float(round(sum(sizes) / max(1, len(sizes)), 3)) if sizes else 0.0,
		"avg_candidate_files": float(round(sum(files) / max(1, len(files)), 3)) if files else 0.0,
		"avg_added": float(round(sum(added) / max(1, len(added)), 3)) if added else 0.0,
		"avg_deleted": float(round(sum(deleted) / max(1, len(deleted)), 3)) if deleted else 0.0,
		"revert_rate": float(round(reverts / max(1, total), 4)),
		"flake_rate": float(round(flake_runs / max(1, total), 4)),
		"avg_coverage": float(round(sum(covered) / max(1, len(covered)), 3)) if covered else 0.0,
		"policy_reject_rate": float(round(policy_rejects / max(1, total), 4)),
	}

	outp = Path(args.out)
	outp.parent.mkdir(parents=True, exist_ok=True)
	outp.write_text(json.dumps(summary, ensure_ascii=False, indent=2), encoding="utf-8")
	print(json.dumps({"ok": True, "out": str(outp)}))
	return 0

if __name__ == "__main__":
	raise SystemExit(main())


from __future__ import annotations
import logging

import argparse
import json
from pathlib import Path
from typing import Any, Dict, List

def iter_jsonl(path: Path):
	if not path.exists():
		return
	with path.open("r", encoding="utf-8") as f:
		for line in f:
			s = line.strip()
			if not s:
				continue
			try:
				yield json.loads(s)
			except Exception:
				continue

def main() -> int:
	ap = argparse.ArgumentParser()
	root = Path(__file__).resolve().parents[2]
	ap.add_argument("--traces", default=str(root / "data" / "traces" / "dev_loop.jsonl"))
	ap.add_argument("--out", default=str(root / "data" / "dashboards" / "devloop.json"))
	args = ap.parse_args()

	p = Path(args.traces)
	total = 0
	ok = 0
	elapsed: List[float] = []
	sizes: List[int] = []
	files: List[int] = []
	added: List[int] = []
	deleted: List[int] = []
	reverts = 0

	flake_runs = 0


import argparse
import json
from pathlib import Path
from typing import Any, Dict, List

def iter_jsonl(path: Path):
	if not path.exists():
		return
	with path.open("r", encoding="utf-8") as f:
		for line in f:
			s = line.strip()
			if not s:
				continue
			try:
				yield json.loads(s)
			except Exception:
				continue

def main() -> int:
	ap = argparse.ArgumentParser()
	root = Path(__file__).resolve().parents[2]
	ap.add_argument("--traces", default=str(root / "data" / "traces" / "dev_loop.jsonl"))
	ap.add_argument("--out", default=str(root / "data" / "dashboards" / "devloop.json"))
	args = ap.parse_args()

	p = Path(args.traces)
	total = 0
	ok = 0
	elapsed: List[float] = []
	sizes: List[int] = []
	files: List[int] = []
	added: List[int] = []
	deleted: List[int] = []
	reverts = 0

	flake_runs = 0
	covered: List[float] = []
	policy_rejects = 0
	for rec in iter_jsonl(p) or []:
		total += 1
		try:
			status = str((rec.get("result") or {}).get("status", "")).lower()
			if status == "ok":
				ok += 1
		except Exception:
			pass
		try:
			elapsed.append(float((rec.get("elapsed_sec") or 0.0)))
		except Exception:
			pass
		try:
			argsd = (rec.get("action") or {}).get("args", {})
			sizes.append(int(argsd.get("size", 0) or 0))
			files.append(int(argsd.get("files", 0) or 0))
		except Exception:
			pass
		try:
			ch = (rec.get("churn") or {})
			added.append(int(ch.get("added", 0) or 0))
			deleted.append(int(ch.get("deleted", 0) or 0))
		except Exception:
			pass
		try:
			if bool((rec.get("result") or {}).get("revert_applied")):
				reverts += 1
		except Exception:
			pass
		try:
			meta = rec.get("meta") or {}
			if isinstance(meta, dict) and bool(meta.get("flake_hint")):
				flake_runs += 1
		except Exception:
			pass
		try:
			res = rec.get("result") or {}
			if str(res.get("status", "")).lower() == "rejected":
				policy_rejects += 1
		except Exception:
			pass
		try:
			cov = (rec.get("result") or {}).get("coverage") or {}
			pct = float(cov.get("percent", 0.0)) if isinstance(cov, dict) else 0.0
			if pct > 0.0:
				covered.append(pct)
		except Exception:
			pass

	def p90(xs: List[float]) -> float:
		if not xs:
			return 0.0
		xs2 = sorted(xs)
		idx = min(len(xs2) - 1, max(0, int(round(0.9 * (len(xs2) - 1)))))
		return float(xs2[idx])

	summary: Dict[str, Any] = {
		"runs": int(total),
		"ok": int(ok),
		"success_rate": float(round(ok / max(1, total), 4)),
		"p90_elapsed_sec": float(round(p90(elapsed), 3)),
		"avg_candidate_size": float(round(sum(sizes) / max(1, len(sizes)), 3)) if sizes else 0.0,
		"avg_candidate_files": float(round(sum(files) / max(1, len(files)), 3)) if files else 0.0,
		"avg_added": float(round(sum(added) / max(1, len(added)), 3)) if added else 0.0,
		"avg_deleted": float(round(sum(deleted) / max(1, len(deleted)), 3)) if deleted else 0.0,
		"revert_rate": float(round(reverts / max(1, total), 4)),
		"flake_rate": float(round(flake_runs / max(1, total), 4)),
		"avg_coverage": float(round(sum(covered) / max(1, len(covered)), 3)) if covered else 0.0,
		"policy_reject_rate": float(round(policy_rejects / max(1, total), 4)),
	}

	outp = Path(args.out)
	outp.parent.mkdir(parents=True, exist_ok=True)
	outp.write_text(json.dumps(summary, ensure_ascii=False, indent=2), encoding="utf-8")
	print(json.dumps({"ok": True, "out": str(outp)}))
	return 0

if __name__ == "__main__":
	raise SystemExit(main())


		try:
			meta = rec.get("meta") or {}
			if isinstance(meta, dict) and bool(meta.get("flake_hint")):
				flake_runs += 1
		except Exception:
			pass
		try:
			res = rec.get("result") or {}
			if str(res.get("status", "")).lower() == "rejected":
				policy_rejects += 1
		except Exception:
			pass
		try:
			cov = (rec.get("result") or {}).get("coverage") or {}
			pct = float(cov.get("percent", 0.0)) if isinstance(cov, dict) else 0.0
			if pct > 0.0:
				covered.append(pct)
		except Exception:
			pass

	def p90(xs: List[float]) -> float:
		if not xs:
			return 0.0
		xs2 = sorted(xs)
		idx = min(len(xs2) - 1, max(0, int(round(0.9 * (len(xs2) - 1)))))
		return float(xs2[idx])

	summary: Dict[str, Any] = {
		"runs": int(total),
		"ok": int(ok),
		"success_rate": float(round(ok / max(1, total), 4)),
		"p90_elapsed_sec": float(round(p90(elapsed), 3)),
		"avg_candidate_size": float(round(sum(sizes) / max(1, len(sizes)), 3)) if sizes else 0.0,
		"avg_candidate_files": float(round(sum(files) / max(1, len(files)), 3)) if files else 0.0,
		"avg_added": float(round(sum(added) / max(1, len(added)), 3)) if added else 0.0,
		"avg_deleted": float(round(sum(deleted) / max(1, len(deleted)), 3)) if deleted else 0.0,
		"revert_rate": float(round(reverts / max(1, total), 4)),
		"flake_rate": float(round(flake_runs / max(1, total), 4)),
		"avg_coverage": float(round(sum(covered) / max(1, len(covered)), 3)) if covered else 0.0,
		"policy_reject_rate": float(round(policy_rejects / max(1, total), 4)),
	}

	outp = Path(args.out)
	outp.parent.mkdir(parents=True, exist_ok=True)
	outp.write_text(json.dumps(summary, ensure_ascii=False, indent=2), encoding="utf-8")
	print(json.dumps({"ok": True, "out": str(outp)}))


import logging
import argparse
import json
from pathlib import Path
from typing import Any, Dict, List

def _safe_float(x: Any, default: float = 0.0) -> float:
	try:
		return float(x)
	except Exception:
		return float(default)

def _p90(values: List[float]) -> float:
	if not values:
		return 0.0
	vals = sorted(values)
	idx = int(min(len(vals) - 1, max(0, round(0.9 * (len(vals) - 1)))))
	return float(vals[idx])

def _iter_jsonl(path: Path):
	if not path.exists():
		return
	with path.open("r", encoding="utf-8") as f:
		for line in f:
			s = line.strip()
			if not s:
				continue
			try:
				yield json.loads(s)
			except Exception:
				continue

def main() -> int:
	ap = argparse.ArgumentParser()
	root = Path(__file__).resolve().parents[1]
	ap.add_argument("--ci-batch", default=str(root / "data" / "ci" / "devloop_batch.json"))
	ap.add_argument("--traces", default=str(root / "data" / "traces" / "dev_loop.ci.jsonl"))
	ap.add_argument("--out", default=str(root / "data" / "dashboards" / "devloop.json"))
	args = ap.parse_args()

	# CI summary (SR / p90)
	sr = 0.0
	p90 = 0.0
	n = 0
	try:
		cb = Path(args.ci_batch)
		if cb.exists():
			pack = json.loads(cb.read_text(encoding="utf-8"))
			summ = pack.get("summary", {}) if isinstance(pack, dict) else {}
			sr = float(summ.get("sr", 0.0))
			p90 = float(summ.get("p90_sec", 0.0))
			n = int(summ.get("n", len(pack.get("runs", []) if isinstance(pack, dict) else [])))
	except Exception:
		sr, p90, n = (0.0, 0.0, 0)

	# Trace-derived KPIs
	attempts = 0
	reverts = 0
	sizes: List[float] = []
	files: List[float] = []
	try:
		tp = Path(args.traces)
		for rec in _iter_jsonl(tp):
			if not isinstance(rec, dict):
				continue
			# Count attempts based on presence of a code.patch.apply action
			try:
				action = rec.get("action", {}) if isinstance(rec.get("action"), dict) else {}
				if str(action.get("tool", "")) == "code.patch.apply":
					attempts += 1
					aargs = action.get("args", {}) if isinstance(action.get("args"), dict) else {}
					sizes.append(_safe_float(aargs.get("size"), 0.0))
					files.append(_safe_float(aargs.get("files"), 0.0))
			except Exception:
				pass
			# Count reverts (result.revert_applied)
			try:
				res = rec.get("result", {}) if isinstance(rec.get("result"), dict) else {}
				if bool(res.get("revert_applied")):
					reverts += 1
			except Exception:
				pass
	except Exception:
		attempts, reverts = (attempts, reverts)

	avg_size = float(round((sum(sizes) / max(1, len(sizes))) if sizes else 0.0, 3))
	avg_files = float(round((sum(files) / max(1, len(files))) if files else 0.0, 3))
	revert_rate = float(round((float(reverts) / max(1.0, float(attempts))), 3)) if attempts else 0.0

	outp = Path(args.out)
	outp.parent.mkdir(parents=True, exist_ok=True)
	payload: Dict[str, Any] = {
		"summary": {
			"n": int(n),
			"success_rate": float(round(sr, 3)),
			"p90_elapsed_sec": float(round(p90, 3)),
		},
		"kpis": {
			"attempts": int(attempts),
			"reverts": int(reverts),
			"revert_rate": revert_rate,
			"avg_candidate_size": avg_size,
			"avg_candidate_files": avg_files,
		},
	}
	outp.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
	print(json.dumps({"ok": True, "out": str(outp)}))
	return 0

if __name__ == "__main__":
	raise SystemExit(main())


import logging
import argparse
import json
from pathlib import Path
from typing import Any, Dict, List

def _safe_float(x: Any, default: float = 0.0) -> float:
	try:
		return float(x)
	except Exception:
		return float(default)

def _p90(values: List[float]) -> float:
	if not values:
		return 0.0
	vals = sorted(values)
	idx = int(min(len(vals) - 1, max(0, round(0.9 * (len(vals) - 1)))))
	return float(vals[idx])

def _iter_jsonl(path: Path):
	if not path.exists():
		return
	with path.open("r", encoding="utf-8") as f:
		for line in f:
			s = line.strip()
			if not s:
				continue
			try:
				yield json.loads(s)


import logging
import argparse
import json
from pathlib import Path
from typing import Any, Dict, List

def _safe_float(x: Any, default: float = 0.0) -> float:
	try:
		return float(x)
	except Exception:
		return float(default)

def _p90(values: List[float]) -> float:
	if not values:
		return 0.0
	vals = sorted(values)
	idx = int(min(len(vals) - 1, max(0, round(0.9 * (len(vals) - 1)))))
	return float(vals[idx])

def _iter_jsonl(path: Path):
	if not path.exists():
		return
	with path.open("r", encoding="utf-8") as f:
		for line in f:
			s = line.strip()
			if not s:
				continue
			try:
				yield json.loads(s)
			except Exception:
				continue

def main() -> int:
	ap = argparse.ArgumentParser()
	root = Path(__file__).resolve().parents[1]
	ap.add_argument("--ci-batch", default=str(root / "data" / "ci" / "devloop_batch.json"))


import json
from pathlib import Path
from typing import Any, Dict, List

def _safe_float(x: Any, default: float = 0.0) -> float:
	try:
		return float(x)
	except Exception:
		return float(default)

def _p90(values: List[float]) -> float:
	if not values:
		return 0.0
	vals = sorted(values)
	idx = int(min(len(vals) - 1, max(0, round(0.9 * (len(vals) - 1)))))
	return float(vals[idx])

def _iter_jsonl(path: Path):
	if not path.exists():
		return
	with path.open("r", encoding="utf-8") as f:
		for line in f:
			s = line.strip()
			if not s:
				continue
			try:
				yield json.loads(s)
			except Exception:
				continue

def main() -> int:
	ap = argparse.ArgumentParser()
	root = Path(__file__).resolve().parents[1]
	ap.add_argument("--ci-batch", default=str(root / "data" / "ci" / "devloop_batch.json"))
	ap.add_argument("--traces", default=str(root / "data" / "traces" / "dev_loop.ci.jsonl"))
	ap.add_argument("--out", default=str(root / "data" / "dashboards" / "devloop.json"))
	args = ap.parse_args()

	# CI summary (SR / p90)
	sr = 0.0
	p90 = 0.0
	n = 0
	try:
		cb = Path(args.ci_batch)
		if cb.exists():
			pack = json.loads(cb.read_text(encoding="utf-8"))
			summ = pack.get("summary", {}) if isinstance(pack, dict) else {}
			sr = float(summ.get("sr", 0.0))


		return 0.0
	vals = sorted(values)
	idx = int(min(len(vals) - 1, max(0, round(0.9 * (len(vals) - 1)))))
	return float(vals[idx])

def _iter_jsonl(path: Path):
	if not path.exists():
		return
	with path.open("r", encoding="utf-8") as f:
		for line in f:
			s = line.strip()
			if not s:
				continue
			try:
				yield json.loads(s)
			except Exception:
				continue

def main() -> int:
	ap = argparse.ArgumentParser()
	root = Path(__file__).resolve().parents[1]
	ap.add_argument("--ci-batch", default=str(root / "data" / "ci" / "devloop_batch.json"))
	ap.add_argument("--traces", default=str(root / "data" / "traces" / "dev_loop.ci.jsonl"))
	ap.add_argument("--out", default=str(root / "data" / "dashboards" / "devloop.json"))
	args = ap.parse_args()

	# CI summary (SR / p90)
	sr = 0.0
	p90 = 0.0
	n = 0
	try:
		cb = Path(args.ci_batch)
		if cb.exists():
			pack = json.loads(cb.read_text(encoding="utf-8"))
			summ = pack.get("summary", {}) if isinstance(pack, dict) else {}
			sr = float(summ.get("sr", 0.0))
			p90 = float(summ.get("p90_sec", 0.0))
			n = int(summ.get("n", len(pack.get("runs", []) if isinstance(pack, dict) else [])))
	except Exception:
		sr, p90, n = (0.0, 0.0, 0)

	# Trace-derived KPIs
	attempts = 0
	reverts = 0
	sizes: List[float] = []
	files: List[float] = []
	try:
		tp = Path(args.traces)
		for rec in _iter_jsonl(tp):
			if not isinstance(rec, dict):
				continue
			# Count attempts based on presence of a code.patch.apply action
			try:
				action = rec.get("action", {}) if isinstance(rec.get("action"), dict) else {}
				if str(action.get("tool", "")) == "code.patch.apply":
					attempts += 1
					aargs = action.get("args", {}) if isinstance(action.get("args"), dict) else {}
					sizes.append(_safe_float(aargs.get("size"), 0.0))
					files.append(_safe_float(aargs.get("files"), 0.0))
			except Exception:
				pass
			# Count reverts (result.revert_applied)
			try:
				res = rec.get("result", {}) if isinstance(rec.get("result"), dict) else {}
				if bool(res.get("revert_applied")):
					reverts += 1
			except Exception:
				pass
	except Exception:
		attempts, reverts = (attempts, reverts)

	avg_size = float(round((sum(sizes) / max(1, len(sizes))) if sizes else 0.0, 3))
	avg_files = float(round((sum(files) / max(1, len(files))) if files else 0.0, 3))
	revert_rate = float(round((float(reverts) / max(1.0, float(attempts))), 3)) if attempts else 0.0

	outp = Path(args.out)
	outp.parent.mkdir(parents=True, exist_ok=True)
	payload: Dict[str, Any] = {
		"summary": {
			"n": int(n),
			"success_rate": float(round(sr, 3)),
			"p90_elapsed_sec": float(round(p90, 3)),
		},
		"kpis": {
			"attempts": int(attempts),
			"reverts": int(reverts),
			"revert_rate": revert_rate,
			"avg_candidate_size": avg_size,
			"avg_candidate_files": avg_files,
		},
	}
	outp.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
	print(json.dumps({"ok": True, "out": str(outp)}))
	return 0

if __name__ == "__main__":
	raise SystemExit(main())


from __future__ import annotations
import logging

import argparse
import json
from pathlib import Path

def _read_json(path: Path) -> dict:
	try:
		if path.exists():
			return json.loads(path.read_text(encoding="utf-8"))
	except Exception:
		pass
	return {}

def main() -> int:
	ap = argparse.ArgumentParser()
	root = Path(__file__).resolve().parents[2]
	ap.add_argument("--devloop", default=str(root / "data" / "dashboards" / "devloop.json"))
	ap.add_argument("--rl", default=str(root / "models" / "coder_rl" / "metrics.json"))
	ap.add_argument("--nearmiss", default=str(root / "models" / "coder_nearmiss" / "model.json"))
	ap.add_argument("--verifier", default=str(root / "data" / "dashboards" / "verifier.json"))
	ap.add_argument("--out-md", default=str(root / "data" / "dashboards" / "summary.md"))
	ap.add_argument("--out-html", default=str(root / "data" / "dashboards" / "summary.html"))
	args = ap.parse_args()

	devloop = _read_json(Path(args.devloop))
	rl = _read_json(Path(args.rl))
	nearmiss = _read_json(Path(args.nearmiss))
	verifier = _read_json(Path(args.verifier))

	sr = float((devloop.get("summary") or devloop).get("success_rate", 0.0))
	p90 = float((devloop.get("summary") or devloop).get("p90_elapsed_sec", 0.0))
	avg_size = float((devloop.get("summary") or devloop).get("avg_candidate_size", 0.0))
	avg_files = float((devloop.get("summary") or devloop).get("avg_candidate_files", 0.0))

	rl_reward = float(rl.get("avg_reward", rl.get("reward", 0.0))) if isinstance(rl, dict) else 0.0
	rl_sr = float(rl.get("success_rate", 0.0)) if isinstance(rl, dict) else 0.0
	rl_files = float(rl.get("avg_files", 0.0)) if isinstance(rl, dict) else 0.0
	rl_loc = float(rl.get("avg_loc", 0.0)) if isinstance(rl, dict) else 0.0

	nm_kind = str(nearmiss.get("kind", "")) if isinstance(nearmiss, dict) else ""
	nm_items = int(nearmiss.get("items", 0)) if isinstance(nearmiss, dict) else 0

	verifier_ece = float(verifier.get("ece", 0.0)) if isinstance(verifier, dict) else 0.0
	verifier_threshold = float(verifier.get("threshold", 0.0)) if isinstance(verifier, dict) else 0.0
	verifier_accuracy = float(verifier.get("avg_accuracy", 0.0)) if isinstance(verifier, dict) else 0.0

	md = []
	md.append("### Dev-loop Summary")
	md.append(f"- **SR**: {sr:.3f}")
	md.append(f"- **p90 elapsed (s)**: {p90:.3f}")
	md.append(f"- **avg candidate size**: {avg_size:.1f}")
	md.append(f"- **avg candidate files**: {avg_files:.2f}")
	md.append("")
	md.append("### RL Metrics")
	md.append(f"- **avg reward**: {rl_reward:.3f}")
	md.append(f"- **RL SR**: {rl_sr:.3f}")
	md.append(f"- **avg files**: {rl_files:.2f}")
	md.append(f"- **avg LOC proxy**: {rl_loc:.1f}")
	md.append("")
	md.append("### Near-miss Replay")
	md.append(f"- **artifact**: {nm_kind or 'n/a'}")
	md.append(f"- **items**: {nm_items}")
	md.append("")
	md.append("### Verifier Calibration")
	md.append(f"- **ECE**: {verifier_ece:.4f}")
	md.append(f"- **threshold**: {verifier_threshold:.4f}")
	md.append(f"- **accuracy**: {verifier_accuracy:.4f}")
	md_text = "\n".join(md) + "\n"

	out_md = Path(args.out_md)
	out_md.parent.mkdir(parents=True, exist_ok=True)
	out_md.write_text(md_text, encoding="utf-8")

	html = [
		"<meta charset=\"utf-8\"><title>Compact Summary</title>",
		"",
		"",
		"
Dev-loop Summary
",
		f"

SR: {sr:.3f}

p90 elapsed (s): {p90:.3f}

avg candidate size: {avg_size:.1f}

avg candidate files: {avg_files:.2f}

",
		"
RL Metrics
",
		f"

avg reward: {rl_reward:.3f}

RL SR: {rl_sr:.3f}

avg files: {rl_files:.2f}

avg LOC proxy: {rl_loc:.1f}

",
		"
Near-miss Replay
",
		f"

artifact: {nm_kind or 'n/a'}

items: {nm_items}

",
		"
Verifier Calibration
",
		f"

ECE: {verifier_ece:.4f}

threshold: {verifier_threshold:.4f}

accuracy: {verifier_accuracy:.4f}

",
		"",
	]
	out_html = Path(args.out_html)
	out_html.parent.mkdir(parents=True, exist_ok=True)
	out_html.write_text("".join(html), encoding="utf-8")

	print(json.dumps({"ok": True, "out_md": str(out_md), "out_html": str(out_html)}))
	return 0

if __name__ == "__main__":
	raise SystemExit(main())


from __future__ import annotations
import logging

import argparse
import json
from pathlib import Path

def _read_json(path: Path) -> dict:
	try:
		if path.exists():
			return json.loads(path.read_text(encoding="utf-8"))
	except Exception:
		pass
	return {}

def main() -> int:
	ap = argparse.ArgumentParser()
	root = Path(__file__).resolve().parents[2]
	ap.add_argument("--devloop", default=str(root / "data" / "dashboards" / "devloop.json"))
	ap.add_argument("--rl", default=str(root / "models" / "coder_rl" / "metrics.json"))
	ap.add_argument("--nearmiss", default=str(root / "models" / "coder_nearmiss" / "model.json"))
	ap.add_argument("--verifier", default=str(root / "data" / "dashboards" / "verifier.json"))
	ap.add_argument("--out-md", default=str(root / "data" / "dashboards" / "summary.md"))
	ap.add_argument("--out-html", default=str(root / "data" / "dashboards" / "summary.html"))
	args = ap.parse_args()

	devloop = _read_json(Path(args.devloop))
	rl = _read_json(Path(args.rl))
	nearmiss = _read_json(Path(args.nearmiss))
	verifier = _read_json(Path(args.verifier))

	sr = float((devloop.get("summary") or devloop).get("success_rate", 0.0))
	p90 = float((devloop.get("summary") or devloop).get("p90_elapsed_sec", 0.0))


import logging
import argparse
import json
from pathlib import Path

def main() -> int:
	ap = argparse.ArgumentParser()
	root = Path(__file__).resolve().parents[1]
	ap.add_argument("--traces", default=str(root / "data" / "traces" / "loop_run.jsonl"))
	ap.add_argument("--out", default=str(root / "data" / "dashboards" / "trajectory_summary.json"))
	args = ap.parse_args()

	trp = Path(args.traces)
	if not trp.exists():
		print(json.dumps({"ok": False, "error": "traces_missing", "path": str(trp)}))
		return 1

	n_total = 0
	n_success = 0
	steps_total = 0
	steps_success = 0
	domains = {}
	with trp.open("r", encoding="utf-8") as f:
		for line in f:
			line = line.strip()
			if not line:
				continue
			try:
				rec = json.loads(line)
			except Exception:
				continue
			n_total += 1
			success = bool((rec.get("result") or {}).get("status", "").lower() == "ok") if isinstance(rec, dict) else False
			steps = int((rec.get("metrics") or {}).get("steps", (rec.get("steps") or 0))) if isinstance(rec, dict) else 0
			steps_total += steps
			steps_success += (steps if success else 0)
			domain = str(rec.get("domain", ""))
			if domain:
				dd = domains.get(domain, {"n": 0, "ok": 0, "steps": 0})
				dd["n"] += 1
				dd["ok"] += 1 if success else 0
				dd["steps"] += steps
				domains[domain] = dd

	summary = {
		"total": int(n_total),
		"ok": int(n_success),
		"success_rate": (float(n_success) / max(1.0, float(n_total))),
		"avg_steps": (float(steps_total) / max(1.0, float(n_total))),
		"avg_steps_success": (float(steps_success) / max(1.0, float(n_success))) if n_success > 0 else None,
		"by_domain": {k: {"n": v["n"], "ok": v["ok"], "success_rate": (float(v["ok"]) / max(1.0, float(v["n"]))), "avg_steps": (float(v["steps"]) / max(1.0, float(v["n"]))) } for k, v in domains.items()},
	}
	outp = Path(args.out)
	outp.parent.mkdir(parents=True, exist_ok=True)
	outp.write_text(json.dumps(summary, ensure_ascii=False, indent=2), encoding="utf-8")
	print(json.dumps({"ok": True, "out": str(outp)}))
	return 0

if __name__ == "__main__":
	import sys
	sys.exit(main())


#!/usr/bin/env python3
import logging
import argparse
import json
from pathlib import Path

def parse_args():
	root = Path(__file__).resolve().parents[1]
	ap = argparse.ArgumentParser(description="Aggregate LLM benchmark results into dashboard section")
	ap.add_argument("--results", default=str(root / "data" / "llm_bench" / "results.json"))
	ap.add_argument("--dashboard", default=str(root / "data" / "dashboards" / "summary.json"))
	ap.add_argument("--section", default="llm_bench")
	ap.add_argument("--lmeval", default=str(root / "data" / "llm_bench" / "lmeval.json"))
	return ap.parse_args()

def main() -> int:
	args = parse_args()
	results_path = Path(args.results)
	dash_path = Path(args.dashboard)
	dash_path.parent.mkdir(parents=True, exist_ok=True)

	summary = {}
	if dash_path.exists():
		try:
			summary = json.loads(dash_path.read_text(encoding="utf-8"))
		except Exception:
			summary = {}

	section = {}
	if results_path.exists():
		try:
			section = json.loads(results_path.read_text(encoding="utf-8"))
		except Exception:
			section = {"error": "results_parse_error"}
	else:
		section = {"error": "results_missing"}

	# Optionally attach lm-eval-harness results
	lmeval_path = Path(args.lmeval)
	if lmeval_path.exists():
		try:
			lmeval = json.loads(lmeval_path.read_text(encoding="utf-8"))
			section["lmeval"] = lmeval
		except Exception:
			section["lmeval"] = {"error": "lmeval_parse_error"}

	summary[args.section] = section
	dash_path.write_text(json.dumps(summary, ensure_ascii=False, indent=2), encoding="utf-8")
	print(json.dumps({"ok": True, "section": args.section, "dashboard": str(dash_path)}))
	return 0

if __name__ == "__main__":
	raise SystemExit(main())


#!/usr/bin/env python3
import logging
import argparse
import json
from pathlib import Path

def parse_args():
	root = Path(__file__).resolve().parents[1]
	ap = argparse.ArgumentParser(description="Aggregate LLM benchmark results into dashboard section")
	ap.add_argument("--results", default=str(root / "data" / "llm_bench" / "results.json"))
	ap.add_argument("--dashboard", default=str(root / "data" / "dashboards" / "summary.json"))
	ap.add_argument("--section", default="llm_bench")
	ap.add_argument("--lmeval", default=str(root / "data" / "llm_bench" / "lmeval.json"))
	return ap.parse_args()

def main() -> int:
	args = parse_args()
	results_path = Path(args.results)
	dash_path = Path(args.dashboard)
	dash_path.parent.mkdir(parents=True, exist_ok=True)

	summary = {}
	if dash_path.exists():
		try:
			summary = json.loads(dash_path.read_text(encoding="utf-8"))
		except Exception:
			summary = {}

	section = {}
	if results_path.exists():
		try:
			section = json.loads(results_path.read_text(encoding="utf-8"))
		except Exception:


import logging
import argparse
import json
from pathlib import Path

def main() -> int:
	ap = argparse.ArgumentParser()
	ap.add_argument("--verified", required=True)
	ap.add_argument("--out", default=None)
	args = ap.parse_args()

	path = Path(args.verified)
	if not path.exists():
		print("verified file not found:", str(path))
		return 2
	total = 0
	ok = 0
	latencies = []
	risks = []
	with path.open("r", encoding="utf-8") as f:
		for line in f:
			line = line.strip()
			if not line:
				continue
			try:
				obj = json.loads(line)
			except Exception:
				continue
			total += 1
			try:
				status = str(obj.get("result", {}).get("status", "")).lower()
				if status == "ok":
					ok += 1
			except Exception:
				pass
			try:
				lat = obj.get("result", {}).get("latency_ms")
				if lat is not None:
					latencies.append(float(lat))
			except Exception:
				pass
			try:
				risks.append(float(obj.get("critique", {}).get("risk", 0.5)))
			except Exception:
				pass
	r = {
		"total": total,
		"ok": ok,
		"success_rate": (ok / total) if total else 0.0,
		"avg_latency_ms": (sum(latencies) / len(latencies)) if latencies else None,
		"avg_risk": (sum(risks) / len(risks)) if risks else None,
	}
	js = json.dumps(r, ensure_ascii=False)
	print(js)
	if args.out:
		Path(args.out).write_text(js, encoding="utf-8")
	return 0

if __name__ == "__main__":
	import sys
	sys.exit(main())


#!/usr/bin/env python3
import logging
import argparse
import json
from pathlib import Path

def parse_args():
	root = Path(__file__).resolve().parents[1]
	ap = argparse.ArgumentParser(description="Aggregate coding benchmark results into dashboard section")
	ap.add_argument("--results", default=str(root / "data" / "llm_bench" / "code_results.json"))
	ap.add_argument("--dashboard", default=str(root / "data" / "dashboards" / "summary.json"))
	ap.add_argument("--section", default="code_bench")
	ap.add_argument("--style", default=str(root / "data" / "llm_bench" / "code_style.json"))
	return ap.parse_args()

def main() -> int:
	args = parse_args()
	dash_path = Path(args.dashboard)
	dash_path.parent.mkdir(parents=True, exist_ok=True)

	summary = {}
	if dash_path.exists():
		try:
			summary = json.loads(dash_path.read_text(encoding="utf-8"))
		except Exception:
			summary = {}

	section = {"error": "results_missing"}
	res_path = Path(args.results)
	if res_path.exists():
		try:
			section = json.loads(res_path.read_text(encoding="utf-8"))
			# Aggregate APPS categories and DS1000 domains if present
			try:
				bm = section.get("benchmarks", {})
				agg: dict[str, any] = {}
				apps = bm.get("apps") or {}
				if isinstance(apps, dict):
					cats = apps.get("categories_observed") or {}
					if isinstance(cats, dict):
						agg["apps_categories"] = cats
				ds = bm.get("ds1000") or {}
				if isinstance(ds, dict):
					doms = ds.get("domains_observed") or {}
					if isinstance(doms, dict):
						agg["ds1000_domains"] = doms
				if agg:
					section.setdefault("aggregates", {}).update(agg)
			except Exception:
				pass
		except Exception:
			section = {"error": "results_parse_error"}

	# Optionally attach style/type check summary
	style_path = Path(args.style)
	if style_path.exists():
		try:
			style = json.loads(style_path.read_text(encoding="utf-8"))
			section["style"] = style
		except Exception:
			section["style"] = {"error": "style_parse_error"}

	# Attach contamination summary if present
	try:
		cont = Path(str(Path(args.results).parent / "contamination.json"))
		if cont.exists():
			section["contamination"] = json.loads(cont.read_text(encoding="utf-8"))
	except Exception:
		pass

	summary[args.section] = section
	dash_path.write_text(json.dumps(summary, ensure_ascii=False, indent=2), encoding="utf-8")
	print(json.dumps({"ok": True, "section": args.section, "dashboard": str(dash_path)}))
	return 0

if __name__ == "__main__":
	raise SystemExit(main())


#!/usr/bin/env python3
import logging
import argparse
import json
from pathlib import Path

def parse_args():
	root = Path(__file__).resolve().parents[1]
	ap = argparse.ArgumentParser(description="Aggregate coding benchmark results into dashboard section")
	ap.add_argument("--results", default=str(root / "data" / "llm_bench" / "code_results.json"))
	ap.add_argument("--dashboard", default=str(root / "data" / "dashboards" / "summary.json"))
	ap.add_argument("--section", default="code_bench")
	ap.add_argument("--style", default=str(root / "data" / "llm_bench" / "code_style.json"))
	return ap.parse_args()

def main() -> int:
	args = parse_args()
	dash_path = Path(args.dashboard)
	dash_path.parent.mkdir(parents=True, exist_ok=True)

	summary = {}
	if dash_path.exists():
		try:
			summary = json.loads(dash_path.read_text(encoding="utf-8"))
		except Exception:
			summary = {}

	section = {"error": "results_missing"}
	res_path = Path(args.results)
	if res_path.exists():
		try:
			section = json.loads(res_path.read_text(encoding="utf-8"))
			# Aggregate APPS categories and DS1000 domains if present


import logging
import argparse
import json
from collections import defaultdict
from pathlib import Path

def main() -> int:
	ap = argparse.ArgumentParser()
	root = Path(__file__).resolve().parents[1]
	ap.add_argument("--verified", default=str(root / "data" / "traces" / "web_dom.verified.jsonl"))
	ap.add_argument("--out", default=str(root / "data" / "traces" / "web_dom.summary.json"))
	args = ap.parse_args()

	path = Path(args.verified)
	outp = Path(args.out)
	if not path.exists():
		print(json.dumps({"ok": False, "error": "verified_missing", "path": str(path)}))
		return 1

	total = 0
	ok = 0
	by_url = defaultdict(lambda: {"n": 0, "ok": 0})
	issues = defaultdict(int)
	net_blocked = 0
	with path.open("r", encoding="utf-8") as f:
		for line in f:
			line = line.strip()
			if not line:
				continue
			try:
				obj = json.loads(line)
			except Exception:
				continue
			total += 1
			res = obj.get("result", {}) if isinstance(obj, dict) else {}
			status = str(res.get("status", "")).lower()
			ok += 1 if status == "ok" else 0
			meta = (obj.get("obs", {}) or {}).get("meta", {}) if isinstance(obj, dict) else {}
			url = str(meta.get("url", ""))
			if url:
				by_url[url]["n"] += 1
				by_url[url]["ok"] += 1 if status == "ok" else 0
			# Critique issues summary if present
			crit = obj.get("critique", {}) if isinstance(obj, dict) else {}
			iss = crit.get("issues") if isinstance(crit, dict) else None
			if isinstance(iss, list):
				for it in iss:
					try:
						issues[str(it)] += 1
					except Exception:
						pass
			if int(obj.get("net_blocked", 0) or 0) > 0:
				net_blocked += int(obj.get("net_blocked", 0) or 0)

	by_url_summary = {u: {"n": v["n"], "ok": v["ok"], "success_rate": float(round((float(v["ok"]) / max(1.0, float(v["n"]))), 3))} for u, v in by_url.items()}
	summary = {
		"total": int(total),
		"ok": int(ok),
		"success_rate": float(round((float(ok) / max(1.0, float(total))), 3)),
		"by_url": by_url_summary,
		"issues": dict(sorted(issues.items(), key=lambda kv: kv[1], reverse=True)),
		"net_blocked": int(net_blocked),
	}
	outp.parent.mkdir(parents=True, exist_ok=True)
	outp.write_text(json.dumps(summary, ensure_ascii=False, indent=2), encoding="utf-8")
	print(json.dumps({"ok": True, "out": str(outp), "total": int(total), "success_rate": summary["success_rate"]}))
	return 0

if __name__ == "__main__":
	import sys
	sys.exit(main())

import argparse
import json
from collections import Counter, defaultdict
from pathlib import Path
from typing import Any, Dict, List, Tuple

def host_from_url(url: str) -> str:
	try:
		from urllib.parse import urlparse  # type: ignore
		u = urlparse(url)
		return u.netloc or ""
	except Exception:
		return ""

def load_jsonl(path: Path) -> List[Dict[str, Any]]:
	rows: List[Dict[str, Any]] = []
	if not path.exists():
		return rows
	with path.open("r", encoding="utf-8") as f:
		for line in f:
			line = line.strip()
			if not line:
				continue
			try:
				rows.append(json.loads(line))
			except Exception:
				continue
	return rows

def categorize_error(critique: str) -> str:
	text = (critique or "").lower()
	if any(k in text for k in ["not found", "no element", "missing selector", "empty result"]):
		return "missing_selector"
	if any(k in text for k in ["ambiguous", "multiple elements", "too many matches"]):
		return "ambiguous_selector"
	if any(k in text for k in ["dynamic", "timing", "loaded later", "async"]):
		return "dynamic_content"
	return "other"

def summarize(paths: List[Path]) -> Dict[str, Any]:
	stats: Dict[str, Any] = {
		"total": 0,
		"ok": 0,
		"error": 0,
		"by_host": {},
		"error_types": {},
		"top_selectors": [],
		"examples": {"errors": []},
	}
	by_host_counts: Dict[str, Dict[str, int]] = defaultdict(lambda: {"total": 0, "ok": 0})
	error_types = Counter()
	selector_counts = Counter()
	examples: List[Dict[str, Any]] = []

	for p in paths:
		for tr in load_jsonl(p):
			stats["total"] += 1
			obs = tr.get("obs", {}) if isinstance(tr, dict) else {}
			meta = obs.get("meta", {}) if isinstance(obs, dict) else {}
			url = str(meta.get("url", ""))
			sel = str(meta.get("selector", ""))
			selector_counts[sel] += 1 if sel else 0
			h = host_from_url(url) if url else ""
			if h:
				by_host_counts[h]["total"] += 1
			result = tr.get("result", {}) if isinstance(tr, dict) else {}
			status = str(result.get("status", "")).lower()
			ok = (status == "ok")
			if ok:
				stats["ok"] += 1
				by_host_counts[h]["ok"] += 1 if h else 0
			else:
				stats["error"] += 1
				crit = tr.get("critique", {}) if isinstance(tr, dict) else {}
				crit_txt = str(crit.get("issues", "")) if isinstance(crit, dict) else ""
				etype = categorize_error(crit_txt)
				error_types[etype] += 1
				if len(examples) < 10:
					examples.append({"url": url, "selector": sel, "error_type": etype, "critique": crit_txt[:200]})

	# finalize
	stats["by_host"] = {k: {"total": v["total"], "ok": v["ok"], "success_rate": (v["ok"] / max(1, v["total"]))} for k, v in by_host_counts.items()}
	stats["error_types"] = dict(error_types)
	stats["top_selectors"] = [s for s, _ in selector_counts.most_common(10)]
	stats["examples"]["errors"] = examples
	stats["success_rate"] = (stats["ok"] / max(1, stats["total"]))
	return stats

def main() -> int:
	ap = argparse.ArgumentParser()
	root = Path(__file__).resolve().parents[1]
	ap.add_argument("--verified", nargs="+", default=[str(root / "data" / "traces" / "web_dom.verified.jsonl")])
	ap.add_argument("--out", default=str(root / "data" / "traces" / "web_dom_summary.json"))
	args = ap.parse_args()

	paths = [Path(p) for p in args.verified]
	out = summarize(paths)
	out_path = Path(args.out)
	out_path.parent.mkdir(parents=True, exist_ok=True)
	out_path.write_text(json.dumps(out, ensure_ascii=False, indent=2), encoding="utf-8")
	print(json.dumps({"ok": True, "out": str(out_path), "success_rate": out.get("success_rate", 0.0)}))
	return 0

if __name__ == "__main__":
	raise SystemExit(main())

import argparse
import json
from pathlib import Path

def main() -> int:
	ap = argparse.ArgumentParser()
	root = Path(__file__).resolve().parents[1]
	ap.add_argument("--verified", default=str(root / "data" / "traces" / "web_dom.verified.jsonl"))
	args = ap.parse_args()

	path = Path(args.verified)
	if not path.exists():
		print(json.dumps({"total": 0, "ok": 0, "avg_risk": 0.0, "note": "missing"}))
		return 0

	total = 0
	ok = 0
	risks = []
	for line in path.read_text(encoding="utf-8").splitlines():
		line = line.strip()
		if not line:
			continue
		try:
			obj = json.loads(line)
		except Exception:
			continue
		total += 1
		status = str(obj.get("result", {}).get("status", "")).lower()
		if status == "ok":
			ok += 1
		try:
			risks.append(float(obj.get("critique", {}).get("risk", 0.5)))
		except Exception:
			pass
	avg_risk = sum(risks) / len(risks) if risks else 0.0
	print(json.dumps({"total": total, "ok": ok, "avg_risk": avg_risk}))
	return 0

if __name__ == "__main__":
	raise SystemExit(main())

import argparse
import json
from pathlib import Path
from typing import Dict

def main() -> int:
	ap = argparse.ArgumentParser()
	root = Path(__file__).resolve().parents[1]
	ap.add_argument("--verified", default=str(root / "data" / "traces" / "web_dom.verified.jsonl"))
	args = ap.parse_args()

	path = Path(args.verified)
	if not path.exists():
		print(json.dumps({"n": 0, "success_rate": 0.0}))
		return 0

	n = 0
	success = 0
	with path.open("r", encoding="utf-8") as f:
		for line in f:
			line = line.strip()
			if not line:
				continue
			try:
				tr: Dict = json.loads(line)
				status = (tr.get("result") or {}).get("status")
				n += 1
				if status == "ok":
					success += 1
			except Exception:
				continue
	rate = (success / n) if n else 0.0
	print(json.dumps({"n": n, "success": success, "success_rate": rate}))
	return 0

if __name__ == "__main__":
	raise SystemExit(main())


		if status == "ok":
			ok += 1
		try:
			risks.append(float(obj.get("critique", {}).get("risk", 0.5)))
		except Exception:
			pass
	avg_risk = sum(risks) / len(risks) if risks else 0.0
	print(json.dumps({"total": total, "ok": ok, "avg_risk": avg_risk}))
	return 0

if __name__ == "__main__":
	raise SystemExit(main())

import argparse
import json
from pathlib import Path
from typing import Dict

def main() -> int:
	ap = argparse.ArgumentParser()
	root = Path(__file__).resolve().parents[1]
	ap.add_argument("--verified", default=str(root / "data" / "traces" / "web_dom.verified.jsonl"))
	args = ap.parse_args()

	path = Path(args.verified)
	if not path.exists():
		print(json.dumps({"n": 0, "success_rate": 0.0}))
		return 0

	n = 0
	success = 0
	with path.open("r", encoding="utf-8") as f:
		for line in f:
			line = line.strip()
			if not line:
				continue
			try:
				tr: Dict = json.loads(line)
				status = (tr.get("result") or {}).get("status")
				n += 1
				if status == "ok":
					success += 1
			except Exception:
				continue
	rate = (success / n) if n else 0.0
	print(json.dumps({"n": n, "success": success, "success_rate": rate}))
	return 0

if __name__ == "__main__":
	raise SystemExit(main())


		"issues": dict(sorted(issues.items(), key=lambda kv: kv[1], reverse=True)),
		"net_blocked": int(net_blocked),
	}
	outp.parent.mkdir(parents=True, exist_ok=True)
	outp.write_text(json.dumps(summary, ensure_ascii=False, indent=2), encoding="utf-8")
	print(json.dumps({"ok": True, "out": str(outp), "total": int(total), "success_rate": summary["success_rate"]}))
	return 0

if __name__ == "__main__":
	import sys
	sys.exit(main())

import argparse
import json
from collections import Counter, defaultdict
from pathlib import Path
from typing import Any, Dict, List, Tuple

def host_from_url(url: str) -> str:
	try:
		from urllib.parse import urlparse  # type: ignore
		u = urlparse(url)
		return u.netloc or ""
	except Exception:
		return ""

def load_jsonl(path: Path) -> List[Dict[str, Any]]:
	rows: List[Dict[str, Any]] = []
	if not path.exists():
		return rows
	with path.open("r", encoding="utf-8") as f:
		for line in f:
			line = line.strip()
			if not line:
				continue
			try:
				rows.append(json.loads(line))
			except Exception:
				continue
	return rows

def categorize_error(critique: str) -> str:
	text = (critique or "").lower()


if __name__ == "__main__":
	import sys
	sys.exit(main())

import argparse
import json
from collections import Counter, defaultdict
from pathlib import Path
from typing import Any, Dict, List, Tuple

def host_from_url(url: str) -> str:
	try:
		from urllib.parse import urlparse  # type: ignore
		u = urlparse(url)
		return u.netloc or ""
	except Exception:
		return ""

def load_jsonl(path: Path) -> List[Dict[str, Any]]:
	rows: List[Dict[str, Any]] = []
	if not path.exists():
		return rows
	with path.open("r", encoding="utf-8") as f:
		for line in f:
			line = line.strip()
			if not line:
				continue
			try:
				rows.append(json.loads(line))
			except Exception:
				continue
	return rows

def categorize_error(critique: str) -> str:
	text = (critique or "").lower()
	if any(k in text for k in ["not found", "no element", "missing selector", "empty result"]):
		return "missing_selector"
	if any(k in text for k in ["ambiguous", "multiple elements", "too many matches"]):
		return "ambiguous_selector"
	if any(k in text for k in ["dynamic", "timing", "loaded later", "async"]):
		return "dynamic_content"
	return "other"

def summarize(paths: List[Path]) -> Dict[str, Any]:
	stats: Dict[str, Any] = {
		"total": 0,
		"ok": 0,
		"error": 0,
		"by_host": {},
		"error_types": {},


	except Exception:
		return ""

def load_jsonl(path: Path) -> List[Dict[str, Any]]:
	rows: List[Dict[str, Any]] = []
	if not path.exists():
		return rows
	with path.open("r", encoding="utf-8") as f:
		for line in f:
			line = line.strip()
			if not line:
				continue
			try:
				rows.append(json.loads(line))
			except Exception:
				continue
	return rows

def categorize_error(critique: str) -> str:
	text = (critique or "").lower()
	if any(k in text for k in ["not found", "no element", "missing selector", "empty result"]):
		return "missing_selector"
	if any(k in text for k in ["ambiguous", "multiple elements", "too many matches"]):
		return "ambiguous_selector"
	if any(k in text for k in ["dynamic", "timing", "loaded later", "async"]):
		return "dynamic_content"
	return "other"

def summarize(paths: List[Path]) -> Dict[str, Any]:
	stats: Dict[str, Any] = {
		"total": 0,
		"ok": 0,
		"error": 0,
		"by_host": {},
		"error_types": {},
		"top_selectors": [],
		"examples": {"errors": []},
	}
	by_host_counts: Dict[str, Dict[str, int]] = defaultdict(lambda: {"total": 0, "ok": 0})
	error_types = Counter()
	selector_counts = Counter()
	examples: List[Dict[str, Any]] = []

	for p in paths:
		for tr in load_jsonl(p):
			stats["total"] += 1


summarize

			if not line:
				continue
			try:
				rows.append(json.loads(line))
			except Exception:
				continue
	return rows

def categorize_error(critique: str) -> str:
	text = (critique or "").lower()
	if any(k in text for k in ["not found", "no element", "missing selector", "empty result"]):
		return "missing_selector"
	if any(k in text for k in ["ambiguous", "multiple elements", "too many matches"]):
		return "ambiguous_selector"
	if any(k in text for k in ["dynamic", "timing", "loaded later", "async"]):
		return "dynamic_content"
	return "other"

def summarize(paths: List[Path]) -> Dict[str, Any]:
	stats: Dict[str, Any] = {
		"total": 0,
		"ok": 0,
		"error": 0,
		"by_host": {},
		"error_types": {},
		"top_selectors": [],
		"examples": {"errors": []},
	}
	by_host_counts: Dict[str, Dict[str, int]] = defaultdict(lambda: {"total": 0, "ok": 0})
	error_types = Counter()
	selector_counts = Counter()
	examples: List[Dict[str, Any]] = []

	for p in paths:
		for tr in load_jsonl(p):
			stats["total"] += 1
			obs = tr.get("obs", {}) if isinstance(tr, dict) else {}
			meta = obs.get("meta", {}) if isinstance(obs, dict) else {}
			url = str(meta.get("url", ""))
			sel = str(meta.get("selector", ""))
			selector_counts[sel] += 1 if sel else 0
			h = host_from_url(url) if url else ""
			if h:
				by_host_counts[h]["total"] += 1
			result = tr.get("result", {}) if isinstance(tr, dict) else {}
			status = str(result.get("status", "")).lower()
			ok = (status == "ok")
			if ok:
				stats["ok"] += 1
				by_host_counts[h]["ok"] += 1 if h else 0
			else:
				stats["error"] += 1
				crit = tr.get("critique", {}) if isinstance(tr, dict) else {}
				crit_txt = str(crit.get("issues", "")) if isinstance(crit, dict) else ""
				etype = categorize_error(crit_txt)
				error_types[etype] += 1
				if len(examples) < 10:
					examples.append({"url": url, "selector": sel, "error_type": etype, "critique": crit_txt[:200]})

	# finalize
	stats["by_host"] = {k: {"total": v["total"], "ok": v["ok"], "success_rate": (v["ok"] / max(1, v["total"]))} for k, v in by_host_counts.items()}
	stats["error_types"] = dict(error_types)
	stats["top_selectors"] = [s for s, _ in selector_counts.most_common(10)]
	stats["examples"]["errors"] = examples
	stats["success_rate"] = (stats["ok"] / max(1, stats["total"]))
	return stats

def main() -> int:
	ap = argparse.ArgumentParser()
	root = Path(__file__).resolve().parents[1]
	ap.add_argument("--verified", nargs="+", default=[str(root / "data" / "traces" / "web_dom.verified.jsonl")])
	ap.add_argument("--out", default=str(root / "data" / "traces" / "web_dom_summary.json"))
	args = ap.parse_args()

	paths = [Path(p) for p in args.verified]
	out = summarize(paths)
	out_path = Path(args.out)
	out_path.parent.mkdir(parents=True, exist_ok=True)
	out_path.write_text(json.dumps(out, ensure_ascii=False, indent=2), encoding="utf-8")
	print(json.dumps({"ok": True, "out": str(out_path), "success_rate": out.get("success_rate", 0.0)}))
	return 0

if __name__ == "__main__":
	raise SystemExit(main())


from __future__ import annotations
import logging

import argparse
import json
from pathlib import Path
from typing import Any, Dict, List

def iter_jsonl(path: Path):
	if not path.exists():
		return
	with path.open("r", encoding="utf-8") as f:
		for line in f:
			s = line.strip()
			if not s:
				continue
			try:
				yield json.loads(s)
			except Exception:
				continue

def compute_ece(predictions: List[float], labels: List[int], num_bins: int = 10) -> float:
	"""Compute Expected Calibration Error."""
	if not predictions or not labels:
		return 0.0

	# Create bins and initialize counters
	bins = [[] for _ in range(num_bins)]
	for pred, label in zip(predictions, labels):
		bin_idx = min(num_bins - 1, int(pred * num_bins))
		bins[bin_idx].append((pred, label))

	# Compute ECE
	ece = 0.0
	total_samples = len(predictions)

	for bin_items in bins:
		if not bin_items:
			continue

		bin_size = len(bin_items)
		bin_confidence = sum(pred for pred, _ in bin_items) / bin_size
		bin_accuracy = sum(label for _, label in bin_items) / bin_size

		ece += (bin_size / total_samples) * abs(bin_confidence - bin_accuracy)

	return float(ece)

def main() -> int:
	ap = argparse.ArgumentParser()
	root = Path(__file__).resolve().parents[2]
	ap.add_argument("--verifier-logs", default=str(root / "data" / "logs" / "verifier.jsonl"))
	ap.add_argument("--out", default=str(root / "data" / "dashboards" / "verifier.json"))
	args = ap.parse_args()

	predictions: List[float] = []
	labels: List[int] = []
	thresholds: List[float] = []

	for rec in iter_jsonl(Path(args.verifier_logs)) or []:
		try:
			pred = float(rec.get("confidence", 0.0))
			label = int(rec.get("correct", 0))
			threshold = float(rec.get("threshold", 0.0))

			predictions.append(pred)
			labels.append(label)
			thresholds.append(threshold)
		except Exception:
			continue

	summary: Dict[str, Any] = {
		"ece": float(round(compute_ece(predictions, labels), 4)),
		"avg_confidence": float(round(sum(predictions) / max(1, len(predictions)), 4)),
		"avg_accuracy": float(round(sum(labels) / max(1, len(labels)), 4)),
		"threshold": float(round(sum(thresholds) / max(1, len(thresholds)), 4)) if thresholds else 0.0,
		"samples": len(predictions)
	}

	outp = Path(args.out)
	outp.parent.mkdir(parents=True, exist_ok=True)
	outp.write_text(json.dumps(summary, ensure_ascii=False, indent=2), encoding="utf-8")
	print(json.dumps({"ok": True, "out": str(outp)}))
	return 0

if __name__ == "__main__":
	raise SystemExit(main())


from __future__ import annotations
import logging

import argparse
import json
from pathlib import Path
from typing import Any, Dict, List

def iter_jsonl(path: Path):
	if not path.exists():
		return
	with path.open("r", encoding="utf-8") as f:
		for line in f:
			s = line.strip()
			if not s:
				continue
			try:
				yield json.loads(s)
			except Exception:
				continue

def compute_ece(predictions: List[float], labels: List[int], num_bins: int = 10) -> float:
	"""Compute Expected Calibration Error."""
	if not predictions or not labels:
		return 0.0

	# Create bins and initialize counters
	bins = [[] for _ in range(num_bins)]
	for pred, label in zip(predictions, labels):
		bin_idx = min(num_bins - 1, int(pred * num_bins))
		bins[bin_idx].append((pred, label))

	# Compute ECE
	ece = 0.0
	total_samples = len(predictions)

	for bin_items in bins:
		if not bin_items:
			continue


import argparse
import json
from pathlib import Path
from typing import Any, Dict, List

def iter_jsonl(path: Path):
	if not path.exists():
		return
	with path.open("r", encoding="utf-8") as f:
		for line in f:
			s = line.strip()
			if not s:
				continue
			try:
				yield json.loads(s)
			except Exception:
				continue

def compute_ece(predictions: List[float], labels: List[int], num_bins: int = 10) -> float:
	"""Compute Expected Calibration Error."""
	if not predictions or not labels:
		return 0.0

	# Create bins and initialize counters
	bins = [[] for _ in range(num_bins)]
	for pred, label in zip(predictions, labels):
		bin_idx = min(num_bins - 1, int(pred * num_bins))
		bins[bin_idx].append((pred, label))

	# Compute ECE
	ece = 0.0
	total_samples = len(predictions)

	for bin_items in bins:
		if not bin_items:
			continue

		bin_size = len(bin_items)
		bin_confidence = sum(pred for pred, _ in bin_items) / bin_size
		bin_accuracy = sum(label for _, label in bin_items) / bin_size

		ece += (bin_size / total_samples) * abs(bin_confidence - bin_accuracy)

	return float(ece)

def main() -> int:
	ap = argparse.ArgumentParser()
	root = Path(__file__).resolve().parents[2]
	ap.add_argument("--verifier-logs", default=str(root / "data" / "logs" / "verifier.jsonl"))
	ap.add_argument("--out", default=str(root / "data" / "dashboards" / "verifier.json"))
	args = ap.parse_args()

	predictions: List[float] = []
	labels: List[int] = []
	thresholds: List[float] = []

	for rec in iter_jsonl(Path(args.verifier_logs)) or []:
		try:
			pred = float(rec.get("confidence", 0.0))
			label = int(rec.get("correct", 0))
			threshold = float(rec.get("threshold", 0.0))

			predictions.append(pred)


main

		bin_idx = min(num_bins - 1, int(pred * num_bins))
		bins[bin_idx].append((pred, label))

	# Compute ECE
	ece = 0.0
	total_samples = len(predictions)

	for bin_items in bins:
		if not bin_items:
			continue

		bin_size = len(bin_items)
		bin_confidence = sum(pred for pred, _ in bin_items) / bin_size
		bin_accuracy = sum(label for _, label in bin_items) / bin_size

		ece += (bin_size / total_samples) * abs(bin_confidence - bin_accuracy)

	return float(ece)

def main() -> int:
	ap = argparse.ArgumentParser()
	root = Path(__file__).resolve().parents[2]
	ap.add_argument("--verifier-logs", default=str(root / "data" / "logs" / "verifier.jsonl"))
	ap.add_argument("--out", default=str(root / "data" / "dashboards" / "verifier.json"))
	args = ap.parse_args()

	predictions: List[float] = []
	labels: List[int] = []
	thresholds: List[float] = []

	for rec in iter_jsonl(Path(args.verifier_logs)) or []:
		try:
			pred = float(rec.get("confidence", 0.0))
			label = int(rec.get("correct", 0))
			threshold = float(rec.get("threshold", 0.0))

			predictions.append(pred)
			labels.append(label)
			thresholds.append(threshold)
		except Exception:
			continue

	summary: Dict[str, Any] = {
		"ece": float(round(compute_ece(predictions, labels), 4)),
		"avg_confidence": float(round(sum(predictions) / max(1, len(predictions)), 4)),
		"avg_accuracy": float(round(sum(labels) / max(1, len(labels)), 4)),
		"threshold": float(round(sum(thresholds) / max(1, len(thresholds)), 4)) if thresholds else 0.0,
		"samples": len(predictions)
	}

	outp = Path(args.out)
	outp.parent.mkdir(parents=True, exist_ok=True)
	outp.write_text(json.dumps(summary, ensure_ascii=False, indent=2), encoding="utf-8")
	print(json.dumps({"ok": True, "out": str(outp)}))
	return 0

if __name__ == "__main__":
	raise SystemExit(main())