import logging
import argparse
import json
import os
from pathlib import Path
from datetime import datetime
from typing import Optional

from agi_dw.tools.git import GitTool
from agi_dw.tools.test_runner import TestRunner

def _print(msg: str) -> None:
	print(msg, flush=True)

def emit_plan(model: str, out_path: Path) -> bool:
	from agi_dw.scripts.misc.emit_refactor_plan import main as emit_main  # type: ignore
	# emit_refactor_plan writes to args.out and prints the path
	try:
		# Build argv-like context
		import sys  # type: ignore
		argv0 = list(sys.argv)
		sys.argv = ["emit_refactor_plan.py", "--backend", "hf", "--model", model, "--out", str(out_path)]
		emit_main()
		return out_path.exists()
	finally:
		sys.argv = argv0  # type: ignore

def validate_plan(plan_path: Path) -> bool:
	from agi_dw.scripts.data.validate_refactor_plan import main as validate_main  # type: ignore
	try:
		import sys  # type: ignore
		argv0 = list(sys.argv)
		sys.argv = ["validate_refactor_plan.py", str(plan_path)]
		ret = validate_main()
		return ret == 0
	finally:
		sys.argv = argv0  # type: ignore

def apply_plan(repo: Path, plan_path: Path, dry_run: bool, max_edits: int, allow: list[str], block: list[str]) -> list[str]:
	from agi_dw.scripts.misc.apply_refactor_plan import main as apply_main  # type: ignore
	logs: list[str] = []
	try:
		import sys  # type: ignore
		argv0 = list(sys.argv)
		args = ["apply_refactor_plan.py", "--plan", str(plan_path), "--repo", str(repo), "--validate", "--max-edits", str(int(max_edits))]
		if dry_run:
			args.append("--dry-run")
		# Forward allow/block patterns
		for pat in (allow or []):
			args.extend(["--allow", pat])
		for pat in (block or []):
			args.extend(["--block", pat])
		sys.argv = args
		# capture print output by temporarily redirecting stdout
		import io  # type: ignore
		import contextlib  # type: ignore
		buf = io.StringIO()
		with contextlib.redirect_stdout(buf):
			apply_main()
		logs = [line for line in buf.getvalue().splitlines() if line.strip()]
	finally:
		sys.argv = argv0  # type: ignore
	return logs

def run_tests(repo: Path, pytest_args: Optional[list[str]] = None, timeout: int = 900, nodes: Optional[list[str]] = None) -> dict:
	tr = TestRunner(str(repo))
	env = dict(**os.environ)
	env["PYTHONPATH"] = str(repo)
	env["PYTEST_DISABLE_PLUGIN_AUTOLOAD"] = "1"
	env["PYTHONDONTWRITEBYTECODE"] = "1"
	args = (pytest_args or [])
	# Prefer tests directory if present
	test_dir = None
	for name in ("tests", "testing", "test"):
		p = repo / name
		if p.exists() and p.is_dir():
			test_dir = str(p)
			break
	if nodes:
		# Run targeted nodes explicitly
		target_args = args + nodes
		res = tr.run_pytest(args=target_args, timeout=timeout, env=env)
		return res
	# No specific nodes: run default scope
	if test_dir:
		args = args + [test_dir]
	else:
		args = args + [str(repo)]
	return tr.run_pytest(args=args, timeout=timeout, env=env)

def main() -> int:
	ap = argparse.ArgumentParser()
	root = Path(__file__).resolve().parents[1]
	ap.add_argument("--repo", default=str(root))
	ap.add_argument("--model", default="meta-llama/Llama-3.2-3B")
	ap.add_argument("--plan", default=str(root / "data" / "traces" / "refactor_plan.json"))
	ap.add_argument("--dry-run", action="store_true")
	ap.add_argument("--skip-emit", action="store_true", help="Skip emit step and use existing plan file")
	ap.add_argument("--commit", action="store_true", help="Commit applied edits if tests pass")
	ap.add_argument("--pytest-args", nargs='*', default=[])
	ap.add_argument("--max-edits", type=int, default=200)
	ap.add_argument("--allow", nargs='*', default=[])
	ap.add_argument("--block", nargs='*', default=[])
	args = ap.parse_args()

	repo = Path(args.repo).resolve()
	plan_path = Path(args.plan).resolve()
	plan_path.parent.mkdir(parents=True, exist_ok=True)
	git = GitTool(str(repo))
	# Load plan if exists (for tests/summary)
	plan_data = {}
	try:
		if plan_path.exists():
			plan_data = json.loads(plan_path.read_text(encoding="utf-8"))
	except Exception:
		plan_data = {}

	if not bool(args.skip_emit):
		_print(json.dumps({"stage": "emit"}))
		ok_emit = emit_plan(args.model, plan_path)
		if not ok_emit:
			_print(json.dumps({"stage": "emit", "ok": False}))
			return 2

	_print(json.dumps({"stage": "validate"}))
	ok_val = validate_plan(plan_path)
	if not ok_val:
		_print(json.dumps({"stage": "validate", "ok": False}))
		return 2

	_print(json.dumps({"stage": "apply", "dry_run": bool(args.dry_run)}))
	apply_logs = apply_plan(repo, plan_path, args.dry_run, args.max_edits, list(args.allow or []), list(args.block or []))
	_print(json.dumps({"stage": "apply", "logs": apply_logs}))

	# If dry run, stop before tests
	if args.dry_run:
		return 0

	# If plan specifies targeted tests, run them first
	target_nodes: Optional[list[str]] = None
	try:
		cand = plan_data.get("tests") if isinstance(plan_data, dict) else None
		if isinstance(cand, list) and cand:
			target_nodes = [str(x) for x in cand if isinstance(x, str)]
	except Exception:
		target_nodes = None
	_print(json.dumps({"stage": "pytest"}))
	res = run_tests(repo, pytest_args=args.pytest_args, nodes=target_nodes)
	_print(json.dumps({"stage": "pytest", **res}))
	if not res.get("ok") and res.get("returncode") != 5:
		# restore working tree to avoid leaving partial edits
		try:
			git.reset_hard()
			git.clean(directories=True, force=True)
		except Exception:
			pass
		return 2
	# If targeted tests passed, run default scope to be safe
	if target_nodes:
		res_full = run_tests(repo, pytest_args=args.pytest_args)
		_print(json.dumps({"stage": "pytest_full", **res_full}))
		if not res_full.get("ok") and res_full.get("returncode") != 5:
			try:
				git.reset_hard()
				git.clean(directories=True, force=True)
			except Exception:
				pass
			return 2

	if args.commit:
		_print(json.dumps({"stage": "commit"}))
		try:
			git.add(".")
			# Include plan summary in commit message if present
			summary = None
			try:
				summary = plan_data.get("summary") if isinstance(plan_data, dict) else None
			except Exception:
				summary = None
			ts = datetime.utcnow().strftime('%Y-%m-%d %H:%M:%SZ')
			msg = f"Apply refactor plan: {summary or 'auto'} ({ts})"
			cm = git.commit(msg)
			_print(json.dumps({"stage": "commit", "returncode": cm.returncode}))
		except Exception:
			pass
	return 0

if __name__ == "__main__":
	raise SystemExit(main())


import logging
import argparse
import json
import os
from pathlib import Path
from datetime import datetime
from typing import Optional

from agi_dw.tools.git import GitTool
from agi_dw.tools.test_runner import TestRunner

def _print(msg: str) -> None:
	print(msg, flush=True)

def emit_plan(model: str, out_path: Path) -> bool:
	from agi_dw.scripts.misc.emit_refactor_plan import main as emit_main  # type: ignore
	# emit_refactor_plan writes to args.out and prints the path
	try:
		# Build argv-like context
		import sys  # type: ignore
		argv0 = list(sys.argv)
		sys.argv = ["emit_refactor_plan.py", "--backend", "hf", "--model", model, "--out", str(out_path)]
		emit_main()
		return out_path.exists()
	finally:
		sys.argv = argv0  # type: ignore

def validate_plan(plan_path: Path) -> bool:
	from agi_dw.scripts.data.validate_refactor_plan import main as validate_main  # type: ignore
	try:
		import sys  # type: ignore


import logging
import argparse
import json
import os
from pathlib import Path
from datetime import datetime
from typing import Optional

from agi_dw.tools.git import GitTool
from agi_dw.tools.test_runner import TestRunner

def _print(msg: str) -> None:
	print(msg, flush=True)

def emit_plan(model: str, out_path: Path) -> bool:
	from agi_dw.scripts.misc.emit_refactor_plan import main as emit_main  # type: ignore
	# emit_refactor_plan writes to args.out and prints the path
	try:
		# Build argv-like context
		import sys  # type: ignore
		argv0 = list(sys.argv)
		sys.argv = ["emit_refactor_plan.py", "--backend", "hf", "--model", model, "--out", str(out_path)]
		emit_main()
		return out_path.exists()
	finally:
		sys.argv = argv0  # type: ignore

def validate_plan(plan_path: Path) -> bool:
	from agi_dw.scripts.data.validate_refactor_plan import main as validate_main  # type: ignore
	try:
		import sys  # type: ignore
		argv0 = list(sys.argv)
		sys.argv = ["validate_refactor_plan.py", str(plan_path)]
		ret = validate_main()
		return ret == 0
	finally:
		sys.argv = argv0  # type: ignore

def apply_plan(repo: Path, plan_path: Path, dry_run: bool, max_edits: int, allow: list[str], block: list[str]) -> list[str]:
	from agi_dw.scripts.misc.apply_refactor_plan import main as apply_main  # type: ignore
	logs: list[str] = []
	try:
		import sys  # type: ignore
		argv0 = list(sys.argv)


def _print(msg: str) -> None:
	print(msg, flush=True)

def emit_plan(model: str, out_path: Path) -> bool:
	from agi_dw.scripts.misc.emit_refactor_plan import main as emit_main  # type: ignore
	# emit_refactor_plan writes to args.out and prints the path
	try:
		# Build argv-like context
		import sys  # type: ignore
		argv0 = list(sys.argv)
		sys.argv = ["emit_refactor_plan.py", "--backend", "hf", "--model", model, "--out", str(out_path)]
		emit_main()
		return out_path.exists()
	finally:
		sys.argv = argv0  # type: ignore

def validate_plan(plan_path: Path) -> bool:
	from agi_dw.scripts.data.validate_refactor_plan import main as validate_main  # type: ignore
	try:
		import sys  # type: ignore
		argv0 = list(sys.argv)
		sys.argv = ["validate_refactor_plan.py", str(plan_path)]
		ret = validate_main()
		return ret == 0
	finally:
		sys.argv = argv0  # type: ignore

def apply_plan(repo: Path, plan_path: Path, dry_run: bool, max_edits: int, allow: list[str], block: list[str]) -> list[str]:
	from agi_dw.scripts.misc.apply_refactor_plan import main as apply_main  # type: ignore
	logs: list[str] = []
	try:
		import sys  # type: ignore
		argv0 = list(sys.argv)
		args = ["apply_refactor_plan.py", "--plan", str(plan_path), "--repo", str(repo), "--validate", "--max-edits", str(int(max_edits))]
		if dry_run:
			args.append("--dry-run")
		# Forward allow/block patterns
		for pat in (allow or []):
			args.extend(["--allow", pat])
		for pat in (block or []):
			args.extend(["--block", pat])
		sys.argv = args
		# capture print output by temporarily redirecting stdout
		import io  # type: ignore
		import contextlib  # type: ignore


		argv0 = list(sys.argv)
		sys.argv = ["emit_refactor_plan.py", "--backend", "hf", "--model", model, "--out", str(out_path)]
		emit_main()
		return out_path.exists()
	finally:
		sys.argv = argv0  # type: ignore

def validate_plan(plan_path: Path) -> bool:
	from agi_dw.scripts.data.validate_refactor_plan import main as validate_main  # type: ignore
	try:
		import sys  # type: ignore
		argv0 = list(sys.argv)
		sys.argv = ["validate_refactor_plan.py", str(plan_path)]
		ret = validate_main()
		return ret == 0
	finally:
		sys.argv = argv0  # type: ignore

def apply_plan(repo: Path, plan_path: Path, dry_run: bool, max_edits: int, allow: list[str], block: list[str]) -> list[str]:
	from agi_dw.scripts.misc.apply_refactor_plan import main as apply_main  # type: ignore
	logs: list[str] = []
	try:
		import sys  # type: ignore
		argv0 = list(sys.argv)
		args = ["apply_refactor_plan.py", "--plan", str(plan_path), "--repo", str(repo), "--validate", "--max-edits", str(int(max_edits))]
		if dry_run:
			args.append("--dry-run")
		# Forward allow/block patterns
		for pat in (allow or []):
			args.extend(["--allow", pat])
		for pat in (block or []):
			args.extend(["--block", pat])
		sys.argv = args
		# capture print output by temporarily redirecting stdout
		import io  # type: ignore
		import contextlib  # type: ignore
		buf = io.StringIO()
		with contextlib.redirect_stdout(buf):
			apply_main()
		logs = [line for line in buf.getvalue().splitlines() if line.strip()]
	finally:
		sys.argv = argv0  # type: ignore
	return logs

def run_tests(repo: Path, pytest_args: Optional[list[str]] = None, timeout: int = 900, nodes: Optional[list[str]] = None) -> dict:
	tr = TestRunner(str(repo))
	env = dict(**os.environ)
	env["PYTHONPATH"] = str(repo)
	env["PYTEST_DISABLE_PLUGIN_AUTOLOAD"] = "1"
	env["PYTHONDONTWRITEBYTECODE"] = "1"
	args = (pytest_args or [])
	# Prefer tests directory if present
	test_dir = None
	for name in ("tests", "testing", "test"):
		p = repo / name
		if p.exists() and p.is_dir():
			test_dir = str(p)
			break
	if nodes:
		# Run targeted nodes explicitly
		target_args = args + nodes
		res = tr.run_pytest(args=target_args, timeout=timeout, env=env)


		if dry_run:
			args.append("--dry-run")
		# Forward allow/block patterns
		for pat in (allow or []):
			args.extend(["--allow", pat])
		for pat in (block or []):
			args.extend(["--block", pat])
		sys.argv = args
		# capture print output by temporarily redirecting stdout
		import io  # type: ignore
		import contextlib  # type: ignore
		buf = io.StringIO()
		with contextlib.redirect_stdout(buf):
			apply_main()
		logs = [line for line in buf.getvalue().splitlines() if line.strip()]
	finally:
		sys.argv = argv0  # type: ignore
	return logs

def run_tests(repo: Path, pytest_args: Optional[list[str]] = None, timeout: int = 900, nodes: Optional[list[str]] = None) -> dict:
	tr = TestRunner(str(repo))
	env = dict(**os.environ)
	env["PYTHONPATH"] = str(repo)
	env["PYTEST_DISABLE_PLUGIN_AUTOLOAD"] = "1"
	env["PYTHONDONTWRITEBYTECODE"] = "1"
	args = (pytest_args or [])
	# Prefer tests directory if present
	test_dir = None
	for name in ("tests", "testing", "test"):
		p = repo / name
		if p.exists() and p.is_dir():
			test_dir = str(p)
			break
	if nodes:
		# Run targeted nodes explicitly
		target_args = args + nodes
		res = tr.run_pytest(args=target_args, timeout=timeout, env=env)
		return res
	# No specific nodes: run default scope
	if test_dir:
		args = args + [test_dir]
	else:
		args = args + [str(repo)]
	return tr.run_pytest(args=args, timeout=timeout, env=env)

def main() -> int:
	ap = argparse.ArgumentParser()
	root = Path(__file__).resolve().parents[1]
	ap.add_argument("--repo", default=str(root))
	ap.add_argument("--model", default="meta-llama/Llama-3.2-3B")
	ap.add_argument("--plan", default=str(root / "data" / "traces" / "refactor_plan.json"))
	ap.add_argument("--dry-run", action="store_true")
	ap.add_argument("--skip-emit", action="store_true", help="Skip emit step and use existing plan file")
	ap.add_argument("--commit", action="store_true", help="Commit applied edits if tests pass")
	ap.add_argument("--pytest-args", nargs='*', default=[])
	ap.add_argument("--max-edits", type=int, default=200)
	ap.add_argument("--allow", nargs='*', default=[])
	ap.add_argument("--block", nargs='*', default=[])
	args = ap.parse_args()

	repo = Path(args.repo).resolve()
	plan_path = Path(args.plan).resolve()
	plan_path.parent.mkdir(parents=True, exist_ok=True)


main

	# Prefer tests directory if present
	test_dir = None
	for name in ("tests", "testing", "test"):
		p = repo / name
		if p.exists() and p.is_dir():
			test_dir = str(p)
			break
	if nodes:
		# Run targeted nodes explicitly
		target_args = args + nodes
		res = tr.run_pytest(args=target_args, timeout=timeout, env=env)
		return res
	# No specific nodes: run default scope
	if test_dir:
		args = args + [test_dir]
	else:
		args = args + [str(repo)]
	return tr.run_pytest(args=args, timeout=timeout, env=env)

def main() -> int:
	ap = argparse.ArgumentParser()
	root = Path(__file__).resolve().parents[1]
	ap.add_argument("--repo", default=str(root))
	ap.add_argument("--model", default="meta-llama/Llama-3.2-3B")
	ap.add_argument("--plan", default=str(root / "data" / "traces" / "refactor_plan.json"))
	ap.add_argument("--dry-run", action="store_true")
	ap.add_argument("--skip-emit", action="store_true", help="Skip emit step and use existing plan file")
	ap.add_argument("--commit", action="store_true", help="Commit applied edits if tests pass")
	ap.add_argument("--pytest-args", nargs='*', default=[])
	ap.add_argument("--max-edits", type=int, default=200)
	ap.add_argument("--allow", nargs='*', default=[])
	ap.add_argument("--block", nargs='*', default=[])
	args = ap.parse_args()

	repo = Path(args.repo).resolve()
	plan_path = Path(args.plan).resolve()
	plan_path.parent.mkdir(parents=True, exist_ok=True)
	git = GitTool(str(repo))
	# Load plan if exists (for tests/summary)
	plan_data = {}
	try:
		if plan_path.exists():
			plan_data = json.loads(plan_path.read_text(encoding="utf-8"))
	except Exception:
		plan_data = {}

	if not bool(args.skip_emit):
		_print(json.dumps({"stage": "emit"}))
		ok_emit = emit_plan(args.model, plan_path)
		if not ok_emit:
			_print(json.dumps({"stage": "emit", "ok": False}))
			return 2

	_print(json.dumps({"stage": "validate"}))
	ok_val = validate_plan(plan_path)
	if not ok_val:
		_print(json.dumps({"stage": "validate", "ok": False}))
		return 2

	_print(json.dumps({"stage": "apply", "dry_run": bool(args.dry_run)}))
	apply_logs = apply_plan(repo, plan_path, args.dry_run, args.max_edits, list(args.allow or []), list(args.block or []))
	_print(json.dumps({"stage": "apply", "logs": apply_logs}))

	# If dry run, stop before tests
	if args.dry_run:
		return 0

	# If plan specifies targeted tests, run them first
	target_nodes: Optional[list[str]] = None
	try:
		cand = plan_data.get("tests") if isinstance(plan_data, dict) else None
		if isinstance(cand, list) and cand:
			target_nodes = [str(x) for x in cand if isinstance(x, str)]
	except Exception:
		target_nodes = None
	_print(json.dumps({"stage": "pytest"}))
	res = run_tests(repo, pytest_args=args.pytest_args, nodes=target_nodes)
	_print(json.dumps({"stage": "pytest", **res}))
	if not res.get("ok") and res.get("returncode") != 5:
		# restore working tree to avoid leaving partial edits
		try:
			git.reset_hard()
			git.clean(directories=True, force=True)
		except Exception:
			pass
		return 2
	# If targeted tests passed, run default scope to be safe
	if target_nodes:
		res_full = run_tests(repo, pytest_args=args.pytest_args)
		_print(json.dumps({"stage": "pytest_full", **res_full}))
		if not res_full.get("ok") and res_full.get("returncode") != 5:
			try:
				git.reset_hard()
				git.clean(directories=True, force=True)
			except Exception:
				pass
			return 2

	if args.commit:
		_print(json.dumps({"stage": "commit"}))
		try:
			git.add(".")
			# Include plan summary in commit message if present
			summary = None
			try:
				summary = plan_data.get("summary") if isinstance(plan_data, dict) else None
			except Exception:
				summary = None
			ts = datetime.utcnow().strftime('%Y-%m-%d %H:%M:%SZ')
			msg = f"Apply refactor plan: {summary or 'auto'} ({ts})"
			cm = git.commit(msg)
			_print(json.dumps({"stage": "commit", "returncode": cm.returncode}))
		except Exception:
			pass
	return 0

if __name__ == "__main__":
	raise SystemExit(main())


#!/usr/bin/env python3
import logging
"""
Convert software development tasks from inspiration folder into AGI training format.
This script extracts tasks from AgentLab, WorkArena, OSWorld, and other benchmarks
and converts them into our training trace format.
"""

import json
import os
import random
from pathlib import Path
from typing import Dict, List, Any

def load_osworld_tasks(examples_dir: str) -> List[Dict]:
	"""Load OSWorld tasks from evaluation examples."""
	tasks = []
	examples_path = Path(examples_dir)

	for app_dir in examples_path.iterdir():
		if not app_dir.is_dir():
			continue

		app_name = app_dir.name
		print(f"Processing {app_name} tasks...")

		for task_file in app_dir.glob("*.json"):
			try:
				with open(task_file, 'r') as f:
					task_data = json.load(f)

				# Convert OSWorld task to our format
				task = {
					"task_id": task_data.get("id", str(task_file.stem)),
					"obs": {
						"kind": "dev",
						"content": task_data.get("instruction", ""),
						"meta": {
							"app": app_name,
							"source": task_data.get("source", ""),
							"related_apps": task_data.get("related_apps", [])
						}
					},
					"plan": {
						"subgoals": [task_data.get("instruction", "")],
						"tools": ["dev"],
						"constraints": {}
					},
					"action": {
						"tool": "dev.execute",
						"args": {
							"instruction": task_data.get("instruction", ""),
							"app": app_name,
							"config": task_data.get("config", [])
						}
					},
					"result": {
						"stdout": f"Task completed in {app_name}",
						"stderr": "",
						"status": "ok"
					},
					"reward": {
						"scalar": 1.0,
						"components": {
							"success": 1,
							"latency": 0,
							"side_effect": 1
						}
					},
					"critique": {
						"issues": [],
						"risk": 0.1,
						"proposal": ""
					}
				}
				tasks.append(task)

			except Exception as e:
				print(f"Error processing {task_file}: {e}")
				continue

	return tasks

def load_agentlab_tasks(agentlab_dir: str) -> List[Dict]:
	"""Load AgentLab tasks and convert to our format."""
	tasks = []

	# AgentLab has various benchmark tasks
	# We'll create synthetic tasks based on the framework
	agentlab_tasks = [
		"Navigate to a specific webpage and extract information",
		"Fill out a form with user data",
		"Search for products on an e-commerce site",
		"Book a flight on a travel website",
		"Create an account on a social media platform",
		"Download a file from a website",
		"Send an email through a web interface",
		"Schedule a meeting using a calendar app",
		"Upload a file to a cloud storage service",
		"Configure settings in a web application"
	]

	for i, task_desc in enumerate(agentlab_tasks):
		task = {
			"task_id": f"agentlab_{i:04d}",
			"obs": {
				"kind": "web",
				"content": task_desc,
				"meta": {
					"framework": "AgentLab",
					"domain": "web_automation"
				}
			},
			"plan": {
				"subgoals": [task_desc],
				"tools": ["web"],
				"constraints": {}
			},
			"action": {
				"tool": "web.execute",
				"args": {
					"instruction": task_desc,
					"framework": "AgentLab"
				}
			},
			"result": {
				"stdout": f"Web automation task completed: {task_desc}",
				"stderr": "",
				"status": "ok"
			},
			"reward": {
				"scalar": 1.0,
				"components": {
					"success": 1,
					"latency": 0,
					"side_effect": 1
				}
			},
			"critique": {
				"issues": [],
				"risk": 0.1,
				"proposal": ""
			}
		}
		tasks.append(task)

	return tasks

def load_workarena_tasks(workarena_dir: str) -> List[Dict]:
	"""Load WorkArena tasks and convert to our format."""
	tasks = []

	# WorkArena knowledge work tasks
	workarena_tasks = [
		"Create a new incident ticket in ServiceNow",
		"Update user information in the employee database",
		"Generate a report from the analytics dashboard",
		"Configure a new service catalog item",
		"Assign a task to a team member",
		"Update the knowledge base with new information",
		"Create a new user account with specific permissions",
		"Process a service request workflow",
		"Generate a performance report",
		"Configure system settings for a department"
	]

	for i, task_desc in enumerate(workarena_tasks):
		task = {
			"task_id": f"workarena_{i:04d}",
			"obs": {
				"kind": "enterprise",
				"content": task_desc,
				"meta": {
					"framework": "WorkArena",
					"domain": "knowledge_work"
				}
			},
			"plan": {
				"subgoals": [task_desc],
				"tools": ["enterprise"],
				"constraints": {}
			},
			"action": {
				"tool": "enterprise.execute",
				"args": {
					"instruction": task_desc,
					"framework": "WorkArena"
				}
			},
			"result": {
				"stdout": f"Enterprise task completed: {task_desc}",
				"stderr": "",
				"status": "ok"
			},
			"reward": {
				"scalar": 1.0,
				"components": {
					"success": 1,
					"latency": 0,
					"side_effect": 1
				}
			},
			"critique": {
				"issues": [],
				"risk": 0.1,
				"proposal": ""
			}
		}
		tasks.append(task)

	return tasks

def load_code_benchmark_tasks() -> List[Dict]:
	"""Load code generation benchmark tasks."""
	tasks = []

	code_tasks = [
		"Implement a binary search algorithm in Python",
		"Create a REST API endpoint for user authentication",
		"Write a function to parse JSON configuration files",
		"Implement a caching mechanism for database queries",
		"Create a unit test for a sorting function",
		"Write a script to process CSV files",
		"Implement error handling for file operations",
		"Create a data validation function",
		"Write a logging utility class",
		"Implement a simple web scraper"
	]

	for i, task_desc in enumerate(code_tasks):
		task = {
			"task_id": f"code_{i:04d}",
			"obs": {
				"kind": "code",
				"content": task_desc,
				"meta": {
					"framework": "CodeBenchmark",
					"domain": "programming"
				}
			},
			"plan": {
				"subgoals": [task_desc],
				"tools": ["code"],
				"constraints": {}
			},
			"action": {
				"tool": "code.execute",
				"args": {
					"instruction": task_desc,
					"framework": "CodeBenchmark"
				}
			},
			"result": {
				"stdout": f"Code generation task completed: {task_desc}",
				"stderr": "",
				"status": "ok"
			},
			"reward": {
				"scalar": 1.0,
				"components": {
					"success": 1,
					"latency": 0,
					"side_effect": 1
				}
			},
			"critique": {
				"issues": [],
				"risk": 0.1,
				"proposal": ""
			}
		}
		tasks.append(task)

	return tasks

def main():
	"""Convert inspiration tasks to training format."""
	inspiration_dir = "/data/agiattempt/inspiration"
	output_file = "/data/agiattempt/agi_dw/data/traces/seed_dev_tasks.jsonl"

	all_tasks = []

	# Load OSWorld tasks
	osworld_examples = os.path.join(inspiration_dir, "OSWorld", "evaluation_examples", "examples")
	if os.path.exists(osworld_examples):
		print("Loading OSWorld tasks...")
		osworld_tasks = load_osworld_tasks(osworld_examples)
		all_tasks.extend(osworld_tasks)
		print(f"Loaded {len(osworld_tasks)} OSWorld tasks")

	# Load AgentLab tasks
	print("Loading AgentLab tasks...")
	agentlab_tasks = load_agentlab_tasks(os.path.join(inspiration_dir, "AgentLab"))
	all_tasks.extend(agentlab_tasks)
	print(f"Loaded {len(agentlab_tasks)} AgentLab tasks")

	# Load WorkArena tasks
	print("Loading WorkArena tasks...")
	workarena_tasks = load_workarena_tasks(os.path.join(inspiration_dir, "WorkArena"))
	all_tasks.extend(workarena_tasks)
	print(f"Loaded {len(workarena_tasks)} WorkArena tasks")

	# Load code benchmark tasks
	print("Loading code benchmark tasks...")
	code_tasks = load_code_benchmark_tasks()
	all_tasks.extend(code_tasks)
	print(f"Loaded {len(code_tasks)} code benchmark tasks")

	# Shuffle and limit tasks
	random.shuffle(all_tasks)
	print(f"Total tasks loaded: {len(all_tasks)}")

	# Write to output file
	os.makedirs(os.path.dirname(output_file), exist_ok=True)
	with open(output_file, 'w') as f:
		for task in all_tasks:
			f.write(json.dumps(task) + '\n')

	print(f"Saved {len(all_tasks)} development tasks to {output_file}")

if __name__ == "__main__":
	main()


#!/usr/bin/env python3
import logging
"""
Convert software development tasks from inspiration folder into AGI training format.
This script extracts tasks from AgentLab, WorkArena, OSWorld, and other benchmarks
and converts them into our training trace format.
"""

import json
import os
import random
from pathlib import Path
from typing import Dict, List, Any

def load_osworld_tasks(examples_dir: str) -> List[Dict]:
	"""Load OSWorld tasks from evaluation examples."""
	tasks = []
	examples_path = Path(examples_dir)

	for app_dir in examples_path.iterdir():
		if not app_dir.is_dir():
			continue

		app_name = app_dir.name
		print(f"Processing {app_name} tasks...")

		for task_file in app_dir.glob("*.json"):
			try:
				with open(task_file, 'r') as f:
					task_data = json.load(f)

				# Convert OSWorld task to our format
				task = {
					"task_id": task_data.get("id", str(task_file.stem)),
					"obs": {
						"kind": "dev",
						"content": task_data.get("instruction", ""),
						"meta": {
							"app": app_name,
							"source": task_data.get("source", ""),
							"related_apps": task_data.get("related_apps", [])
						}
					},
					"plan": {
						"subgoals": [task_data.get("instruction", "")],
						"tools": ["dev"],
						"constraints": {}
					},
					"action": {
						"tool": "dev.execute",
						"args": {
							"instruction": task_data.get("instruction", ""),
							"app": app_name,
							"config": task_data.get("config", [])
						}
					},
					"result": {
						"stdout": f"Task completed in {app_name}",
						"stderr": "",
						"status": "ok"
					},
					"reward": {
						"scalar": 1.0,
						"components": {
							"success": 1,
							"latency": 0,
							"side_effect": 1
						}
					},
					"critique": {
						"issues": [],
						"risk": 0.1,
						"proposal": ""
					}
				}
				tasks.append(task)

			except Exception as e:
				print(f"Error processing {task_file}: {e}")
				continue

	return tasks

def load_agentlab_tasks(agentlab_dir: str) -> List[Dict]:
	"""Load AgentLab tasks and convert to our format."""
	tasks = []

	# AgentLab has various benchmark tasks
	# We'll create synthetic tasks based on the framework
	agentlab_tasks = [
		"Navigate to a specific webpage and extract information",
		"Fill out a form with user data",
		"Search for products on an e-commerce site",
		"Book a flight on a travel website",
		"Create an account on a social media platform",
		"Download a file from a website",
		"Send an email through a web interface",
		"Schedule a meeting using a calendar app",
		"Upload a file to a cloud storage service",
		"Configure settings in a web application"
	]


						"components": {
							"success": 1,
							"latency": 0,
							"side_effect": 1
						}
					},
					"critique": {
						"issues": [],
						"risk": 0.1,
						"proposal": ""
					}
				}
				tasks.append(task)

			except Exception as e:
				print(f"Error processing {task_file}: {e}")
				continue

	return tasks

def load_agentlab_tasks(agentlab_dir: str) -> List[Dict]:
	"""Load AgentLab tasks and convert to our format."""
	tasks = []

	# AgentLab has various benchmark tasks
	# We'll create synthetic tasks based on the framework
	agentlab_tasks = [
		"Navigate to a specific webpage and extract information",
		"Fill out a form with user data",
		"Search for products on an e-commerce site",
		"Book a flight on a travel website",
		"Create an account on a social media platform",
		"Download a file from a website",
		"Send an email through a web interface",
		"Schedule a meeting using a calendar app",
		"Upload a file to a cloud storage service",
		"Configure settings in a web application"
	]

	for i, task_desc in enumerate(agentlab_tasks):
		task = {
			"task_id": f"agentlab_{i:04d}",
			"obs": {
				"kind": "web",
				"content": task_desc,
				"meta": {
					"framework": "AgentLab",
					"domain": "web_automation"
				}
			},
			"plan": {
				"subgoals": [task_desc],
				"tools": ["web"],
				"constraints": {}
			},
			"action": {
				"tool": "web.execute",
				"args": {
					"instruction": task_desc,
					"framework": "AgentLab"
				}
			},
			"result": {
				"stdout": f"Web automation task completed: {task_desc}",
				"stderr": "",
				"status": "ok"
			},
			"reward": {
				"scalar": 1.0,
				"components": {
					"success": 1,
					"latency": 0,
					"side_effect": 1
				}
			},
			"critique": {
				"issues": [],
				"risk": 0.1,
				"proposal": ""
			}
		}
		tasks.append(task)

	return tasks

def load_workarena_tasks(workarena_dir: str) -> List[Dict]:
	"""Load WorkArena tasks and convert to our format."""
	tasks = []

	# WorkArena knowledge work tasks
	workarena_tasks = [
		"Create a new incident ticket in ServiceNow",
		"Update user information in the employee database",
		"Generate a report from the analytics dashboard",
		"Configure a new service catalog item",
		"Assign a task to a team member",
		"Update the knowledge base with new information",
		"Create a new user account with specific permissions",
		"Process a service request workflow",
		"Generate a performance report",
		"Configure system settings for a department"
	]

	for i, task_desc in enumerate(workarena_tasks):


				"status": "ok"
			},
			"reward": {
				"scalar": 1.0,
				"components": {
					"success": 1,
					"latency": 0,
					"side_effect": 1
				}
			},
			"critique": {
				"issues": [],
				"risk": 0.1,
				"proposal": ""
			}
		}
		tasks.append(task)

	return tasks

def load_workarena_tasks(workarena_dir: str) -> List[Dict]:
	"""Load WorkArena tasks and convert to our format."""
	tasks = []

	# WorkArena knowledge work tasks
	workarena_tasks = [
		"Create a new incident ticket in ServiceNow",
		"Update user information in the employee database",
		"Generate a report from the analytics dashboard",
		"Configure a new service catalog item",
		"Assign a task to a team member",
		"Update the knowledge base with new information",
		"Create a new user account with specific permissions",
		"Process a service request workflow",
		"Generate a performance report",
		"Configure system settings for a department"
	]

	for i, task_desc in enumerate(workarena_tasks):
		task = {
			"task_id": f"workarena_{i:04d}",
			"obs": {
				"kind": "enterprise",
				"content": task_desc,
				"meta": {
					"framework": "WorkArena",
					"domain": "knowledge_work"
				}
			},
			"plan": {
				"subgoals": [task_desc],
				"tools": ["enterprise"],
				"constraints": {}
			},
			"action": {
				"tool": "enterprise.execute",
				"args": {
					"instruction": task_desc,
					"framework": "WorkArena"
				}
			},
			"result": {
				"stdout": f"Enterprise task completed: {task_desc}",
				"stderr": "",
				"status": "ok"
			},
			"reward": {
				"scalar": 1.0,
				"components": {
					"success": 1,
					"latency": 0,
					"side_effect": 1
				}
			},
			"critique": {
				"issues": [],
				"risk": 0.1,
				"proposal": ""
			}
		}
		tasks.append(task)

	return tasks

def load_code_benchmark_tasks() -> List[Dict]:
	"""Load code generation benchmark tasks."""
	tasks = []

	code_tasks = [
		"Implement a binary search algorithm in Python",
		"Create a REST API endpoint for user authentication",
		"Write a function to parse JSON configuration files",
		"Implement a caching mechanism for database queries",
		"Create a unit test for a sorting function",
		"Write a script to process CSV files",
		"Implement error handling for file operations",
		"Create a data validation function",
		"Write a logging utility class",
		"Implement a simple web scraper"
	]

	for i, task_desc in enumerate(code_tasks):
		task = {


				"status": "ok"
			},
			"reward": {
				"scalar": 1.0,
				"components": {
					"success": 1,
					"latency": 0,
					"side_effect": 1
				}
			},
			"critique": {
				"issues": [],
				"risk": 0.1,
				"proposal": ""
			}
		}
		tasks.append(task)

	return tasks

def load_code_benchmark_tasks() -> List[Dict]:
	"""Load code generation benchmark tasks."""
	tasks = []

	code_tasks = [
		"Implement a binary search algorithm in Python",
		"Create a REST API endpoint for user authentication",
		"Write a function to parse JSON configuration files",
		"Implement a caching mechanism for database queries",
		"Create a unit test for a sorting function",
		"Write a script to process CSV files",
		"Implement error handling for file operations",
		"Create a data validation function",
		"Write a logging utility class",
		"Implement a simple web scraper"
	]

	for i, task_desc in enumerate(code_tasks):
		task = {
			"task_id": f"code_{i:04d}",
			"obs": {
				"kind": "code",
				"content": task_desc,
				"meta": {
					"framework": "CodeBenchmark",
					"domain": "programming"
				}
			},
			"plan": {
				"subgoals": [task_desc],
				"tools": ["code"],
				"constraints": {}
			},
			"action": {
				"tool": "code.execute",
				"args": {
					"instruction": task_desc,
					"framework": "CodeBenchmark"
				}
			},
			"result": {
				"stdout": f"Code generation task completed: {task_desc}",
				"stderr": "",
				"status": "ok"
			},
			"reward": {
				"scalar": 1.0,
				"components": {
					"success": 1,
					"latency": 0,
					"side_effect": 1
				}
			},
			"critique": {
				"issues": [],
				"risk": 0.1,
				"proposal": ""
			}
		}
		tasks.append(task)

	return tasks

def main():
	"""Convert inspiration tasks to training format."""
	inspiration_dir = "/data/agiattempt/inspiration"
	output_file = "/data/agiattempt/agi_dw/data/traces/seed_dev_tasks.jsonl"

	all_tasks = []

	# Load OSWorld tasks
	osworld_examples = os.path.join(inspiration_dir, "OSWorld", "evaluation_examples", "examples")
	if os.path.exists(osworld_examples):
		print("Loading OSWorld tasks...")
		osworld_tasks = load_osworld_tasks(osworld_examples)
		all_tasks.extend(osworld_tasks)
		print(f"Loaded {len(osworld_tasks)} OSWorld tasks")

	# Load AgentLab tasks
	print("Loading AgentLab tasks...")
	agentlab_tasks = load_agentlab_tasks(os.path.join(inspiration_dir, "AgentLab"))
	all_tasks.extend(agentlab_tasks)


				"status": "ok"
			},
			"reward": {
				"scalar": 1.0,
				"components": {
					"success": 1,
					"latency": 0,
					"side_effect": 1
				}
			},
			"critique": {
				"issues": [],
				"risk": 0.1,
				"proposal": ""
			}
		}
		tasks.append(task)

	return tasks

def main():
	"""Convert inspiration tasks to training format."""
	inspiration_dir = "/data/agiattempt/inspiration"
	output_file = "/data/agiattempt/agi_dw/data/traces/seed_dev_tasks.jsonl"

	all_tasks = []

	# Load OSWorld tasks
	osworld_examples = os.path.join(inspiration_dir, "OSWorld", "evaluation_examples", "examples")
	if os.path.exists(osworld_examples):
		print("Loading OSWorld tasks...")
		osworld_tasks = load_osworld_tasks(osworld_examples)
		all_tasks.extend(osworld_tasks)
		print(f"Loaded {len(osworld_tasks)} OSWorld tasks")

	# Load AgentLab tasks
	print("Loading AgentLab tasks...")
	agentlab_tasks = load_agentlab_tasks(os.path.join(inspiration_dir, "AgentLab"))
	all_tasks.extend(agentlab_tasks)
	print(f"Loaded {len(agentlab_tasks)} AgentLab tasks")

	# Load WorkArena tasks
	print("Loading WorkArena tasks...")
	workarena_tasks = load_workarena_tasks(os.path.join(inspiration_dir, "WorkArena"))
	all_tasks.extend(workarena_tasks)
	print(f"Loaded {len(workarena_tasks)} WorkArena tasks")

	# Load code benchmark tasks
	print("Loading code benchmark tasks...")
	code_tasks = load_code_benchmark_tasks()
	all_tasks.extend(code_tasks)
	print(f"Loaded {len(code_tasks)} code benchmark tasks")

	# Shuffle and limit tasks
	random.shuffle(all_tasks)
	print(f"Total tasks loaded: {len(all_tasks)}")

	# Write to output file
	os.makedirs(os.path.dirname(output_file), exist_ok=True)
	with open(output_file, 'w') as f:
		for task in all_tasks:
			f.write(json.dumps(task) + '\n')

	print(f"Saved {len(all_tasks)} development tasks to {output_file}")

if __name__ == "__main__":
	main()


from __future__ import annotations
import logging

import argparse
import json
import os
import subprocess
import time
from pathlib import Path
from typing import Any, Dict, List

def _run(argv: List[str], cwd: Path) -> Dict[str, Any]:
	start = time.time()
	try:
		p = subprocess.run(argv, cwd=cwd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, encoding="utf-8")
		out = (p.stdout or "")
		err = (p.stderr or "")
		rc = int(p.returncode)
	except Exception as e:
		out = ""
		err = str(e)
		rc = 1
	end = time.time()
	return {"argv": argv, "rc": rc, "stdout": out, "stderr": err, "elapsed_sec": float(round(max(0.0, end - start), 3))}

def _read_json_safe(p: Path) -> Any:
	try:
		if p.exists():
			return json.loads(p.read_text(encoding="utf-8"))
	except Exception:
		pass
	return None

def build_world_snapshot(root: Path, out_path: Path, refresh: bool = False) -> Dict[str, Any]:
	root = root.resolve()
	out_path.parent.mkdir(parents=True, exist_ok=True)
	# Ensure PYTHONPATH includes monorepo root by default
	os.environ.setdefault("PYTHONPATH", str(root.parent))

	manifest_path = root / "data" / "sandbox" / "tmp" / "repo_manifest.json"
	code_index_path = root / "data" / "sandbox" / "tmp" / "code_index.json"
	bm25_index_path = root / "data" / "sandbox" / "tmp" / "bm25_index.json"
	embed_index_path = root / "data" / "sandbox" / "tmp" / "embed_index.json"
	# Inspiration artifacts
	insp_index_path = root / "data" / "sandbox" / "inspiration" / "index.json"
	insp_embed_path = root / "data" / "sandbox" / "inspiration" / "embed_index.json"
	metrics_path = root / "data" / "devtools" / "metrics.json"
	traces_dir = root / "data" / "traces"
	tools_registry = root / "config" / "tools_refactor.json"
	tools_registry_qa = root / "config" / "tools_qa.json"
	qa_traces_path = root / "data" / "devtools" / "qa.traces.jsonl"

	steps: List[Dict[str, Any]] = []
	if refresh:
		steps.append(_run(["python3", str(root / "tools" / "repo_manifest.py"), "--root", str(root), "--out", str(manifest_path)], cwd=root))
		steps.append(_run(["python3", str(root / "tools" / "code_index.py"), "--root", str(root), "--out", str(code_index_path)], cwd=root))

	# Collect mk files and simple target names (best-effort)
	mk_dir = root / "mk"
	mk_files = [p.as_posix() for p in mk_dir.glob("*.mk")]

	# Recent traces and plans
	trace_files: List[str] = []
	if traces_dir.exists():
		for p in sorted(traces_dir.glob("**/*")):
			if p.suffix in (".json", ".jsonl"):
				trace_files.append(p.as_posix())
	trace_files = trace_files[-100:]

	world: Dict[str, Any] = {
		"version": "0.1",
		"generated_at": int(time.time()),
		"repo_root": root.as_posix(),
		"manifest_path": manifest_path.as_posix(),
		"code_index_path": code_index_path.as_posix(),
		"bm25_index_path": bm25_index_path.as_posix(),
		"embed_index_path": embed_index_path.as_posix(),
		"inspiration_index_path": insp_index_path.as_posix(),
		"inspiration_embed_path": insp_embed_path.as_posix(),
		"devtools_metrics_path": metrics_path.as_posix(),
		"tools_registry_path": tools_registry.as_posix(),
		"tools_registry_qa_path": tools_registry_qa.as_posix(),
		"qa_traces_path": qa_traces_path.as_posix(),
		"mk_files": mk_files,
		"recent_traces": trace_files,
		"manifest": _read_json_safe(manifest_path),
		"code_index": _read_json_safe(code_index_path),
		"devtools_metrics": _read_json_safe(metrics_path),
		"inspiration_index": _read_json_safe(insp_index_path),
		"inspiration_embed": _read_json_safe(insp_embed_path),
		"steps": steps,
	}

	out_path.write_text(json.dumps(world, ensure_ascii=False, indent=2), encoding="utf-8")
	return world

def main() -> int:
	ap = argparse.ArgumentParser()
	root = Path(__file__).resolve().parents[2]
	ap.add_argument("--root", default=str(root))
	ap.add_argument("--out", default=str(root / "data" / "sandbox" / "tmp" / "world.json"))
	ap.add_argument("--refresh", action="store_true", help="Regenerate manifest and code index before snapshot")
	args = ap.parse_args()

	world = build_world_snapshot(Path(args.root), Path(args.out), refresh=bool(args.refresh))
	print(json.dumps({"ok": True, "wrote": str(args.out), "manifest": bool(world.get("manifest") is not None), "code_index": bool(world.get("code_index") is not None)}))
	return 0

if __name__ == "__main__":
	raise SystemExit(main())


from __future__ import annotations
import logging

import argparse
import json
import os
import subprocess
import time
from pathlib import Path
from typing import Any, Dict, List

def _run(argv: List[str], cwd: Path) -> Dict[str, Any]:
	start = time.time()
	try:
		p = subprocess.run(argv, cwd=cwd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, encoding="utf-8")
		out = (p.stdout or "")
		err = (p.stderr or "")
		rc = int(p.returncode)
	except Exception as e:
		out = ""
		err = str(e)
		rc = 1
	end = time.time()
	return {"argv": argv, "rc": rc, "stdout": out, "stderr": err, "elapsed_sec": float(round(max(0.0, end - start), 3))}

def _read_json_safe(p: Path) -> Any:
	try:
		if p.exists():
			return json.loads(p.read_text(encoding="utf-8"))
	except Exception:
		pass
	return None

def build_world_snapshot(root: Path, out_path: Path, refresh: bool = False) -> Dict[str, Any]:
	root = root.resolve()
	out_path.parent.mkdir(parents=True, exist_ok=True)
	# Ensure PYTHONPATH includes monorepo root by default
	os.environ.setdefault("PYTHONPATH", str(root.parent))

	manifest_path = root / "data" / "sandbox" / "tmp" / "repo_manifest.json"
	code_index_path = root / "data" / "sandbox" / "tmp" / "code_index.json"
	bm25_index_path = root / "data" / "sandbox" / "tmp" / "bm25_index.json"


import time
from pathlib import Path
from typing import Any, Dict, List

def _run(argv: List[str], cwd: Path) -> Dict[str, Any]:
	start = time.time()
	try:
		p = subprocess.run(argv, cwd=cwd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, encoding="utf-8")
		out = (p.stdout or "")
		err = (p.stderr or "")
		rc = int(p.returncode)
	except Exception as e:
		out = ""
		err = str(e)
		rc = 1
	end = time.time()
	return {"argv": argv, "rc": rc, "stdout": out, "stderr": err, "elapsed_sec": float(round(max(0.0, end - start), 3))}

def _read_json_safe(p: Path) -> Any:
	try:
		if p.exists():
			return json.loads(p.read_text(encoding="utf-8"))
	except Exception:
		pass
	return None

def build_world_snapshot(root: Path, out_path: Path, refresh: bool = False) -> Dict[str, Any]:
	root = root.resolve()
	out_path.parent.mkdir(parents=True, exist_ok=True)
	# Ensure PYTHONPATH includes monorepo root by default
	os.environ.setdefault("PYTHONPATH", str(root.parent))

	manifest_path = root / "data" / "sandbox" / "tmp" / "repo_manifest.json"
	code_index_path = root / "data" / "sandbox" / "tmp" / "code_index.json"
	bm25_index_path = root / "data" / "sandbox" / "tmp" / "bm25_index.json"
	embed_index_path = root / "data" / "sandbox" / "tmp" / "embed_index.json"
	# Inspiration artifacts
	insp_index_path = root / "data" / "sandbox" / "inspiration" / "index.json"
	insp_embed_path = root / "data" / "sandbox" / "inspiration" / "embed_index.json"
	metrics_path = root / "data" / "devtools" / "metrics.json"
	traces_dir = root / "data" / "traces"
	tools_registry = root / "config" / "tools_refactor.json"
	tools_registry_qa = root / "config" / "tools_qa.json"
	qa_traces_path = root / "data" / "devtools" / "qa.traces.jsonl"


		out = (p.stdout or "")
		err = (p.stderr or "")
		rc = int(p.returncode)
	except Exception as e:
		out = ""
		err = str(e)
		rc = 1
	end = time.time()
	return {"argv": argv, "rc": rc, "stdout": out, "stderr": err, "elapsed_sec": float(round(max(0.0, end - start), 3))}

def _read_json_safe(p: Path) -> Any:
	try:
		if p.exists():
			return json.loads(p.read_text(encoding="utf-8"))
	except Exception:
		pass
	return None

def build_world_snapshot(root: Path, out_path: Path, refresh: bool = False) -> Dict[str, Any]:
	root = root.resolve()
	out_path.parent.mkdir(parents=True, exist_ok=True)
	# Ensure PYTHONPATH includes monorepo root by default
	os.environ.setdefault("PYTHONPATH", str(root.parent))

	manifest_path = root / "data" / "sandbox" / "tmp" / "repo_manifest.json"
	code_index_path = root / "data" / "sandbox" / "tmp" / "code_index.json"
	bm25_index_path = root / "data" / "sandbox" / "tmp" / "bm25_index.json"
	embed_index_path = root / "data" / "sandbox" / "tmp" / "embed_index.json"
	# Inspiration artifacts
	insp_index_path = root / "data" / "sandbox" / "inspiration" / "index.json"
	insp_embed_path = root / "data" / "sandbox" / "inspiration" / "embed_index.json"
	metrics_path = root / "data" / "devtools" / "metrics.json"
	traces_dir = root / "data" / "traces"
	tools_registry = root / "config" / "tools_refactor.json"
	tools_registry_qa = root / "config" / "tools_qa.json"
	qa_traces_path = root / "data" / "devtools" / "qa.traces.jsonl"

	steps: List[Dict[str, Any]] = []
	if refresh:
		steps.append(_run(["python3", str(root / "tools" / "repo_manifest.py"), "--root", str(root), "--out", str(manifest_path)], cwd=root))
		steps.append(_run(["python3", str(root / "tools" / "code_index.py"), "--root", str(root), "--out", str(code_index_path)], cwd=root))

	# Collect mk files and simple target names (best-effort)
	mk_dir = root / "mk"
	mk_files = [p.as_posix() for p in mk_dir.glob("*.mk")]

	# Recent traces and plans
	trace_files: List[str] = []
	if traces_dir.exists():
		for p in sorted(traces_dir.glob("**/*")):
			if p.suffix in (".json", ".jsonl"):
				trace_files.append(p.as_posix())
	trace_files = trace_files[-100:]

	world: Dict[str, Any] = {
		"version": "0.1",
		"generated_at": int(time.time()),
		"repo_root": root.as_posix(),
		"manifest_path": manifest_path.as_posix(),
		"code_index_path": code_index_path.as_posix(),
		"bm25_index_path": bm25_index_path.as_posix(),
		"embed_index_path": embed_index_path.as_posix(),
		"inspiration_index_path": insp_index_path.as_posix(),
		"inspiration_embed_path": insp_embed_path.as_posix(),
		"devtools_metrics_path": metrics_path.as_posix(),
		"tools_registry_path": tools_registry.as_posix(),
		"tools_registry_qa_path": tools_registry_qa.as_posix(),
		"qa_traces_path": qa_traces_path.as_posix(),
		"mk_files": mk_files,
		"recent_traces": trace_files,
		"manifest": _read_json_safe(manifest_path),
		"code_index": _read_json_safe(code_index_path),
		"devtools_metrics": _read_json_safe(metrics_path),
		"inspiration_index": _read_json_safe(insp_index_path),
		"inspiration_embed": _read_json_safe(insp_embed_path),
		"steps": steps,
	}

	out_path.write_text(json.dumps(world, ensure_ascii=False, indent=2), encoding="utf-8")
	return world

def main() -> int:
	ap = argparse.ArgumentParser()
	root = Path(__file__).resolve().parents[2]
	ap.add_argument("--root", default=str(root))
	ap.add_argument("--out", default=str(root / "data" / "sandbox" / "tmp" / "world.json"))
	ap.add_argument("--refresh", action="store_true", help="Regenerate manifest and code index before snapshot")
	args = ap.parse_args()

	world = build_world_snapshot(Path(args.root), Path(args.out), refresh=bool(args.refresh))
	print(json.dumps({"ok": True, "wrote": str(args.out), "manifest": bool(world.get("manifest") is not None), "code_index": bool(world.get("code_index") is not None)}))
	return 0

if __name__ == "__main__":
	raise SystemExit(main())


		"inspiration_index_path": insp_index_path.as_posix(),
		"inspiration_embed_path": insp_embed_path.as_posix(),
		"devtools_metrics_path": metrics_path.as_posix(),
		"tools_registry_path": tools_registry.as_posix(),
		"tools_registry_qa_path": tools_registry_qa.as_posix(),
		"qa_traces_path": qa_traces_path.as_posix(),
		"mk_files": mk_files,
		"recent_traces": trace_files,
		"manifest": _read_json_safe(manifest_path),
		"code_index": _read_json_safe(code_index_path),
		"devtools_metrics": _read_json_safe(metrics_path),
		"inspiration_index": _read_json_safe(insp_index_path),
		"inspiration_embed": _read_json_safe(insp_embed_path),
		"steps": steps,
	}

	out_path.write_text(json.dumps(world, ensure_ascii=False, indent=2), encoding="utf-8")
	return world

def main() -> int:
	ap = argparse.ArgumentParser()
	root = Path(__file__).resolve().parents[2]
	ap.add_argument("--root", default=str(root))
	ap.add_argument("--out", default=str(root / "data" / "sandbox" / "tmp" / "world.json"))
	ap.add_argument("--refresh", action="store_true", help="Regenerate manifest and code index before snapshot")
	args = ap.parse_args()

	world = build_world_snapshot(Path(args.root), Path(args.out), refresh=bool(args.refresh))
	print(json.dumps({"ok": True, "wrote": str(args.out), "manifest": bool(world.get("manifest") is not None), "code_index": bool(world.get("code_index") is not None)}))
	return 0

if __name__ == "__main__":
	raise SystemExit(main())


import logging
import argparse
import json
import time
from pathlib import Path
from subprocess import run, PIPE  # type: ignore
from typing import Dict, Any, List, Optional

def run_cmd(cmd: List[str]) -> Dict[str, Any]:
	start = time.time()
	p = run(cmd, stdout=PIPE, stderr=PIPE, encoding="utf-8")
	dur = time.time() - start
	ok = False
	risk = None
	mem_hit = None
	try:
		lines = p.stdout.strip().splitlines() if p.stdout else []
		# Look for the line with "status" field (not just the last line)
		obj = {}
		for line in lines:
			if line.startswith("{") and "status" in line:
				obj = json.loads(line)
				break
		ok = str(obj.get("status", "")).lower() == "ok"
		risk = float(obj.get("risk", 0.5)) if "risk" in obj else None
		if "mem_hit" in obj:
			mem_hit = bool(obj.get("mem_hit"))
	except Exception:
		ok = False
	return {"ok": bool(ok), "risk": risk, "dur": float(dur), "rc": int(p.returncode), "mem_hit": mem_hit}

def bench_cli(root: Path, runs: int, budget_sec: Optional[float] = None, cost_per_sec: Optional[float] = None) -> Dict[str, Any]:
	cli_tasks = ["count_lines", "grep_error"]
	res: Dict[str, Any] = {}
	# Aggregate across all tasks and runs
	total_success = 0
	total_success_budgeted = 0
	total_runs = 0
	total_latency = 0.0
	total_over_budget = 0
	total_cost = 0.0
	total_mem_hits = 0
	for task in cli_tasks:
		ok_n = 0
		ok_budget_n = 0
		dur_sum = 0.0
		over_budget_n = 0
		cost_sum = 0.0
		mem_hit_n = 0
		for _ in range(runs):
			cmd = [
				"python3", str(root / "scripts" / "run_loop_oscli.py"),
				"--planner-backend", "hf",
				"--verifier-backend", "hf",
				"--planner-model", "meta-llama/Llama-3.2-3B",
				"--verifier-model", "meta-llama/Llama-3.2-3B",
				"--timeout", "20",
				"--task", task,
			]
			# Optionally enable memory usage for auditing cache/batch policy
			try:
				import os as _os
				if _os.environ.get("AGI_BENCH_USE_MEMORY_OSCLI", "0") not in ("0", "false", "False"):
					cmd.extend(["--use-memory"])
			except Exception:
				pass
			out = run_cmd(cmd)
			over = bool(budget_sec is not None and float(out["dur"]) > float(budget_sec))
			ok_n += 1 if out["ok"] else 0
			ok_budget_n += 1 if (out["ok"] and not over) else 0
			over_budget_n += 1 if over else 0
			dur_sum += float(out["dur"])
			if cost_per_sec is not None:
				cost_sum += float(out["dur"]) * float(cost_per_sec)
			if out.get("mem_hit") is True:
				mem_hit_n += 1
		# Aggregate into totals across tasks
		total_success += int(ok_n)
		total_success_budgeted += int(ok_budget_n)
		total_runs += int(runs)
		total_latency += float(dur_sum)
		total_over_budget += int(over_budget_n)
		total_cost += float(cost_sum)
		total_mem_hits += int(mem_hit_n)
		res[f"cli:{task}"] = {
			"success": ok_n,
			"success_budgeted": ok_budget_n,
			"runs": runs,
			"success_rate": (ok_n / max(1, runs)),
			"success_rate_budgeted": (ok_budget_n / max(1, runs)),
			"avg_latency_sec": (dur_sum / max(1, runs)),
			"over_budget": over_budget_n,
			"budget_sec": float(budget_sec) if budget_sec is not None else None,
			"cost_total": float(cost_sum) if cost_per_sec is not None else None,
			"cost_per_success": (float(cost_sum) / max(1, ok_n)) if cost_per_sec is not None else None,
			"memory_hit_rate": (mem_hit_n / max(1, runs)),
		}
	res["cli_summary"] = {
		"success": int(total_success),
		"success_budgeted": int(total_success_budgeted),
		"runs": int(total_runs),
		"success_rate": (float(total_success) / max(1, float(total_runs))),
		"success_rate_budgeted": (float(total_success_budgeted) / max(1, float(total_runs))),
		"avg_latency_sec": (float(total_latency) / max(1.0, float(total_runs))),
		"over_budget_total_runs": int(total_over_budget),
		"over_budget_rate": (float(total_over_budget) / max(1.0, float(total_runs))),
		"budget_sec": float(budget_sec) if budget_sec is not None else None,
		"cost_total": float(total_cost) if cost_per_sec is not None else None,
		"cost_per_success": (float(total_cost) / max(1.0, float(total_success))) if cost_per_sec is not None else None,
		"memory_hit_rate": (float(total_mem_hits) / max(1.0, float(total_runs))),
	}
	return res

def bench_dom(root: Path, runs: int, budget_sec: Optional[float] = None, cost_per_sec: Optional[float] = None) -> Dict[str, Any]:
	dom_cases = [
		{"url": "https://example.com", "selector": "h1"},
		{"url": "https://en.wikipedia.org/wiki/Alan_Turing", "selector": "#firstHeading"},
	]
	res: Dict[str, Any] = {}
	total_success = 0
	total_success_budgeted = 0
	total_runs = 0
	total_latency = 0.0
	total_over_budget = 0
	total_cost = 0.0
	total_mem_hits = 0
	for case in dom_cases:
		ok_n = 0
		ok_budget_n = 0
		dur_sum = 0.0
		over_budget_n = 0
		cost_sum = 0.0
		mem_hit_n = 0
		for _ in range(runs):
			cmd = [
				"python3", str(root / "scripts" / "run_loop_webdom.py"),
				"--planner-backend", "hf",
				"--verifier-backend", "hf",
				"--planner-model", "meta-llama/Llama-3.2-3B",
				"--verifier-model", "meta-llama/Llama-3.2-3B",
				"--timeout", "25",
				"--url", case["url"],
				"--selector", case["selector"],
			]
			# Optionally enable memory usage for auditing cache/batch policy
			try:
				import os as _os
				if _os.environ.get("AGI_BENCH_USE_MEMORY_DOM", "0") not in ("0", "false", "False"):
					cmd.extend(["--use-memory"])
			except Exception:
				pass
			out = run_cmd(cmd)
			over = bool(budget_sec is not None and float(out["dur"]) > float(budget_sec))
			ok_n += 1 if out["ok"] else 0
			ok_budget_n += 1 if (out["ok"] and not over) else 0
			over_budget_n += 1 if over else 0
			dur_sum += float(out["dur"])
			if cost_per_sec is not None:
				cost_sum += float(out["dur"]) * float(cost_per_sec)
			if out.get("mem_hit") is True:
				mem_hit_n += 1
			# Accumulate across runs for this case
		total_success += int(ok_n)
		total_success_budgeted += int(ok_budget_n)
		total_runs += int(runs)
		total_latency += float(dur_sum)
		total_over_budget += int(over_budget_n)
		total_cost += float(cost_sum)
		total_mem_hits += int(mem_hit_n)
		key = f"dom:{case['url']}#{case['selector']}"
		res[key] = {
			"success": ok_n,
			"success_budgeted": ok_budget_n,
			"runs": runs,
			"success_rate": (ok_n / max(1, runs)),
			"success_rate_budgeted": (ok_budget_n / max(1, runs)),
			"avg_latency_sec": (dur_sum / max(1, runs)),
			"over_budget": over_budget_n,
			"budget_sec": float(budget_sec) if budget_sec is not None else None,
			"cost_total": float(cost_sum) if cost_per_sec is not None else None,
			"cost_per_success": (float(cost_sum) / max(1, ok_n)) if cost_per_sec is not None else None,
			"memory_hit_rate": (mem_hit_n / max(1, runs)),
		}
	res["dom_summary"] = {
		"success": int(total_success),
		"success_budgeted": int(total_success_budgeted),
		"runs": int(total_runs),
		"success_rate": (float(total_success) / max(1.0, float(total_runs))) ,
		"success_rate_budgeted": (float(total_success_budgeted) / max(1.0, float(total_runs))) ,
		"avg_latency_sec": (float(total_latency) / max(1.0, float(total_runs))),
		"over_budget_total_runs": int(total_over_budget),
		"over_budget_rate": (float(total_over_budget) / max(1.0, float(total_runs))),
		"budget_sec": float(budget_sec) if budget_sec is not None else None,
		"cost_total": float(total_cost) if cost_per_sec is not None else None,
		"cost_per_success": (float(total_cost) / max(1.0, float(total_success))) if cost_per_sec is not None else None,
		"memory_hit_rate": (float(total_mem_hits) / max(1.0, float(total_runs))),
	}
	return res

def bench_office(root: Path, runs: int) -> Dict[str, Any]:
	"""Office/Docs micro-suite: run docs Q&A tasks and aggregate simple KPIs."""
	res: Dict[str, Any] = {}
	# Run the docs suite once to produce results JSONL
	out_jsonl = root / "data" / "benchmarks" / "docs_results.jsonl"
	try:
		p = run([
			"python3", str(root / "scripts" / "run_docs_suite.py"),
			"--tasks", str(root / "data" / "docs" / "tasks.jsonl"),
			"--docs", str(root / "data" / "docs"),
			"--out", str(out_jsonl),
		], stdout=PIPE, stderr=PIPE, encoding="utf-8")
		# proceed even if returncode != 0; we'll compute from file if present
	except Exception:
		pass
	# Read results and compute simple success metrics
	total = 0
	ok = 0
	try:
		if out_jsonl.exists():
			with out_jsonl.open("r", encoding="utf-8") as f:
				for line in f:
					line = line.strip()
					if not line:
						continue
					try:
						rec = json.loads(line)
					except Exception:
						continue
					total += 1
					ok += 1 if bool(rec.get("ok", False)) else 0
	except Exception:
		pass
	res["office:docs_suite"] = {
		"success": int(ok),
		"runs": int(max(1, total)),
		"success_rate": (float(ok) / max(1.0, float(total))),
	}
	res["office_summary"] = {
		"success": int(ok),
		"runs": int(max(1, total)),
		"success_rate": (float(ok) / max(1.0, float(total))),
	}
	return res

def main() -> int:
	ap = argparse.ArgumentParser()
	root = Path(__file__).resolve().parents[1]
	ap.add_argument("--runs", type=int, default=2)
	ap.add_argument("--budget-cli-sec", type=float, default=None, help="Optional per-run latency budget (seconds) for CLI tasks")
	ap.add_argument("--budget-dom-sec", type=float, default=None, help="Optional per-run latency budget (seconds) for DOM tasks")
	ap.add_argument("--cost-cli-per-sec", type=float, default=None, help="Optional estimated cost rate ($/sec) for CLI loop")
	ap.add_argument("--cost-dom-per-sec", type=float, default=None, help="Optional estimated cost rate ($/sec) for DOM loop")
	ap.add_argument("--include-practice", action="store_true", help="If set, run coding practice suite before aggregating dashboard")
	ap.add_argument("--practice-tier", default="T1")
	ap.add_argument("--practice-out", default=str(root / "data" / "logs" / "practice_results.jsonl"))
	ap.add_argument("--domains", default="cli,dom", help="Comma-separated domains to include: cli,dom,office")
	ap.add_argument("--out", default=str(root / "data" / "benchmarks" / "kpi.json"))
	args = ap.parse_args()

	root.mkdir(parents=True, exist_ok=True)
	out_dir = Path(args.out).parent
	out_dir.mkdir(parents=True, exist_ok=True)

	kpis: Dict[str, Any] = {}
	requested = [d.strip() for d in str(args.domains).split(",") if d.strip()]
	if "cli" in requested:
		kpis.update(bench_cli(
			root,
			max(1, int(args.runs)),
			float(args.budget_cli_sec) if args.budget_cli_sec is not None else None,
			float(args.cost_cli_per_sec) if args.cost_cli_per_sec is not None else None,
		))
	if "dom" in requested:
		kpis.update(bench_dom(
			root,
			max(1, int(args.runs)),
			float(args.budget_dom_sec) if args.budget_dom_sec is not None else None,
			float(args.cost_dom_per_sec) if args.cost_dom_per_sec is not None else None,
		))
	if "office" in requested:
		kpis.update(bench_office(root, max(1, int(args.runs))))
	kpis["meta"] = {
		"runs_per_case": int(args.runs),
		"ts": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
		"budget_cli_sec": (float(args.budget_cli_sec) if args.budget_cli_sec is not None else None),
		"budget_dom_sec": (float(args.budget_dom_sec) if args.budget_dom_sec is not None else None),
		"cost_cli_per_sec": (float(args.cost_cli_per_sec) if args.cost_cli_per_sec is not None else None),
		"cost_dom_per_sec": (float(args.cost_dom_per_sec) if args.cost_dom_per_sec is not None else None),
	}

	Path(args.out).write_text(json.dumps(kpis, ensure_ascii=False, indent=2), encoding="utf-8")
	# Optionally run coding practice suite (tiny repos) to produce practice logs consumed by dashboard aggregator
	if args.include_practice:
		try:
			cmd = [
				"python3", str(root / "scripts" / "run_practice_suite.py"),
				"--tier", str(args.practice_tier),
				"--out", str(args.practice_out),
			]
			_ = run(cmd, stdout=PIPE, stderr=PIPE, encoding="utf-8")  # type: ignore
		except Exception:
			pass
	print(json.dumps({"ok": True, "out": str(args.out)}))
	return 0

if __name__ == "__main__":
	raise SystemExit(main())


import logging
import argparse
import json
import time
from pathlib import Path
from subprocess import run, PIPE  # type: ignore
from typing import Dict, Any, List, Optional

def run_cmd(cmd: List[str]) -> Dict[str, Any]:
	start = time.time()
	p = run(cmd, stdout=PIPE, stderr=PIPE, encoding="utf-8")
	dur = time.time() - start
	ok = False
	risk = None
	mem_hit = None
	try:
		lines = p.stdout.strip().splitlines() if p.stdout else []
		# Look for the line with "status" field (not just the last line)
		obj = {}
		for line in lines:
			if line.startswith("{") and "status" in line:
				obj = json.loads(line)
				break
		ok = str(obj.get("status", "")).lower() == "ok"
		risk = float(obj.get("risk", 0.5)) if "risk" in obj else None
		if "mem_hit" in obj:
			mem_hit = bool(obj.get("mem_hit"))
	except Exception:
		ok = False
	return {"ok": bool(ok), "risk": risk, "dur": float(dur), "rc": int(p.returncode), "mem_hit": mem_hit}

def bench_cli(root: Path, runs: int, budget_sec: Optional[float] = None, cost_per_sec: Optional[float] = None) -> Dict[str, Any]:
	cli_tasks = ["count_lines", "grep_error"]
	res: Dict[str, Any] = {}
	# Aggregate across all tasks and runs
	total_success = 0
	total_success_budgeted = 0
	total_runs = 0
	total_latency = 0.0
	total_over_budget = 0
	total_cost = 0.0
	total_mem_hits = 0
	for task in cli_tasks:
		ok_n = 0
		ok_budget_n = 0
		dur_sum = 0.0
		over_budget_n = 0
		cost_sum = 0.0
		mem_hit_n = 0


	ok = False
	risk = None
	mem_hit = None
	try:
		lines = p.stdout.strip().splitlines() if p.stdout else []
		# Look for the line with "status" field (not just the last line)
		obj = {}
		for line in lines:
			if line.startswith("{") and "status" in line:
				obj = json.loads(line)
				break
		ok = str(obj.get("status", "")).lower() == "ok"
		risk = float(obj.get("risk", 0.5)) if "risk" in obj else None
		if "mem_hit" in obj:
			mem_hit = bool(obj.get("mem_hit"))
	except Exception:
		ok = False
	return {"ok": bool(ok), "risk": risk, "dur": float(dur), "rc": int(p.returncode), "mem_hit": mem_hit}

def bench_cli(root: Path, runs: int, budget_sec: Optional[float] = None, cost_per_sec: Optional[float] = None) -> Dict[str, Any]:
	cli_tasks = ["count_lines", "grep_error"]
	res: Dict[str, Any] = {}
	# Aggregate across all tasks and runs
	total_success = 0
	total_success_budgeted = 0
	total_runs = 0
	total_latency = 0.0
	total_over_budget = 0
	total_cost = 0.0
	total_mem_hits = 0
	for task in cli_tasks:
		ok_n = 0
		ok_budget_n = 0
		dur_sum = 0.0
		over_budget_n = 0
		cost_sum = 0.0
		mem_hit_n = 0
		for _ in range(runs):
			cmd = [
				"python3", str(root / "scripts" / "run_loop_oscli.py"),
				"--planner-backend", "hf",
				"--verifier-backend", "hf",
				"--planner-model", "meta-llama/Llama-3.2-3B",
				"--verifier-model", "meta-llama/Llama-3.2-3B",
				"--timeout", "20",
				"--task", task,
			]
			# Optionally enable memory usage for auditing cache/batch policy
			try:
				import os as _os
				if _os.environ.get("AGI_BENCH_USE_MEMORY_OSCLI", "0") not in ("0", "false", "False"):
					cmd.extend(["--use-memory"])
			except Exception:
				pass
			out = run_cmd(cmd)
			over = bool(budget_sec is not None and float(out["dur"]) > float(budget_sec))
			ok_n += 1 if out["ok"] else 0
			ok_budget_n += 1 if (out["ok"] and not over) else 0
			over_budget_n += 1 if over else 0
			dur_sum += float(out["dur"])
			if cost_per_sec is not None:
				cost_sum += float(out["dur"]) * float(cost_per_sec)
			if out.get("mem_hit") is True:
				mem_hit_n += 1
		# Aggregate into totals across tasks
		total_success += int(ok_n)
		total_success_budgeted += int(ok_budget_n)
		total_runs += int(runs)
		total_latency += float(dur_sum)
		total_over_budget += int(over_budget_n)
		total_cost += float(cost_sum)
		total_mem_hits += int(mem_hit_n)
		res[f"cli:{task}"] = {
			"success": ok_n,
			"success_budgeted": ok_budget_n,
			"runs": runs,
			"success_rate": (ok_n / max(1, runs)),
			"success_rate_budgeted": (ok_budget_n / max(1, runs)),
			"avg_latency_sec": (dur_sum / max(1, runs)),
			"over_budget": over_budget_n,
			"budget_sec": float(budget_sec) if budget_sec is not None else None,
			"cost_total": float(cost_sum) if cost_per_sec is not None else None,
			"cost_per_success": (float(cost_sum) / max(1, ok_n)) if cost_per_sec is not None else None,
			"memory_hit_rate": (mem_hit_n / max(1, runs)),
		}
	res["cli_summary"] = {
		"success": int(total_success),
		"success_budgeted": int(total_success_budgeted),
		"runs": int(total_runs),
		"success_rate": (float(total_success) / max(1, float(total_runs))),
		"success_rate_budgeted": (float(total_success_budgeted) / max(1, float(total_runs))),
		"avg_latency_sec": (float(total_latency) / max(1.0, float(total_runs))),
		"over_budget_total_runs": int(total_over_budget),
		"over_budget_rate": (float(total_over_budget) / max(1.0, float(total_runs))),
		"budget_sec": float(budget_sec) if budget_sec is not None else None,
		"cost_total": float(total_cost) if cost_per_sec is not None else None,
		"cost_per_success": (float(total_cost) / max(1.0, float(total_success))) if cost_per_sec is not None else None,
		"memory_hit_rate": (float(total_mem_hits) / max(1.0, float(total_runs))),
	}
	return res

def bench_dom(root: Path, runs: int, budget_sec: Optional[float] = None, cost_per_sec: Optional[float] = None) -> Dict[str, Any]:
	dom_cases = [
		{"url": "https://example.com", "selector": "h1"},
		{"url": "https://en.wikipedia.org/wiki/Alan_Turing", "selector": "#firstHeading"},
	]
	res: Dict[str, Any] = {}
	total_success = 0
	total_success_budgeted = 0
	total_runs = 0
	total_latency = 0.0
	total_over_budget = 0
	total_cost = 0.0
	total_mem_hits = 0
	for case in dom_cases:
		ok_n = 0
		ok_budget_n = 0
		dur_sum = 0.0
		over_budget_n = 0


			"cost_per_success": (float(cost_sum) / max(1, ok_n)) if cost_per_sec is not None else None,
			"memory_hit_rate": (mem_hit_n / max(1, runs)),
		}
	res["cli_summary"] = {
		"success": int(total_success),
		"success_budgeted": int(total_success_budgeted),
		"runs": int(total_runs),
		"success_rate": (float(total_success) / max(1, float(total_runs))),
		"success_rate_budgeted": (float(total_success_budgeted) / max(1, float(total_runs))),
		"avg_latency_sec": (float(total_latency) / max(1.0, float(total_runs))),
		"over_budget_total_runs": int(total_over_budget),
		"over_budget_rate": (float(total_over_budget) / max(1.0, float(total_runs))),
		"budget_sec": float(budget_sec) if budget_sec is not None else None,
		"cost_total": float(total_cost) if cost_per_sec is not None else None,
		"cost_per_success": (float(total_cost) / max(1.0, float(total_success))) if cost_per_sec is not None else None,
		"memory_hit_rate": (float(total_mem_hits) / max(1.0, float(total_runs))),
	}
	return res

def bench_dom(root: Path, runs: int, budget_sec: Optional[float] = None, cost_per_sec: Optional[float] = None) -> Dict[str, Any]:
	dom_cases = [
		{"url": "https://example.com", "selector": "h1"},
		{"url": "https://en.wikipedia.org/wiki/Alan_Turing", "selector": "#firstHeading"},
	]
	res: Dict[str, Any] = {}
	total_success = 0
	total_success_budgeted = 0
	total_runs = 0
	total_latency = 0.0
	total_over_budget = 0
	total_cost = 0.0
	total_mem_hits = 0
	for case in dom_cases:
		ok_n = 0
		ok_budget_n = 0
		dur_sum = 0.0
		over_budget_n = 0
		cost_sum = 0.0
		mem_hit_n = 0
		for _ in range(runs):
			cmd = [
				"python3", str(root / "scripts" / "run_loop_webdom.py"),
				"--planner-backend", "hf",
				"--verifier-backend", "hf",
				"--planner-model", "meta-llama/Llama-3.2-3B",
				"--verifier-model", "meta-llama/Llama-3.2-3B",
				"--timeout", "25",
				"--url", case["url"],
				"--selector", case["selector"],
			]
			# Optionally enable memory usage for auditing cache/batch policy
			try:
				import os as _os
				if _os.environ.get("AGI_BENCH_USE_MEMORY_DOM", "0") not in ("0", "false", "False"):
					cmd.extend(["--use-memory"])
			except Exception:
				pass
			out = run_cmd(cmd)
			over = bool(budget_sec is not None and float(out["dur"]) > float(budget_sec))
			ok_n += 1 if out["ok"] else 0
			ok_budget_n += 1 if (out["ok"] and not over) else 0
			over_budget_n += 1 if over else 0
			dur_sum += float(out["dur"])
			if cost_per_sec is not None:
				cost_sum += float(out["dur"]) * float(cost_per_sec)
			if out.get("mem_hit") is True:
				mem_hit_n += 1
			# Accumulate across runs for this case
		total_success += int(ok_n)
		total_success_budgeted += int(ok_budget_n)
		total_runs += int(runs)
		total_latency += float(dur_sum)
		total_over_budget += int(over_budget_n)
		total_cost += float(cost_sum)
		total_mem_hits += int(mem_hit_n)
		key = f"dom:{case['url']}#{case['selector']}"
		res[key] = {
			"success": ok_n,
			"success_budgeted": ok_budget_n,
			"runs": runs,
			"success_rate": (ok_n / max(1, runs)),
			"success_rate_budgeted": (ok_budget_n / max(1, runs)),
			"avg_latency_sec": (dur_sum / max(1, runs)),
			"over_budget": over_budget_n,
			"budget_sec": float(budget_sec) if budget_sec is not None else None,
			"cost_total": float(cost_sum) if cost_per_sec is not None else None,
			"cost_per_success": (float(cost_sum) / max(1, ok_n)) if cost_per_sec is not None else None,
			"memory_hit_rate": (mem_hit_n / max(1, runs)),
		}
	res["dom_summary"] = {
		"success": int(total_success),
		"success_budgeted": int(total_success_budgeted),
		"runs": int(total_runs),
		"success_rate": (float(total_success) / max(1.0, float(total_runs))) ,
		"success_rate_budgeted": (float(total_success_budgeted) / max(1.0, float(total_runs))) ,
		"avg_latency_sec": (float(total_latency) / max(1.0, float(total_runs))),
		"over_budget_total_runs": int(total_over_budget),
		"over_budget_rate": (float(total_over_budget) / max(1.0, float(total_runs))),
		"budget_sec": float(budget_sec) if budget_sec is not None else None,
		"cost_total": float(total_cost) if cost_per_sec is not None else None,
		"cost_per_success": (float(total_cost) / max(1.0, float(total_success))) if cost_per_sec is not None else None,
		"memory_hit_rate": (float(total_mem_hits) / max(1.0, float(total_runs))),
	}
	return res

def bench_office(root: Path, runs: int) -> Dict[str, Any]:
	"""Office/Docs micro-suite: run docs Q&A tasks and aggregate simple KPIs."""
	res: Dict[str, Any] = {}
	# Run the docs suite once to produce results JSONL
	out_jsonl = root / "data" / "benchmarks" / "docs_results.jsonl"
	try:
		p = run([
			"python3", str(root / "scripts" / "run_docs_suite.py"),
			"--tasks", str(root / "data" / "docs" / "tasks.jsonl"),
			"--docs", str(root / "data" / "docs"),
			"--out", str(out_jsonl),
		], stdout=PIPE, stderr=PIPE, encoding="utf-8")
		# proceed even if returncode != 0; we'll compute from file if present
	except Exception:
		pass
	# Read results and compute simple success metrics
	total = 0
	ok = 0


			"cost_per_success": (float(cost_sum) / max(1, ok_n)) if cost_per_sec is not None else None,
			"memory_hit_rate": (mem_hit_n / max(1, runs)),
		}
	res["dom_summary"] = {
		"success": int(total_success),
		"success_budgeted": int(total_success_budgeted),
		"runs": int(total_runs),
		"success_rate": (float(total_success) / max(1.0, float(total_runs))) ,
		"success_rate_budgeted": (float(total_success_budgeted) / max(1.0, float(total_runs))) ,
		"avg_latency_sec": (float(total_latency) / max(1.0, float(total_runs))),
		"over_budget_total_runs": int(total_over_budget),
		"over_budget_rate": (float(total_over_budget) / max(1.0, float(total_runs))),
		"budget_sec": float(budget_sec) if budget_sec is not None else None,
		"cost_total": float(total_cost) if cost_per_sec is not None else None,
		"cost_per_success": (float(total_cost) / max(1.0, float(total_success))) if cost_per_sec is not None else None,
		"memory_hit_rate": (float(total_mem_hits) / max(1.0, float(total_runs))),
	}
	return res

def bench_office(root: Path, runs: int) -> Dict[str, Any]:
	"""Office/Docs micro-suite: run docs Q&A tasks and aggregate simple KPIs."""
	res: Dict[str, Any] = {}
	# Run the docs suite once to produce results JSONL
	out_jsonl = root / "data" / "benchmarks" / "docs_results.jsonl"
	try:
		p = run([
			"python3", str(root / "scripts" / "run_docs_suite.py"),
			"--tasks", str(root / "data" / "docs" / "tasks.jsonl"),
			"--docs", str(root / "data" / "docs"),
			"--out", str(out_jsonl),
		], stdout=PIPE, stderr=PIPE, encoding="utf-8")
		# proceed even if returncode != 0; we'll compute from file if present
	except Exception:
		pass
	# Read results and compute simple success metrics
	total = 0
	ok = 0
	try:
		if out_jsonl.exists():
			with out_jsonl.open("r", encoding="utf-8") as f:
				for line in f:
					line = line.strip()
					if not line:
						continue
					try:
						rec = json.loads(line)
					except Exception:
						continue
					total += 1
					ok += 1 if bool(rec.get("ok", False)) else 0
	except Exception:
		pass
	res["office:docs_suite"] = {
		"success": int(ok),
		"runs": int(max(1, total)),
		"success_rate": (float(ok) / max(1.0, float(total))),
	}
	res["office_summary"] = {
		"success": int(ok),
		"runs": int(max(1, total)),
		"success_rate": (float(ok) / max(1.0, float(total))),
	}
	return res

def main() -> int:
	ap = argparse.ArgumentParser()
	root = Path(__file__).resolve().parents[1]
	ap.add_argument("--runs", type=int, default=2)
	ap.add_argument("--budget-cli-sec", type=float, default=None, help="Optional per-run latency budget (seconds) for CLI tasks")
	ap.add_argument("--budget-dom-sec", type=float, default=None, help="Optional per-run latency budget (seconds) for DOM tasks")
	ap.add_argument("--cost-cli-per-sec", type=float, default=None, help="Optional estimated cost rate ($/sec) for CLI loop")
	ap.add_argument("--cost-dom-per-sec", type=float, default=None, help="Optional estimated cost rate ($/sec) for DOM loop")
	ap.add_argument("--include-practice", action="store_true", help="If set, run coding practice suite before aggregating dashboard")
	ap.add_argument("--practice-tier", default="T1")
	ap.add_argument("--practice-out", default=str(root / "data" / "logs" / "practice_results.jsonl"))
	ap.add_argument("--domains", default="cli,dom", help="Comma-separated domains to include: cli,dom,office")
	ap.add_argument("--out", default=str(root / "data" / "benchmarks" / "kpi.json"))
	args = ap.parse_args()

	root.mkdir(parents=True, exist_ok=True)
	out_dir = Path(args.out).parent
	out_dir.mkdir(parents=True, exist_ok=True)


						rec = json.loads(line)
					except Exception:
						continue
					total += 1
					ok += 1 if bool(rec.get("ok", False)) else 0
	except Exception:
		pass
	res["office:docs_suite"] = {
		"success": int(ok),
		"runs": int(max(1, total)),
		"success_rate": (float(ok) / max(1.0, float(total))),
	}
	res["office_summary"] = {
		"success": int(ok),
		"runs": int(max(1, total)),
		"success_rate": (float(ok) / max(1.0, float(total))),
	}
	return res

def main() -> int:
	ap = argparse.ArgumentParser()
	root = Path(__file__).resolve().parents[1]
	ap.add_argument("--runs", type=int, default=2)
	ap.add_argument("--budget-cli-sec", type=float, default=None, help="Optional per-run latency budget (seconds) for CLI tasks")
	ap.add_argument("--budget-dom-sec", type=float, default=None, help="Optional per-run latency budget (seconds) for DOM tasks")
	ap.add_argument("--cost-cli-per-sec", type=float, default=None, help="Optional estimated cost rate ($/sec) for CLI loop")
	ap.add_argument("--cost-dom-per-sec", type=float, default=None, help="Optional estimated cost rate ($/sec) for DOM loop")
	ap.add_argument("--include-practice", action="store_true", help="If set, run coding practice suite before aggregating dashboard")
	ap.add_argument("--practice-tier", default="T1")
	ap.add_argument("--practice-out", default=str(root / "data" / "logs" / "practice_results.jsonl"))
	ap.add_argument("--domains", default="cli,dom", help="Comma-separated domains to include: cli,dom,office")
	ap.add_argument("--out", default=str(root / "data" / "benchmarks" / "kpi.json"))
	args = ap.parse_args()

	root.mkdir(parents=True, exist_ok=True)
	out_dir = Path(args.out).parent
	out_dir.mkdir(parents=True, exist_ok=True)

	kpis: Dict[str, Any] = {}
	requested = [d.strip() for d in str(args.domains).split(",") if d.strip()]
	if "cli" in requested:
		kpis.update(bench_cli(
			root,
			max(1, int(args.runs)),
			float(args.budget_cli_sec) if args.budget_cli_sec is not None else None,
			float(args.cost_cli_per_sec) if args.cost_cli_per_sec is not None else None,
		))
	if "dom" in requested:
		kpis.update(bench_dom(
			root,
			max(1, int(args.runs)),
			float(args.budget_dom_sec) if args.budget_dom_sec is not None else None,
			float(args.cost_dom_per_sec) if args.cost_dom_per_sec is not None else None,
		))
	if "office" in requested:
		kpis.update(bench_office(root, max(1, int(args.runs))))
	kpis["meta"] = {
		"runs_per_case": int(args.runs),
		"ts": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
		"budget_cli_sec": (float(args.budget_cli_sec) if args.budget_cli_sec is not None else None),
		"budget_dom_sec": (float(args.budget_dom_sec) if args.budget_dom_sec is not None else None),
		"cost_cli_per_sec": (float(args.cost_cli_per_sec) if args.cost_cli_per_sec is not None else None),
		"cost_dom_per_sec": (float(args.cost_dom_per_sec) if args.cost_dom_per_sec is not None else None),
	}

	Path(args.out).write_text(json.dumps(kpis, ensure_ascii=False, indent=2), encoding="utf-8")
	# Optionally run coding practice suite (tiny repos) to produce practice logs consumed by dashboard aggregator
	if args.include_practice:
		try:
			cmd = [
				"python3", str(root / "scripts" / "run_practice_suite.py"),
				"--tier", str(args.practice_tier),
				"--out", str(args.practice_out),
			]
			_ = run(cmd, stdout=PIPE, stderr=PIPE, encoding="utf-8")  # type: ignore
		except Exception:
			pass
	print(json.dumps({"ok": True, "out": str(args.out)}))
	return 0

if __name__ == "__main__":
	raise SystemExit(main())


import logging
import argparse
import json
from pathlib import Path

def main() -> int:
	ap = argparse.ArgumentParser()
	root = Path(__file__).resolve().parents[1]
	ap.add_argument("--domain", choices=["cli", "dom"], default="cli")
	ap.add_argument("--wm", default=str(root / "models" / "wm_mlp" / "wm_mlp.joblib"))
	ap.add_argument("--il", default=str(root / "data" / "skills" / "actuator_il.jsonl"))
	ap.add_argument("--t5", default=str(root / "models" / "actuator_il_t5"))
	ap.add_argument("--obs", required=True)
	ap.add_argument("--plan", required=True)
	args = ap.parse_args()

	obs = json.loads(args.obs)
	plan = json.loads(args.plan)

	from agi_dw.core.world_model.api import WorldModelPrior  # type: ignore
	from agi_dw.core.world_model.offpolicy import OffPolicyProposer  # type: ignore

	wm = WorldModelPrior.load(args.wm)
	proposer = OffPolicyProposer(wm)
	if args.domain == "cli":
		res = proposer.propose_cli(obs, plan, args.il, args.t5)
	else:
		res = proposer.propose_dom(obs, plan, args.il, args.t5)
	print(json.dumps(res, ensure_ascii=False))
	return 0

if __name__ == "__main__":
	raise SystemExit(main())


import logging
import argparse
import json

from agi_dw.bench.web_dom.runner import fetch_text

def main() -> None:
	parser = argparse.ArgumentParser()
	parser.add_argument("--url", required=True)
	parser.add_argument("--selector", default="body")
	args = parser.parse_args()
	res = fetch_text(args.url, args.selector)
	print(json.dumps(res, ensure_ascii=False))

if __name__ == "__main__":
	main()


import logging
import argparse
import json
from pathlib import Path

def main() -> int:
	ap = argparse.ArgumentParser()
	root = Path(__file__).resolve().parents[1]
	ap.add_argument("--skill-id", required=True)
	ap.add_argument("--verifier-adapter", default=None)
	ap.add_argument("--planner-adapter", default=None)
	args = ap.parse_args()

	from agi_dw.core.memory.skills import SkillLibrary  # type: ignore

	lib = SkillLibrary(str(root))
	adapters = {}
	if args.verifier_adapter:
		adapters["verifier"] = str(args.verifier_adapter)
	if args.planner_adapter:
		adapters["planner"] = str(args.planner_adapter)
	if not adapters:
		print(json.dumps({"updated": False, "error": "no adapters specified"}))
		return 1
	ok = lib.update_adapters(args.skill_id, adapters)
	lib.save_registry()
	print(json.dumps({"updated": bool(ok), "skill_id": args.skill_id, "adapters": adapters}, ensure_ascii=False))
	return 0

if __name__ == "__main__":
	import sys
	sys.exit(main())


import logging
import argparse
import json
import time
from pathlib import Path
from subprocess import run, PIPE  # type: ignore

def _verify_cmd(root: Path, src: Path, dst: Path, backend: str, batch_size: int) -> list[str]:
	cmd = [
		"python3", str(root / "scripts" / "verify_traces.py"),
		str(src),
		str(dst),
		"--backend", backend,
		"--require-llm",
		"--structured", "json",
		"--warmup",
		"--workers", "1",
		"--batch-size", str(int(batch_size)),
		"--max", "20",
	]
	return cmd

def _timed_call(cmd: list[str]) -> tuple[int, float]:
	start = time.time()
	p = run(cmd, stdout=PIPE, stderr=PIPE, encoding="utf-8")
	dur = time.time() - start
	return (p.returncode, float(dur))

def main() -> int:
	ap = argparse.ArgumentParser()
	root = Path(__file__).resolve().parents[1]
	ap.add_argument("--src", default=str(root / "data" / "traces" / "seed_os_cli.jsonl"))
	ap.add_argument("--out", default=str(root / "data" / "benchmarks" / "batch_audit.json"))
	ap.add_argument("--backend", choices=["hf"], default="hf")
	ap.add_argument("--batch-size", type=int, default=8)
	args = ap.parse_args()

	src = Path(args.src)
	outp = Path(args.out)
	outp.parent.mkdir(parents=True, exist_ok=True)

	non_batch_dst = src.with_suffix(".verified.nobatch.jsonl")
	batch_dst = src.with_suffix(".verified.batch.jsonl")

	# Non-batch run
	rc_nb, dur_nb = _timed_call(_verify_cmd(root, src, non_batch_dst, args.backend, 0))
	# Batched run (only effective for HF backend)
	rc_b, dur_b = _timed_call(_verify_cmd(root, src, batch_dst, args.backend, max(1, int(args.batch_size))))

	# Compute simple metrics
	def _count_lines(p: Path) -> int:
		try:
			with p.open("r", encoding="utf-8") as f:
				return sum(1 for _ in f if _.strip())
		except Exception:
			return 0

	n_nb = _count_lines(non_batch_dst)
	n_b = _count_lines(batch_dst)
	speedup = (dur_nb / dur_b) if (dur_b > 1e-6) else 0.0

	res = {
		"backend": args.backend,
		"items_non_batch": int(n_nb),
		"items_batch": int(n_b),
		"dur_non_batch_sec": float(round(dur_nb, 3)),
		"dur_batch_sec": float(round(dur_b, 3)),
		"speedup": float(round(speedup, 3)),
		"rc_non_batch": int(rc_nb),
		"rc_batch": int(rc_b),
	}
	outp.write_text(json.dumps(res, ensure_ascii=False, indent=2), encoding="utf-8")
	print(json.dumps({"ok": True, "out": str(outp), "speedup": res["speedup"]}))
	return 0

if __name__ == "__main__":
	import sys
	sys.exit(main())


import logging
import argparse
import json
import time
from pathlib import Path
from subprocess import run, PIPE  # type: ignore

def _verify_cmd(root: Path, src: Path, dst: Path, backend: str, batch_size: int) -> list[str]:
	cmd = [
		"python3", str(root / "scripts" / "verify_traces.py"),
		str(src),
		str(dst),
		"--backend", backend,
		"--require-llm",
		"--structured", "json",
		"--warmup",
		"--workers", "1",
		"--batch-size", str(int(batch_size)),
		"--max", "20",
	]
	return cmd

def _timed_call(cmd: list[str]) -> tuple[int, float]:
	start = time.time()
	p = run(cmd, stdout=PIPE, stderr=PIPE, encoding="utf-8")
	dur = time.time() - start
	return (p.returncode, float(dur))

def main() -> int:
	ap = argparse.ArgumentParser()
	root = Path(__file__).resolve().parents[1]
	ap.add_argument("--src", default=str(root / "data" / "traces" / "seed_os_cli.jsonl"))
	ap.add_argument("--out", default=str(root / "data" / "benchmarks" / "batch_audit.json"))
	ap.add_argument("--backend", choices=["hf"], default="hf")
	ap.add_argument("--batch-size", type=int, default=8)
	args = ap.parse_args()

	src = Path(args.src)
	outp = Path(args.out)


from pathlib import Path
from subprocess import run, PIPE  # type: ignore

def _verify_cmd(root: Path, src: Path, dst: Path, backend: str, batch_size: int) -> list[str]:
	cmd = [
		"python3", str(root / "scripts" / "verify_traces.py"),
		str(src),
		str(dst),
		"--backend", backend,
		"--require-llm",
		"--structured", "json",
		"--warmup",
		"--workers", "1",
		"--batch-size", str(int(batch_size)),
		"--max", "20",
	]
	return cmd

def _timed_call(cmd: list[str]) -> tuple[int, float]:
	start = time.time()
	p = run(cmd, stdout=PIPE, stderr=PIPE, encoding="utf-8")
	dur = time.time() - start
	return (p.returncode, float(dur))

def main() -> int:
	ap = argparse.ArgumentParser()
	root = Path(__file__).resolve().parents[1]
	ap.add_argument("--src", default=str(root / "data" / "traces" / "seed_os_cli.jsonl"))
	ap.add_argument("--out", default=str(root / "data" / "benchmarks" / "batch_audit.json"))
	ap.add_argument("--backend", choices=["hf"], default="hf")
	ap.add_argument("--batch-size", type=int, default=8)
	args = ap.parse_args()

	src = Path(args.src)
	outp = Path(args.out)
	outp.parent.mkdir(parents=True, exist_ok=True)

	non_batch_dst = src.with_suffix(".verified.nobatch.jsonl")
	batch_dst = src.with_suffix(".verified.batch.jsonl")

	# Non-batch run
	rc_nb, dur_nb = _timed_call(_verify_cmd(root, src, non_batch_dst, args.backend, 0))


		str(src),
		str(dst),
		"--backend", backend,
		"--require-llm",
		"--structured", "json",
		"--warmup",
		"--workers", "1",
		"--batch-size", str(int(batch_size)),
		"--max", "20",
	]
	return cmd

def _timed_call(cmd: list[str]) -> tuple[int, float]:
	start = time.time()
	p = run(cmd, stdout=PIPE, stderr=PIPE, encoding="utf-8")
	dur = time.time() - start
	return (p.returncode, float(dur))

def main() -> int:
	ap = argparse.ArgumentParser()
	root = Path(__file__).resolve().parents[1]
	ap.add_argument("--src", default=str(root / "data" / "traces" / "seed_os_cli.jsonl"))
	ap.add_argument("--out", default=str(root / "data" / "benchmarks" / "batch_audit.json"))
	ap.add_argument("--backend", choices=["hf"], default="hf")
	ap.add_argument("--batch-size", type=int, default=8)
	args = ap.parse_args()

	src = Path(args.src)
	outp = Path(args.out)
	outp.parent.mkdir(parents=True, exist_ok=True)

	non_batch_dst = src.with_suffix(".verified.nobatch.jsonl")
	batch_dst = src.with_suffix(".verified.batch.jsonl")

	# Non-batch run
	rc_nb, dur_nb = _timed_call(_verify_cmd(root, src, non_batch_dst, args.backend, 0))
	# Batched run (only effective for HF backend)
	rc_b, dur_b = _timed_call(_verify_cmd(root, src, batch_dst, args.backend, max(1, int(args.batch_size))))

	# Compute simple metrics
	def _count_lines(p: Path) -> int:
		try:
			with p.open("r", encoding="utf-8") as f:
				return sum(1 for _ in f if _.strip())
		except Exception:
			return 0

	n_nb = _count_lines(non_batch_dst)
	n_b = _count_lines(batch_dst)
	speedup = (dur_nb / dur_b) if (dur_b > 1e-6) else 0.0

	res = {
		"backend": args.backend,
		"items_non_batch": int(n_nb),
		"items_batch": int(n_b),
		"dur_non_batch_sec": float(round(dur_nb, 3)),
		"dur_batch_sec": float(round(dur_b, 3)),
		"speedup": float(round(speedup, 3)),
		"rc_non_batch": int(rc_nb),
		"rc_batch": int(rc_b),
	}
	outp.write_text(json.dumps(res, ensure_ascii=False, indent=2), encoding="utf-8")
	print(json.dumps({"ok": True, "out": str(outp), "speedup": res["speedup"]}))
	return 0

if __name__ == "__main__":
	import sys
	sys.exit(main())


	root = Path(__file__).resolve().parents[1]
	ap.add_argument("--src", default=str(root / "data" / "traces" / "seed_os_cli.jsonl"))
	ap.add_argument("--out", default=str(root / "data" / "benchmarks" / "batch_audit.json"))
	ap.add_argument("--backend", choices=["hf"], default="hf")
	ap.add_argument("--batch-size", type=int, default=8)
	args = ap.parse_args()

	src = Path(args.src)
	outp = Path(args.out)
	outp.parent.mkdir(parents=True, exist_ok=True)

	non_batch_dst = src.with_suffix(".verified.nobatch.jsonl")
	batch_dst = src.with_suffix(".verified.batch.jsonl")

	# Non-batch run
	rc_nb, dur_nb = _timed_call(_verify_cmd(root, src, non_batch_dst, args.backend, 0))
	# Batched run (only effective for HF backend)
	rc_b, dur_b = _timed_call(_verify_cmd(root, src, batch_dst, args.backend, max(1, int(args.batch_size))))

	# Compute simple metrics
	def _count_lines(p: Path) -> int:
		try:
			with p.open("r", encoding="utf-8") as f:
				return sum(1 for _ in f if _.strip())
		except Exception:
			return 0

	n_nb = _count_lines(non_batch_dst)
	n_b = _count_lines(batch_dst)
	speedup = (dur_nb / dur_b) if (dur_b > 1e-6) else 0.0

	res = {
		"backend": args.backend,
		"items_non_batch": int(n_nb),
		"items_batch": int(n_b),
		"dur_non_batch_sec": float(round(dur_nb, 3)),
		"dur_batch_sec": float(round(dur_b, 3)),
		"speedup": float(round(speedup, 3)),
		"rc_non_batch": int(rc_nb),
		"rc_batch": int(rc_b),
	}
	outp.write_text(json.dumps(res, ensure_ascii=False, indent=2), encoding="utf-8")
	print(json.dumps({"ok": True, "out": str(outp), "speedup": res["speedup"]}))
	return 0


import logging
import argparse
import json
from pathlib import Path
from typing import List, Dict, Any

def main() -> int:
	ap = argparse.ArgumentParser()
	root = Path(__file__).resolve().parents[1]
	ap.add_argument("--include-cli", action="store_true")
	ap.add_argument("--include-dom", action="store_true")
	ap.add_argument("--out", default=str(root / "data" / "benchmarks" / "external_suite.jsonl"))
	args = ap.parse_args()

	recs: List[Dict[str, Any]] = []
	if args.include_cli:
		recs.append({"domain": "cli", "task": {"name": "count_lines", "timeout": 20}})
		recs.append({"domain": "cli", "task": {"name": "grep_error", "timeout": 20}})
	if args.include_dom:
		# Pull from YAML seeds if available
		seeds = root / "data" / "dom_seeds.yaml"
		if seeds.exists():
			try:
				import yaml  # type: ignore
				obj = yaml.safe_load(seeds.read_text(encoding="utf-8"))
				if isinstance(obj, list):
					for row in obj[:10]:  # cap for sanity
						if not isinstance(row, dict):
							continue
						url = str(row.get("url", "")).strip()
						selector = row.get("selector")
						selectors = row.get("selectors")
						if not url:
							continue
						if isinstance(selectors, list) and len(selectors) > 0:
							sel = str(selectors[0])
						else:
							sel = str(selector or "h1")
						recs.append({"domain": "dom", "task": {"url": url, "selector": sel, "timeout": 25}})
			except Exception:
				pass
		# Fallback stable
		if not any(r.get("domain") == "dom" for r in recs):
			recs.append({"domain": "dom", "task": {"url": "https://en.wikipedia.org/wiki/Alan_Turing", "selector": "#firstHeading", "timeout": 25}})

	out = Path(args.out)
	out.parent.mkdir(parents=True, exist_ok=True)
	with out.open("w", encoding="utf-8") as f:
		for r in recs:
			f.write(json.dumps(r, ensure_ascii=False) + "\n")
	print(json.dumps({"ok": True, "out": str(out), "n": len(recs)}))
	return 0

if __name__ == "__main__":
	import sys
	sys.exit(main())


#!/usr/bin/env python3
import logging
import argparse
import json
import re
from pathlib import Path
from typing import List, Dict, Any

def parse_args():
	root = Path(__file__).resolve().parents[1]
	ap = argparse.ArgumentParser(description="Simple contamination checker for code samples")
	ap.add_argument("--samples", nargs="*", default=[
		str(root / "data" / "llm_bench" / "humaneval_samples.jsonl"),
		str(root / "data" / "llm_bench" / "mbpp_samples.jsonl"),
		str(root / "data" / "llm_bench" / "apps_samples.jsonl"),
		str(root / "data" / "llm_bench" / "ds1000_samples.jsonl"),
	])
	ap.add_argument("--corpus", default=str(root / "data" / "corpora" / "train_corpus.txt"), help="Plain text file to scan against")
	ap.add_argument("--out", default=str(root / "data" / "llm_bench" / "contamination.json"))
	return ap.parse_args()

def load_samples(paths: List[str]) -> List[Dict[str, Any]]:
	recs: List[Dict[str, Any]] = []
	for p in paths:
		pp = Path(p)
		if not pp.exists():
			continue
		for line in pp.read_text(encoding="utf-8").splitlines():
			line = line.strip()
			if not line:
				continue
			try:
				obj = json.loads(line)
				recs.append(obj)
			except Exception:
				continue
	return recs

def main() -> int:
	args = parse_args()
	corpus_path = Path(args.corpus)
	corpus_text = corpus_path.read_text(encoding="utf-8") if corpus_path.exists() else ""
	re =cs = load_samples(args.samples)
	flags = 0
	checked = 0
	matches: List[Dict[str, Any]] = []
	for r in re=cs:
		comp = str(r.get("completion", ""))
		if not comp:
			continue
		checked += 1
		# Simple exact/regex heuristics
		snippet = comp[:1600]
		if snippet and snippet in corpus_text:
			flags += 1
			matches.append({"reason": "substring", "len": len(snippet), "task_id": r.get("task_id")})
			continue
		# Regex for function definitions
		try:
			pat = r"def\s+[a-zA-Z_]\w*\s*\("
			for m in re.finditer(pat, comp):
				frag = comp[m.start():m.start()+120]
				if frag in corpus_text:
					flags += 1
					matches.append({"reason": "regex_funcdef", "task_id": r.get("task_id")})
					break
		except Exception:
			pass
	out = {
		"samples": checked,
		"flags": flags,
		"rate": (float(flags) / float(checked)) if checked else 0.0,
		"matches": matches[:50],
	}
	op = Path(args.out)
	op.parent.mkdir(parents=True, exist_ok=True)
	op.write_text(json.dumps(out, ensure_ascii=False, indent=2), encoding="utf-8")
	print(json.dumps({"ok": True, "wrote": str(op), "flags": flags, "checked": checked}))
	return 0

if __name__ == "__main__":
	raise SystemExit(main())