init

		args.append(patch_path)
		return self._run(*args)

	def apply_reverse_patch(self, patch_path: str) -> subprocess.CompletedProcess:
		"""Reverse apply (undo) a patch: git apply -R <patch>."""
		return self._run("apply", "-R", patch_path)

	def reset_hard(self, ref: str = "HEAD") -> subprocess.CompletedProcess:
		"""Reset index and working tree to ref, discarding changes."""
		return self._run("reset", "--hard", ref)

	def clean(self, directories: bool = True, force: bool = True) -> subprocess.CompletedProcess:
		"""Clean untracked files. By default remove untracked files and directories (git clean -fd)."""
		args = ["clean"]
		if directories:
			args.append("-d")
		if force:
			args.append("-f")
		return self._run(*args)

	def init(self) -> subprocess.CompletedProcess:
		"""Initialize a git repository."""
		return self._run("init")

	def get_current_branch(self) -> subprocess.CompletedProcess:
		"""Get the current branch name."""
		return self._run("branch", "--show-current")

	def checkout_branch(self, branch: str, create: bool = False) -> subprocess.CompletedProcess:
		"""Checkout a branch, optionally creating it."""
		args = ["checkout"]
		if create:
			args.extend(["-b", branch])
		else:
			args.append(branch)
		return self._run(*args)


		"""Reverse apply (undo) a patch: git apply -R <patch>."""
		return self._run("apply", "-R", patch_path)

	def reset_hard(self, ref: str = "HEAD") -> subprocess.CompletedProcess:
		"""Reset index and working tree to ref, discarding changes."""
		return self._run("reset", "--hard", ref)

	def clean(self, directories: bool = True, force: bool = True) -> subprocess.CompletedProcess:
		"""Clean untracked files. By default remove untracked files and directories (git clean -fd)."""
		args = ["clean"]
		if directories:
			args.append("-d")
		if force:
			args.append("-f")
		return self._run(*args)

	def init(self) -> subprocess.CompletedProcess:
		"""Initialize a git repository."""
		return self._run("init")

	def get_current_branch(self) -> subprocess.CompletedProcess:
		"""Get the current branch name."""
		return self._run("branch", "--show-current")

	def checkout_branch(self, branch: str, create: bool = False) -> subprocess.CompletedProcess:
		"""Checkout a branch, optionally creating it."""
		args = ["checkout"]
		if create:
			args.extend(["-b", branch])
		else:
			args.append(branch)
		return self._run(*args)


		"""Reset index and working tree to ref, discarding changes."""
		return self._run("reset", "--hard", ref)

	def clean(self, directories: bool = True, force: bool = True) -> subprocess.CompletedProcess:
		"""Clean untracked files. By default remove untracked files and directories (git clean -fd)."""
		args = ["clean"]
		if directories:
			args.append("-d")
		if force:
			args.append("-f")
		return self._run(*args)

	def init(self) -> subprocess.CompletedProcess:
		"""Initialize a git repository."""
		return self._run("init")

	def get_current_branch(self) -> subprocess.CompletedProcess:
		"""Get the current branch name."""
		return self._run("branch", "--show-current")

	def checkout_branch(self, branch: str, create: bool = False) -> subprocess.CompletedProcess:
		"""Checkout a branch, optionally creating it."""
		args = ["checkout"]
		if create:
			args.extend(["-b", branch])
		else:
			args.append(branch)
		return self._run(*args)


from __future__ import annotations
import logging

import subprocess
from pathlib import Path
from typing import Dict, Any, List, Optional

def _git(cwd: str | Path, *args: str) -> tuple[int, str]:
	cmd = ["git"] + list(args)
	p = subprocess.Popen(cmd, cwd=str(cwd), stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True)
	out, _ = p.communicate()
	return p.returncode, out or ""

def clone(repo_url: str, dest_dir: str | Path) -> Dict[str, Any]:
	code, out = _git(Path("."), "clone", "--depth", "1", repo_url, str(dest_dir))
	return {"cmd": "git clone", "code": code, "output": out}

def checkout(repo_dir: str | Path, ref: str) -> Dict[str, Any]:
	code, out = _git(repo_dir, "checkout", "-B", ref)
	return {"cmd": "git checkout", "code": code, "output": out}

def current_branch(repo_dir: str | Path) -> str:
	code, out = _git(repo_dir, "rev-parse", "--abbrev-ref", "HEAD")
	return (out or "").strip()

def diff(repo_dir: str | Path) -> str:
	code, out = _git(repo_dir, "diff")
	return out

def commit_all(repo_dir: str | Path, message: str) -> Dict[str, Any]:
	_git(repo_dir, "add", "-A")
	code, out = _git(repo_dir, "commit", "-m", message)
	return {"cmd": "git commit", "code": code, "output": out}


from __future__ import annotations
import logging

import subprocess
from pathlib import Path
from typing import Dict, Any, List, Optional

def _git(cwd: str | Path, *args: str) -> tuple[int, str]:
	cmd = ["git"] + list(args)
	p = subprocess.Popen(cmd, cwd=str(cwd), stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True)
	out, _ = p.communicate()
	return p.returncode, out or ""

def clone(repo_url: str, dest_dir: str | Path) -> Dict[str, Any]:
	code, out = _git(Path("."), "clone", "--depth", "1", repo_url, str(dest_dir))
	return {"cmd": "git clone", "code": code, "output": out}

def checkout(repo_dir: str | Path, ref: str) -> Dict[str, Any]:
	code, out = _git(repo_dir, "checkout", "-B", ref)
	return {"cmd": "git checkout", "code": code, "output": out}

def current_branch(repo_dir: str | Path) -> str:
	code, out = _git(repo_dir, "rev-parse", "--abbrev-ref", "HEAD")
	return (out or "").strip()

def diff(repo_dir: str | Path) -> str:
	code, out = _git(repo_dir, "diff")
	return out


clone

from __future__ import annotations
import logging

import subprocess
from pathlib import Path
from typing import Dict, Any, List, Optional

def _git(cwd: str | Path, *args: str) -> tuple[int, str]:
	cmd = ["git"] + list(args)
	p = subprocess.Popen(cmd, cwd=str(cwd), stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True)
	out, _ = p.communicate()
	return p.returncode, out or ""

def clone(repo_url: str, dest_dir: str | Path) -> Dict[str, Any]:
	code, out = _git(Path("."), "clone", "--depth", "1", repo_url, str(dest_dir))
	return {"cmd": "git clone", "code": code, "output": out}

def checkout(repo_dir: str | Path, ref: str) -> Dict[str, Any]:
	code, out = _git(repo_dir, "checkout", "-B", ref)
	return {"cmd": "git checkout", "code": code, "output": out}

def current_branch(repo_dir: str | Path) -> str:
	code, out = _git(repo_dir, "rev-parse", "--abbrev-ref", "HEAD")
	return (out or "").strip()

def diff(repo_dir: str | Path) -> str:
	code, out = _git(repo_dir, "diff")
	return out

def commit_all(repo_dir: str | Path, message: str) -> Dict[str, Any]:
	_git(repo_dir, "add", "-A")
	code, out = _git(repo_dir, "commit", "-m", message)


from typing import Dict, Any, List, Optional

def _git(cwd: str | Path, *args: str) -> tuple[int, str]:
	cmd = ["git"] + list(args)
	p = subprocess.Popen(cmd, cwd=str(cwd), stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True)
	out, _ = p.communicate()
	return p.returncode, out or ""

def clone(repo_url: str, dest_dir: str | Path) -> Dict[str, Any]:
	code, out = _git(Path("."), "clone", "--depth", "1", repo_url, str(dest_dir))
	return {"cmd": "git clone", "code": code, "output": out}

def checkout(repo_dir: str | Path, ref: str) -> Dict[str, Any]:
	code, out = _git(repo_dir, "checkout", "-B", ref)
	return {"cmd": "git checkout", "code": code, "output": out}

def current_branch(repo_dir: str | Path) -> str:
	code, out = _git(repo_dir, "rev-parse", "--abbrev-ref", "HEAD")
	return (out or "").strip()

def diff(repo_dir: str | Path) -> str:
	code, out = _git(repo_dir, "diff")
	return out

def commit_all(repo_dir: str | Path, message: str) -> Dict[str, Any]:
	_git(repo_dir, "add", "-A")
	code, out = _git(repo_dir, "commit", "-m", message)
	return {"cmd": "git commit", "code": code, "output": out}


diff

	p = subprocess.Popen(cmd, cwd=str(cwd), stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True)
	out, _ = p.communicate()
	return p.returncode, out or ""

def clone(repo_url: str, dest_dir: str | Path) -> Dict[str, Any]:
	code, out = _git(Path("."), "clone", "--depth", "1", repo_url, str(dest_dir))
	return {"cmd": "git clone", "code": code, "output": out}

def checkout(repo_dir: str | Path, ref: str) -> Dict[str, Any]:
	code, out = _git(repo_dir, "checkout", "-B", ref)
	return {"cmd": "git checkout", "code": code, "output": out}

def current_branch(repo_dir: str | Path) -> str:
	code, out = _git(repo_dir, "rev-parse", "--abbrev-ref", "HEAD")
	return (out or "").strip()

def diff(repo_dir: str | Path) -> str:
	code, out = _git(repo_dir, "diff")
	return out

def commit_all(repo_dir: str | Path, message: str) -> Dict[str, Any]:
	_git(repo_dir, "add", "-A")
	code, out = _git(repo_dir, "commit", "-m", message)
	return {"cmd": "git commit", "code": code, "output": out}


def clone(repo_url: str, dest_dir: str | Path) -> Dict[str, Any]:
	code, out = _git(Path("."), "clone", "--depth", "1", repo_url, str(dest_dir))
	return {"cmd": "git clone", "code": code, "output": out}

def checkout(repo_dir: str | Path, ref: str) -> Dict[str, Any]:
	code, out = _git(repo_dir, "checkout", "-B", ref)
	return {"cmd": "git checkout", "code": code, "output": out}

def current_branch(repo_dir: str | Path) -> str:
	code, out = _git(repo_dir, "rev-parse", "--abbrev-ref", "HEAD")
	return (out or "").strip()

def diff(repo_dir: str | Path) -> str:
	code, out = _git(repo_dir, "diff")
	return out

def commit_all(repo_dir: str | Path, message: str) -> Dict[str, Any]:
	_git(repo_dir, "add", "-A")
	code, out = _git(repo_dir, "commit", "-m", message)
	return {"cmd": "git commit", "code": code, "output": out}


from __future__ import annotations
import logging

import json
from typing import Dict, Any, List

def rank_index_candidates(observation: Dict[str, Any], index_obj: Dict[str, Any], k: int) -> Dict[str, List[Dict[str, str]]]:
	"""Return top-k function/class candidates from a Python code index based on observation text.

	Scoring heuristic favors substring matches of symbol names in observation content/meta,
	with a slight bias toward shorter names.
	"""
	# Return empty when disabled or invalid inputs
	if not isinstance(observation, dict) or int(k) <= 0:
		return {}
	if not isinstance(index_obj, dict) or not index_obj.get("functions") and not index_obj.get("classes"):
		return {}
	text = (str(observation.get("content", "")) + " \n " + json.dumps(observation.get("meta", {}), ensure_ascii=False))
	text_lower = text.lower()
	func_hits: List[tuple[float, str, str]] = []
	class_hits: List[tuple[float, str, str]] = []
	try:
		for fp, funcs in (index_obj.get("functions") or {}).items():
			for f in (funcs or []):
				name = str(f.get("name", ""))
				if not name:
					continue
				s = (1.0 if (name.lower() in text_lower) else 0.0) + max(0.0, 0.5 - min(0.5, len(name) / 40.0))
				func_hits.append((s, name, fp))
		for fp, classes in (index_obj.get("classes") or {}).items():
			for c in (classes or []):
				name = str(c.get("name", ""))
				if not name:
					continue
				s = (1.0 if (name.lower() in text_lower) else 0.0) + max(0.0, 0.5 - min(0.5, len(name) / 40.0))
				class_hits.append((s, name, fp))
	except Exception:
		func_hits = []
		class_hits = []
	func_hits.sort(key=lambda x: (-x[0], x[1]))
	class_hits.sort(key=lambda x: (-x[0], x[1]))
	top_funcs = [{"name": n, "file": p} for _, n, p in func_hits[: int(k)]]
	top_classes = [{"name": n, "file": p} for _, n, p in class_hits[: int(k)]]
	return {"top_functions": top_funcs, "top_classes": top_classes}


#!/usr/bin/env python3
from __future__ import annotations
import logging

import json
from pathlib import Path
from typing import Any, Dict

def main() -> int:
    root = Path(__file__).resolve().parents[2]
    out = root / "data" / "sandbox" / "tmp" / "ir_snapshot.json"
    # Build on top of existing code index graph for now
    try:
        from agi_dw.tools.code_index import build_index  # type: ignore
        idx = build_index(root)
    except Exception:
        idx = {}
    ir = {
        "ok": True,
        "uast": {},  # placeholder for future multi-language UAST
        "ssa": {},   # placeholder
        "pdg": {},   # placeholder
        "code_graph": (idx.get("graph") if isinstance(idx, dict) else {}),
        "out": str(out),
    }
    out.parent.mkdir(parents=True, exist_ok=True)
    out.write_text(json.dumps(ir, ensure_ascii=False, indent=2), encoding="utf-8")
    print(json.dumps({"ok": True, "out": str(out)}))
    return 0

if __name__ == "__main__":
    raise SystemExit(main())


import logging
import argparse
import json
from pathlib import Path

def main() -> int:
	ap = argparse.ArgumentParser(description="Emit a refactor plan to move scripts/* into scripts/<group>/")
	root = Path(__file__).resolve().parents[1]
	ap.add_argument("--audit", default=str(root / "data" / "ci" / "scripts_audit.json"))
	ap.add_argument("--out", default=str(root / "data" / "traces" / "scripts_refactor_plan.json"))
	ap.add_argument("--emit-shims-dir", default=str(root / "scripts" / "shims"))
	args = ap.parse_args()

	audit_path = Path(args.audit)
	audit = json.loads(audit_path.read_text(encoding="utf-8")) if audit_path.exists() else {}
	mapping = audit.get("proposed_mapping") or {}
	edits = []
	# Create group directories and move files
	created_dirs = set()
	for src, dst in mapping.items():
		dst_path = Path(dst)
		if dst_path.parent.as_posix() not in created_dirs:
			edits.append({"op": "create_file", "file": dst_path.parent.as_posix() + "/.keep", "text": ""})
			created_dirs.add(dst_path.parent.as_posix())
		edits.append({"op": "move", "file": str(Path(src).relative_to(root)), "after": dst})
	plan = {
		"version": "0.1",
		"summary": "Modularize scripts/ into grouped subdirectories with .keep sentinels",
		"edits": edits,
	}
	outp = Path(args.out)
	outp.parent.mkdir(parents=True, exist_ok=True)
	outp.write_text(json.dumps(plan, ensure_ascii=False, indent=2), encoding="utf-8")
	print(str(outp))
	return 0

if __name__ == "__main__":
	raise SystemExit(main())


import logging
import argparse
import json
import re
from pathlib import Path
from typing import Dict, List, Tuple

SCRIPT_PATTERNS: List[Tuple[str, str]] = [
	(r"^(run_loop_|loop_|run-?loop-)", "loops"),
	(r"^(train_|trainer_|planner-|planner_|grpo|ppo)", "train"),
	(r"^(eval_|evaluate_|ci_assert_)", "eval"),
	(r"^(build_|build-)", "build"),
	(r"^(aggregate_|summarize_|dashboard|capabilities|report)", "dashboard"),
	(r"^(bench_|run_llm_|llm|lm_eval|code_review|run_.*bench|orchestrate_.*bench|.*_bench$)", "bench"),
	(r"^(seed_|verify_|validate_|unify_|dedupe_|snapshot_|registry_)", "data"),
	(r"^(docs_|generate_docs|run_docs)", "docs"),
	(r"^(devtools_|dev_|rewrite_|updater|task_scheduler|multi_agent|sandbox)", "devtools"),
	(r"^(router_|train_router|eval_router)", "router"),
]

def categorize(name: str) -> str:
	for pat, bucket in SCRIPT_PATTERNS:
		if re.match(pat, name):
			return bucket
	return "misc"

def main() -> int:
	ap = argparse.ArgumentParser(description="Audit scripts/ for modularization and propose directories")
	root = Path(__file__).resolve().parents[1]
	ap.add_argument("--scripts-dir", default=str(root / "scripts"))
	ap.add_argument("--out", default=str(root / "data" / "ci" / "scripts_audit.json"))
	ap.add_argument("--emit-shims-dir", default=None, help="If set, emit import-forwarding shims under this directory")
	args = ap.parse_args()

	scripts_dir = Path(args.scripts_dir)
	mapping: Dict[str, str] = {}
	groups: Dict[str, List[str]] = {}

	for p in sorted(scripts_dir.glob("*.py")):
		if p.name.startswith("_"):
			continue
		if p.name in ("make_refactor_pipeline.py", "emit_make_refactor_plan.py", "check_scripts_modularization.py", "emit_scripts_refactor_plan.py"):
			continue
		stem = p.stem
		bucket = categorize(stem)
		groups.setdefault(bucket, []).append(p.name)
		mapping[str(p.as_posix())] = f"scripts/{bucket}/{p.name}"

	summary = {
		"root": str(root),
		"total_scripts": sum(len(v) for v in groups.values()),
		"groups": {k: sorted(v) for k, v in groups.items()},
		"proposed_mapping": mapping,
	}

	outp = Path(args.out)
	outp.parent.mkdir(parents=True, exist_ok=True)
	outp.write_text(json.dumps(summary, ensure_ascii=False, indent=2), encoding="utf-8")

	# Optional: emit shims that import and forward to new module path
	if args.emit_shims_dir:
		shims_dir = Path(args.emit_shims_dir)
		shims_dir.mkdir(parents=True, exist_ok=True)
		for src, dst in mapping.items():
			dst_mod = dst.replace("/", ".").removesuffix(".py")
			shim_path = shims_dir / Path(src).name
			shim_code = (
				"import importlib, sys\n"
				f"mod = importlib.import_module('{dst_mod}')\n"
				"sys.exit(getattr(mod, 'main', lambda:0)() if hasattr(mod, 'main') else 0)\n"
			)
			shim_path.write_text(shim_code, encoding="utf-8")

	print(json.dumps({"ok": True, "wrote": str(outp), "groups": list(groups.keys()), "proposed": len(mapping)}))
	return 0

if __name__ == "__main__":
	import sys
	sys.exit(main())


categorize

import json
import re
from pathlib import Path
from typing import Dict, List, Tuple

SCRIPT_PATTERNS: List[Tuple[str, str]] = [
	(r"^(run_loop_|loop_|run-?loop-)", "loops"),
	(r"^(train_|trainer_|planner-|planner_|grpo|ppo)", "train"),
	(r"^(eval_|evaluate_|ci_assert_)", "eval"),
	(r"^(build_|build-)", "build"),
	(r"^(aggregate_|summarize_|dashboard|capabilities|report)", "dashboard"),
	(r"^(bench_|run_llm_|llm|lm_eval|code_review|run_.*bench|orchestrate_.*bench|.*_bench$)", "bench"),
	(r"^(seed_|verify_|validate_|unify_|dedupe_|snapshot_|registry_)", "data"),
	(r"^(docs_|generate_docs|run_docs)", "docs"),
	(r"^(devtools_|dev_|rewrite_|updater|task_scheduler|multi_agent|sandbox)", "devtools"),
	(r"^(router_|train_router|eval_router)", "router"),
]

def categorize(name: str) -> str:
	for pat, bucket in SCRIPT_PATTERNS:
		if re.match(pat, name):
			return bucket
	return "misc"

def main() -> int:
	ap = argparse.ArgumentParser(description="Audit scripts/ for modularization and propose directories")
	root = Path(__file__).resolve().parents[1]
	ap.add_argument("--scripts-dir", default=str(root / "scripts"))
	ap.add_argument("--out", default=str(root / "data" / "ci" / "scripts_audit.json"))
	ap.add_argument("--emit-shims-dir", default=None, help="If set, emit import-forwarding shims under this directory")
	args = ap.parse_args()

	scripts_dir = Path(args.scripts_dir)
	mapping: Dict[str, str] = {}
	groups: Dict[str, List[str]] = {}

	for p in sorted(scripts_dir.glob("*.py")):
		if p.name.startswith("_"):
			continue
		if p.name in ("make_refactor_pipeline.py", "emit_make_refactor_plan.py", "check_scripts_modularization.py", "emit_scripts_refactor_plan.py"):
			continue
		stem = p.stem


main

	(r"^(run_loop_|loop_|run-?loop-)", "loops"),
	(r"^(train_|trainer_|planner-|planner_|grpo|ppo)", "train"),
	(r"^(eval_|evaluate_|ci_assert_)", "eval"),
	(r"^(build_|build-)", "build"),
	(r"^(aggregate_|summarize_|dashboard|capabilities|report)", "dashboard"),
	(r"^(bench_|run_llm_|llm|lm_eval|code_review|run_.*bench|orchestrate_.*bench|.*_bench$)", "bench"),
	(r"^(seed_|verify_|validate_|unify_|dedupe_|snapshot_|registry_)", "data"),
	(r"^(docs_|generate_docs|run_docs)", "docs"),
	(r"^(devtools_|dev_|rewrite_|updater|task_scheduler|multi_agent|sandbox)", "devtools"),
	(r"^(router_|train_router|eval_router)", "router"),
]

def categorize(name: str) -> str:
	for pat, bucket in SCRIPT_PATTERNS:
		if re.match(pat, name):
			return bucket
	return "misc"

def main() -> int:
	ap = argparse.ArgumentParser(description="Audit scripts/ for modularization and propose directories")
	root = Path(__file__).resolve().parents[1]
	ap.add_argument("--scripts-dir", default=str(root / "scripts"))
	ap.add_argument("--out", default=str(root / "data" / "ci" / "scripts_audit.json"))
	ap.add_argument("--emit-shims-dir", default=None, help="If set, emit import-forwarding shims under this directory")
	args = ap.parse_args()

	scripts_dir = Path(args.scripts_dir)
	mapping: Dict[str, str] = {}
	groups: Dict[str, List[str]] = {}

	for p in sorted(scripts_dir.glob("*.py")):
		if p.name.startswith("_"):
			continue
		if p.name in ("make_refactor_pipeline.py", "emit_make_refactor_plan.py", "check_scripts_modularization.py", "emit_scripts_refactor_plan.py"):
			continue
		stem = p.stem
		bucket = categorize(stem)
		groups.setdefault(bucket, []).append(p.name)
		mapping[str(p.as_posix())] = f"scripts/{bucket}/{p.name}"

	summary = {
		"root": str(root),
		"total_scripts": sum(len(v) for v in groups.values()),
		"groups": {k: sorted(v) for k, v in groups.items()},
		"proposed_mapping": mapping,
	}

	outp = Path(args.out)
	outp.parent.mkdir(parents=True, exist_ok=True)
	outp.write_text(json.dumps(summary, ensure_ascii=False, indent=2), encoding="utf-8")

	# Optional: emit shims that import and forward to new module path
	if args.emit_shims_dir:
		shims_dir = Path(args.emit_shims_dir)
		shims_dir.mkdir(parents=True, exist_ok=True)
		for src, dst in mapping.items():
			dst_mod = dst.replace("/", ".").removesuffix(".py")
			shim_path = shims_dir / Path(src).name
			shim_code = (
				"import importlib, sys\n"
				f"mod = importlib.import_module('{dst_mod}')\n"
				"sys.exit(getattr(mod, 'main', lambda:0)() if hasattr(mod, 'main') else 0)\n"
			)
			shim_path.write_text(shim_code, encoding="utf-8")

	print(json.dumps({"ok": True, "wrote": str(outp), "groups": list(groups.keys()), "proposed": len(mapping)}))
	return 0

if __name__ == "__main__":
	import sys
	sys.exit(main())


import logging
import argparse
import json
from pathlib import Path

def build_minimal_makefile() -> str:
	# Minimal Makefile preserving env and modular includes; shims optional via REFAC_INCLUDE_SHIMS
	lines: list[str] = []
	lines.append("PYTHONPATH := /data/agiattempt")
	# Core defaults preserved from legacy Makefile
	lines.append("MODEL ?= gemma3:12b")
	lines.append("HF_MODEL ?= meta-llama/Llama-3.2-3B")
	lines.append("NGPU ?= 2")
	lines.append("TIMEOUT ?= 60")
	lines.append("VERIFY_ARGS ?=")
	lines.append("REPO ?= local:/data/agiattempt/agi_dw")
	lines.append("BASELINE_SUMMARY ?= data/dashboards/baseline_summary.json")
	lines.append("")
	# Data/paths used by modular targets
	lines.append("TRACES := data/traces/seed_os_cli.jsonl")
	lines.append("VERIFIED_LLM := data/traces/seed_os_cli.verified.llm.jsonl")
	lines.append("VERIFIED_HEUR := data/traces/seed_os_cli.verified.heur.jsonl")
	lines.append("PLAN_OUT := data/traces/plan_example.json")
	lines.append("IL_OUT := data/skills/actuator_il.jsonl")
	lines.append("WEB_DOM := data/traces/web_dom.jsonl")
	lines.append("WEB_DOM_VER := data/traces/web_dom.verified.jsonl")
	lines.append("IL_DOM_OUT := data/skills/actuator_il_dom.jsonl")
	lines.append("IL_COMBINED := data/skills/actuator_il_combined.jsonl")
	lines.append("IL_REPAIRS := data/skills/actuator_il_repairs.jsonl")
	lines.append("EPOCHS_CI ?= 1")
	lines.append("BSZ_CI ?= 2")
	lines.append("")
	lines.append("# Modular includes (namespaced; legacy targets removed — use shims during transition)")
	lines.append("-include mk/core.mk")
	lines.append("-include mk/tools.mk")
	lines.append("-include mk/ci.mk")
	lines.append("-include mk/data.mk")
	lines.append("-include mk/loops.mk")
	lines.append("-include mk/splits.mk")
	lines.append("-include mk/train.mk")
	lines.append("-include mk/bench.mk")
	lines.append("-include mk/ci_gates.mk")
	lines.append("-include mk/docs.mk")
	lines.append("-include mk/devtools.mk")
	lines.append("")
	lines.append("# Optional: include auto-generated legacy→namespaced shim delegates")
	lines.append("# Enable by exporting REFAC_INCLUDE_SHIMS=1 before invoking make")
	lines.append("ifdef REFAC_INCLUDE_SHIMS")
	lines.append("-include mk/shims.auto.mk")
	lines.append("endif")
	lines.append("")
	lines.append("# Convenience: route `make help` to modular help printer")
	lines.append(".PHONY: help")
	lines.append("help:")
	lines.append("\t@$(MAKE) -C . mk.help")
	lines.append("")
	return "\n".join(lines) + "\n"

def main() -> int:
	ap = argparse.ArgumentParser(description="Emit deterministic refactor plan to simplify top-level Makefile")
	root = Path(__file__).resolve().parents[1]
	ap.add_argument("--out", default=str(root / "data" / "traces" / "refactor_plan.json"))
	args = ap.parse_args()

	minimal = build_minimal_makefile()
	plan = {
		"version": "0.1",
		"summary": "Replace top-level Makefile with minimal modular includes; shims optional.",
		"edits": [
			{"op": "delete_file", "file": "Makefile"},
			{"op": "create_file", "file": "Makefile", "text": minimal},
		],
	}
	outp = Path(args.out)
	outp.parent.mkdir(parents=True, exist_ok=True)
	outp.write_text(json.dumps(plan, ensure_ascii=False, indent=2), encoding="utf-8")
	print(str(outp))
	return 0

if __name__ == "__main__":
	raise SystemExit(main())


import logging
import argparse
import json
from pathlib import Path

def build_minimal_makefile() -> str:
	# Minimal Makefile preserving env and modular includes; shims optional via REFAC_INCLUDE_SHIMS
	lines: list[str] = []
	lines.append("PYTHONPATH := /data/agiattempt")
	# Core defaults preserved from legacy Makefile
	lines.append("MODEL ?= gemma3:12b")
	lines.append("HF_MODEL ?= meta-llama/Llama-3.2-3B")
	lines.append("NGPU ?= 2")
	lines.append("TIMEOUT ?= 60")
	lines.append("VERIFY_ARGS ?=")
	lines.append("REPO ?= local:/data/agiattempt/agi_dw")
	lines.append("BASELINE_SUMMARY ?= data/dashboards/baseline_summary.json")
	lines.append("")
	# Data/paths used by modular targets
	lines.append("TRACES := data/traces/seed_os_cli.jsonl")
	lines.append("VERIFIED_LLM := data/traces/seed_os_cli.verified.llm.jsonl")
	lines.append("VERIFIED_HEUR := data/traces/seed_os_cli.verified.heur.jsonl")
	lines.append("PLAN_OUT := data/traces/plan_example.json")
	lines.append("IL_OUT := data/skills/actuator_il.jsonl")
	lines.append("WEB_DOM := data/traces/web_dom.jsonl")
	lines.append("WEB_DOM_VER := data/traces/web_dom.verified.jsonl")
	lines.append("IL_DOM_OUT := data/skills/actuator_il_dom.jsonl")
	lines.append("IL_COMBINED := data/skills/actuator_il_combined.jsonl")
	lines.append("IL_REPAIRS := data/skills/actuator_il_repairs.jsonl")
	lines.append("EPOCHS_CI ?= 1")
	lines.append("BSZ_CI ?= 2")
	lines.append("")
	lines.append("# Modular includes (namespaced; legacy targets removed — use shims during transition)")
	lines.append("-include mk/core.mk")
	lines.append("-include mk/tools.mk")
	lines.append("-include mk/ci.mk")
	lines.append("-include mk/data.mk")
	lines.append("-include mk/loops.mk")
	lines.append("-include mk/splits.mk")
	lines.append("-include mk/train.mk")
	lines.append("-include mk/bench.mk")
	lines.append("-include mk/ci_gates.mk")
	lines.append("-include mk/docs.mk")
	lines.append("-include mk/devtools.mk")
	lines.append("")
	lines.append("# Optional: include auto-generated legacy→namespaced shim delegates")
	lines.append("# Enable by exporting REFAC_INCLUDE_SHIMS=1 before invoking make")
	lines.append("ifdef REFAC_INCLUDE_SHIMS")
	lines.append("-include mk/shims.auto.mk")
	lines.append("endif")
	lines.append("")
	lines.append("# Convenience: route `make help` to modular help printer")
	lines.append(".PHONY: help")
	lines.append("help:")
	lines.append("\t@$(MAKE) -C . mk.help")
	lines.append("")
	return "\n".join(lines) + "\n"

def main() -> int:
	ap = argparse.ArgumentParser(description="Emit deterministic refactor plan to simplify top-level Makefile")
	root = Path(__file__).resolve().parents[1]
	ap.add_argument("--out", default=str(root / "data" / "traces" / "refactor_plan.json"))
	args = ap.parse_args()

	minimal = build_minimal_makefile()
	plan = {
		"version": "0.1",
		"summary": "Replace top-level Makefile with minimal modular includes; shims optional.",
		"edits": [
			{"op": "delete_file", "file": "Makefile"},
			{"op": "create_file", "file": "Makefile", "text": minimal},
		],
	}
	outp = Path(args.out)
	outp.parent.mkdir(parents=True, exist_ok=True)
	outp.write_text(json.dumps(plan, ensure_ascii=False, indent=2), encoding="utf-8")


	lines.append("-include mk/train.mk")
	lines.append("-include mk/bench.mk")
	lines.append("-include mk/ci_gates.mk")
	lines.append("-include mk/docs.mk")
	lines.append("-include mk/devtools.mk")
	lines.append("")
	lines.append("# Optional: include auto-generated legacy→namespaced shim delegates")
	lines.append("# Enable by exporting REFAC_INCLUDE_SHIMS=1 before invoking make")
	lines.append("ifdef REFAC_INCLUDE_SHIMS")
	lines.append("-include mk/shims.auto.mk")
	lines.append("endif")
	lines.append("")
	lines.append("# Convenience: route `make help` to modular help printer")
	lines.append(".PHONY: help")
	lines.append("help:")
	lines.append("\t@$(MAKE) -C . mk.help")
	lines.append("")
	return "\n".join(lines) + "\n"

def main() -> int:
	ap = argparse.ArgumentParser(description="Emit deterministic refactor plan to simplify top-level Makefile")
	root = Path(__file__).resolve().parents[1]
	ap.add_argument("--out", default=str(root / "data" / "traces" / "refactor_plan.json"))
	args = ap.parse_args()

	minimal = build_minimal_makefile()
	plan = {
		"version": "0.1",
		"summary": "Replace top-level Makefile with minimal modular includes; shims optional.",
		"edits": [
			{"op": "delete_file", "file": "Makefile"},
			{"op": "create_file", "file": "Makefile", "text": minimal},
		],
	}
	outp = Path(args.out)
	outp.parent.mkdir(parents=True, exist_ok=True)
	outp.write_text(json.dumps(plan, ensure_ascii=False, indent=2), encoding="utf-8")
	print(str(outp))
	return 0

if __name__ == "__main__":
	raise SystemExit(main())


import logging
import argparse
import json
import os
import subprocess
from pathlib import Path

def run(cmd: list[str], cwd: Path) -> dict:
	try:
		out = subprocess.run(cmd, cwd=str(cwd), check=False, capture_output=True, text=True)
		return {
			"cmd": cmd,
			"returncode": out.returncode,
			"stdout": out.stdout.strip(),
			"stderr": out.stderr.strip(),
		}
	except Exception as e:
		return {"cmd": cmd, "error": str(e), "returncode": -1, "stdout": "", "stderr": ""}

def main() -> int:
	ap = argparse.ArgumentParser(description="Automate Makefile modularization/refactor pipeline")
	root = Path(__file__).resolve().parents[1]
	ap.add_argument("--repo", default=str(root))
	ap.add_argument("--model", default="meta-llama/Llama-3.2-3B")
	ap.add_argument("--emit-only", action="store_true", help="Stop after emitting audit+shims+archive")
	ap.add_argument("--dry-run", action="store_true", help="Apply refactor plan in dry-run mode")
	ap.add_argument("--max-edits", type=int, default=200)
	args = ap.parse_args()

	repo = Path(args.repo)
	results: list[dict] = []

	# 1) Audit and emit shims
	env = dict(**os.environ)
	env.setdefault("PYTHONPATH", str(root.parent))
	results.append(run(["make", "tools.audit-make", "ARGS=--report --emit-shims mk/shims.auto.mk"], cwd=repo))

	# 2) Archive snapshot of Makefile and audit metadata
	results.append(run(["python3", "scripts/misc/archive_makefile.py"], cwd=repo))

	# 3) Emit refactor plan (deterministic makefile refactor)
	results.append(run(["python3", "scripts/emit_make_refactor_plan.py", "--out", "data/traces/refactor_plan.json"], cwd=repo))

	# 4) Validate plan
	results.append(run(["python3", "scripts/data/validate_refactor_plan.py", "data/traces/refactor_plan.json"], cwd=repo))

	if bool(args.emit_only):
		print(json.dumps({"ok": True, "stage": "emit_only", "steps": results}))
		return 0

	# 5) Apply plan (dry-run by default)
	apply_cmd = [
		"python3",
		"scripts/misc/apply_refactor_plan.py",
		"--plan",
		"data/traces/refactor_plan.json",
		"--repo",
		str(repo),
		"--validate",
		"--max-edits",
		str(int(args.max_edits)),
	]
	if bool(args.dry_run):
		apply_cmd.append("--dry-run")
	results.append(run(apply_cmd, cwd=repo))

	print(json.dumps({"ok": True, "stage": "done", "steps": results}))
	return 0

if __name__ == "__main__":
	raise SystemExit(main())


run

import logging
import argparse
import json
import os
import subprocess
from pathlib import Path

def run(cmd: list[str], cwd: Path) -> dict:
	try:
		out = subprocess.run(cmd, cwd=str(cwd), check=False, capture_output=True, text=True)
		return {
			"cmd": cmd,
			"returncode": out.returncode,
			"stdout": out.stdout.strip(),
			"stderr": out.stderr.strip(),
		}
	except Exception as e:
		return {"cmd": cmd, "error": str(e), "returncode": -1, "stdout": "", "stderr": ""}

def main() -> int:
	ap = argparse.ArgumentParser(description="Automate Makefile modularization/refactor pipeline")
	root = Path(__file__).resolve().parents[1]
	ap.add_argument("--repo", default=str(root))
	ap.add_argument("--model", default="meta-llama/Llama-3.2-3B")
	ap.add_argument("--emit-only", action="store_true", help="Stop after emitting audit+shims+archive")
	ap.add_argument("--dry-run", action="store_true", help="Apply refactor plan in dry-run mode")
	ap.add_argument("--max-edits", type=int, default=200)
	args = ap.parse_args()

	repo = Path(args.repo)
	results: list[dict] = []

	# 1) Audit and emit shims
	env = dict(**os.environ)
	env.setdefault("PYTHONPATH", str(root.parent))
	results.append(run(["make", "tools.audit-make", "ARGS=--report --emit-shims mk/shims.auto.mk"], cwd=repo))


main

import argparse
import json
import os
import subprocess
from pathlib import Path

def run(cmd: list[str], cwd: Path) -> dict:
	try:
		out = subprocess.run(cmd, cwd=str(cwd), check=False, capture_output=True, text=True)
		return {
			"cmd": cmd,
			"returncode": out.returncode,
			"stdout": out.stdout.strip(),
			"stderr": out.stderr.strip(),
		}
	except Exception as e:
		return {"cmd": cmd, "error": str(e), "returncode": -1, "stdout": "", "stderr": ""}

def main() -> int:
	ap = argparse.ArgumentParser(description="Automate Makefile modularization/refactor pipeline")
	root = Path(__file__).resolve().parents[1]
	ap.add_argument("--repo", default=str(root))
	ap.add_argument("--model", default="meta-llama/Llama-3.2-3B")
	ap.add_argument("--emit-only", action="store_true", help="Stop after emitting audit+shims+archive")
	ap.add_argument("--dry-run", action="store_true", help="Apply refactor plan in dry-run mode")
	ap.add_argument("--max-edits", type=int, default=200)
	args = ap.parse_args()

	repo = Path(args.repo)
	results: list[dict] = []

	# 1) Audit and emit shims
	env = dict(**os.environ)
	env.setdefault("PYTHONPATH", str(root.parent))
	results.append(run(["make", "tools.audit-make", "ARGS=--report --emit-shims mk/shims.auto.mk"], cwd=repo))

	# 2) Archive snapshot of Makefile and audit metadata
	results.append(run(["python3", "scripts/misc/archive_makefile.py"], cwd=repo))

	# 3) Emit refactor plan (deterministic makefile refactor)
	results.append(run(["python3", "scripts/emit_make_refactor_plan.py", "--out", "data/traces/refactor_plan.json"], cwd=repo))

	# 4) Validate plan
	results.append(run(["python3", "scripts/data/validate_refactor_plan.py", "data/traces/refactor_plan.json"], cwd=repo))

	if bool(args.emit_only):
		print(json.dumps({"ok": True, "stage": "emit_only", "steps": results}))
		return 0

	# 5) Apply plan (dry-run by default)
	apply_cmd = [
		"python3",
		"scripts/misc/apply_refactor_plan.py",
		"--plan",
		"data/traces/refactor_plan.json",
		"--repo",
		str(repo),
		"--validate",
		"--max-edits",
		str(int(args.max_edits)),
	]
	if bool(args.dry_run):
		apply_cmd.append("--dry-run")
	results.append(run(apply_cmd, cwd=repo))

	print(json.dumps({"ok": True, "stage": "done", "steps": results}))
	return 0

if __name__ == "__main__":
	raise SystemExit(main())


from __future__ import annotations
import logging

import argparse
import json
from pathlib import Path
from typing import Dict, Any, List

def main() -> int:
    ap = argparse.ArgumentParser(description="Build deterministic PlanDAG from intent and code/policy graphs")
    root = Path(__file__).resolve().parents[2]
    ap.add_argument("--intent", default=str(root / "data" / "sandbox" / "plan" / "intent.json"))
    ap.add_argument("--out", default=str(root / "data" / "sandbox" / "plan" / "plan.json"))
    ap.add_argument("--max-pr-files", type=int, default=0, help="Override max files per PR (0 uses policies.yaml)")
    args = ap.parse_args()

    # Load inputs
    intent = json.loads(Path(args.intent).read_text(encoding="utf-8")) if Path(args.intent).exists() else {"intent": {}}
    idx_path = root / "data" / "sandbox" / "tmp" / "index.json"
    pol_path = root / "data" / "sandbox" / "config" / "policies.yaml"
    risk_path = root / "data" / "sandbox" / "tmp" / "plan_risk.json"
    try:
        idx = json.loads(idx_path.read_text(encoding="utf-8")) if idx_path.exists() else {}
    except Exception:
        idx = {}
    try:
        import yaml  # type: ignore
        policies = yaml.safe_load(pol_path.read_text(encoding="utf-8")) if pol_path.exists() else {}
    except Exception:
        policies = {}
    try:
        risk = json.loads(risk_path.read_text(encoding="utf-8")) if risk_path.exists() else {}
    except Exception:
        risk = {}

    # Determine caps
    bpr = ((policies.get("budgets") or {}).get("pr") or {}) if isinstance(policies, dict) else {}
    max_files = int(args.max_pr_files) if int(getattr(args, "max_pr_files", 0) or 0) > 0 else int(bpr.get("max_files", 10) or 10)

    # Very simple chunking strategy: group files into PR packs of size <= max_files
    files = sorted(list(((idx.get("functions") or {}) if isinstance(idx, dict) else {}).keys()))[:200]
    chunks: List[Dict[str, Any]] = []
    cur: List[str] = []
    for fp in files:
        cur.append(fp)
        if len(cur) >= max_files:
            chunks.append({"files": cur, "caps": {"max_files": max_files}})
            cur = []
    if cur:
        chunks.append({"files": cur, "caps": {"max_files": max_files}})

    # Build a simple DAG: linear over chunks
    dag: List[Dict[str, Any]] = []
    prs: List[Dict[str, Any]] = []
    for i, ch in enumerate(chunks, start=1):
        tid = f"T{i}"
        dag.append({"id": tid, "deps": [f"T{i-1}"] if i > 1 else []})
        prs.append({"name": f"auto-pack-{i}", "tasks": [tid], "files": ch.get("files", [])[:max_files], "gates": ["tests", "types", "risk"], "caps": ch.get("caps", {})})

    plan = {
        "overview": {
            "ticket": intent.get("intent", {}).get("ticket", "UNKNOWN"),
            "change_kind": intent.get("intent", {}).get("change_kind", "refactor"),
            "prs": len(prs),
        },
        "dag": dag,
        "prs": prs,
        "caps": {"max_files": max_files},
        "sources": {"index": str(idx_path), "policies": str(pol_path), "risk": str(risk_path)},
    }

    outp = Path(args.out)
    outp.parent.mkdir(parents=True, exist_ok=True)
    outp.write_text(json.dumps({"ok": True, "plan": plan}, ensure_ascii=False, indent=2), encoding="utf-8")
    print(json.dumps({"ok": True, "out": str(outp), "prs": len(prs)}))
    return 0

if __name__ == "__main__":
    raise SystemExit(main())


from __future__ import annotations
import logging

import argparse
import json
from pathlib import Path

def main() -> int:
    ap = argparse.ArgumentParser(description="Render PlanDAG to markdown report (sandbox)")
    root = Path(__file__).resolve().parents[2]
    ap.add_argument("--plan", default=str(root / "data" / "sandbox" / "plan" / "plan.json"))
    ap.add_argument("--out", default=str(root / "data" / "sandbox" / "docs" / "reports" / "plan.md"))
    args = ap.parse_args()

    plan = json.loads(Path(args.plan).read_text(encoding="utf-8")) if Path(args.plan).exists() else {}
    overview = plan.get("plan", {}).get("overview", {})
    dag = plan.get("plan", {}).get("dag", [])
    prs = plan.get("plan", {}).get("prs", [])

    lines = ["# PlanDAG", "", f"Change kind: {overview.get('change_kind','refactor')}", "", "## DAG", ""]
    for it in dag:
        lines.append(f"- {it.get('id')} deps: {','.join(it.get('deps', []))}")
    lines += ["", "## PRs", ""]
    for pr in prs:
        lines.append(f"- {pr.get('name')}: tasks={','.join(pr.get('tasks', []))} gates={','.join(pr.get('gates', []))}")

    outp = Path(args.out)
    outp.parent.mkdir(parents=True, exist_ok=True)
    outp.write_text("\n".join(lines) + "\n", encoding="utf-8")
    print(json.dumps({"ok": True, "out": str(outp)}))
    return 0

if __name__ == "__main__":
    raise SystemExit(main())


from __future__ import annotations
import logging

import argparse
import json
from pathlib import Path
from typing import Dict, Any

def main() -> int:
    ap = argparse.ArgumentParser(description="Normalize raw spec text to structured intent")
    root = Path(__file__).resolve().parents[2]
    ap.add_argument("--spec", required=True, help="Raw spec text or path to file")
    ap.add_argument("--out", default=str(root / "data" / "sandbox" / "plan" / "intent.json"))
    args = ap.parse_args()

    raw = args.spec
    spec_text = raw
    p = Path(raw)
    if p.exists():
        spec_text = p.read_text(encoding="utf-8")

    # Very simple heuristic normalization (can be expanded deterministically)
    intent: Dict[str, Any] = {
        "intent": {
            "capability": "unspecified",
            "change_kind": "refactor",
            "constraints": {},
            "acceptance": [],
        },
        "spec_raw": spec_text.strip(),
    }

    outp = Path(args.out)
    outp.parent.mkdir(parents=True, exist_ok=True)
    outp.write_text(json.dumps(intent, ensure_ascii=False, indent=2), encoding="utf-8")
    print(json.dumps({"ok": True, "out": str(outp)}))
    return 0

if __name__ == "__main__":
    raise SystemExit(main())


from __future__ import annotations

import logging
import argparse
import json
from pathlib import Path

EXAMPLES = [
    {
        "id": "make_flag_fix_limit",
        "category": "bench_infra",
        "problem": "Make target passes unsupported --max; script expects --limit",
        "files": ["Makefile"],
        "before": "--max ${MAX:-10}",
        "after": "--limit ${LIMIT:-10}",
        "verify": {
            "cmd": "make -C agi_dw -n bench.run.humaneval",
            "must_contain": ["--limit"],
        },
    },
    # Removed legacy script-based runner task; registry-driven runner supersedes it.
    {
        "id": "ci_gate_inline_py_one_liner",
        "category": "bench_infra",
        "problem": "Inline Python in make target uses newlines; convert to single expression",
        "files": ["mk/ci_gates.mk"],
        "before": "python3 -c \"...\nfor ...\n...\"",
        "after": "python3 -c \"import json,sys,...; print(json.dumps(...))\"",
        "verify": {
            "cmd": "make -C agi_dw -n ci.gate.train",
            "must_contain": ["python3 -c"],
        },
    },
]

def main() -> int:
    ap = argparse.ArgumentParser(description="Export bench infra refactor tasks (SFT/IL seeds)")
    root = Path(__file__).resolve().parents[2]
    ap.add_argument("--out", default=str(root / "data" / "traces" / "benchinfra_tasks.jsonl"))
    args = ap.parse_args()

    outp = Path(args.out)
    outp.parent.mkdir(parents=True, exist_ok=True)
    with outp.open("w", encoding="utf-8") as f:
        for ex in EXAMPLES:
            f.write(json.dumps(ex) + "\n")
    print(json.dumps({"ok": True, "out": str(outp), "n": len(EXAMPLES)}))
    return 0

if __name__ == "__main__":
    raise SystemExit(main())


            "cmd": "make -C agi_dw -n bench.run.humaneval",
            "must_contain": ["--limit"],
        },
    },
    # Removed legacy script-based runner task; registry-driven runner supersedes it.
    {
        "id": "ci_gate_inline_py_one_liner",
        "category": "bench_infra",
        "problem": "Inline Python in make target uses newlines; convert to single expression",
        "files": ["mk/ci_gates.mk"],
        "before": "python3 -c \"...\nfor ...\n...\"",
        "after": "python3 -c \"import json,sys,...; print(json.dumps(...))\"",
        "verify": {
            "cmd": "make -C agi_dw -n ci.gate.train",
            "must_contain": ["python3 -c"],
        },
    },
]

def main() -> int:
    ap = argparse.ArgumentParser(description="Export bench infra refactor tasks (SFT/IL seeds)")
    root = Path(__file__).resolve().parents[2]
    ap.add_argument("--out", default=str(root / "data" / "traces" / "benchinfra_tasks.jsonl"))
    args = ap.parse_args()

    outp = Path(args.out)
    outp.parent.mkdir(parents=True, exist_ok=True)
    with outp.open("w", encoding="utf-8") as f:
        for ex in EXAMPLES:
            f.write(json.dumps(ex) + "\n")
    print(json.dumps({"ok": True, "out": str(outp), "n": len(EXAMPLES)}))
    return 0

if __name__ == "__main__":
    raise SystemExit(main())


from __future__ import annotations
import logging

import argparse
import json
from pathlib import Path

def main() -> int:
    ap = argparse.ArgumentParser(description="Convert SFT JSONL {prompt,response} to {input,output}")
    ap.add_argument("--in", dest="inp", required=True)
    ap.add_argument("--out", required=True)
    args = ap.parse_args()

    inp = Path(args.inp)
    out = Path(args.out)
    out.parent.mkdir(parents=True, exist_ok=True)

    n = 0
    with inp.open("r", encoding="utf-8") as f, out.open("w", encoding="utf-8") as w:
        for line in f:
            s = line.strip()
            if not s:
                continue
            try:
                row = json.loads(s)
            except Exception:
                continue
            prompt = str(row.get("prompt", ""))
            resp = str(row.get("response", ""))
            if not prompt or not resp:
                continue
            w.write(json.dumps({"input": prompt, "output": resp}, ensure_ascii=False) + "\n")
            n += 1
    print(json.dumps({"ok": True, "in": str(inp), "out": str(out), "n": n}))
    return 0

if __name__ == "__main__":
    raise SystemExit(main())


from __future__ import annotations
import logging

import argparse
import json
from pathlib import Path
from typing import Dict, Any, List

def load_jsonl(path: Path) -> List[Dict[str, Any]]:
    if not path.exists():
        return []
    rows: List[Dict[str, Any]] = []
    for line in path.read_text(encoding="utf-8").splitlines():
        s = line.strip()
        if not s:
            continue
        try:
            rows.append(json.loads(s))
        except Exception:
            continue
    return rows

def main() -> int:
    ap = argparse.ArgumentParser(description="Curate SFT solutions from local sources (license-filtered)")
    root = Path(__file__).resolve().parents[2]
    ap.add_argument("--in", dest="inp", default=str(root / "data" / "sandbox" / "sft_sources"), help="Directory of JSONL files with fields: {problem, solution, license}")
    ap.add_argument("--allow-licenses", default="mit,apache-2.0,bsd-3-clause", help="Comma-separated allowed license ids")
    ap.add_argument("--out", default=str(root / "data" / "sandbox" / "sft"))
    args = ap.parse_args()

    in_dir = Path(args.inp)
    out_dir = Path(args.out)
    out_dir.mkdir(parents=True, exist_ok=True)
    in_dir.mkdir(parents=True, exist_ok=True)

    # If empty, seed a tiny sample
    if not any(in_dir.glob("*.jsonl")):
        sample = {
            "problem": "Write a function add(a, b) returning a+b",
            "solution": "def add(a, b):\n    return a + b\n",
            "license": "mit",
        }
        (in_dir / "sample.jsonl").write_text(json.dumps(sample) + "\n", encoding="utf-8")

    allow = {s.strip().lower() for s in str(getattr(args, "allow_licenses", "") or "").split(",") if s.strip()}
    curated: List[Dict[str, Any]] = []
    for p in sorted(in_dir.glob("*.jsonl")):
        rows = load_jsonl(p)
        for r in rows:
            lic = str((r.get("license") or "").lower())
            if allow and lic not in allow:
                continue
            # minimal schema: instruction → output pairs
            curated.append({
                "instruction": str(r.get("problem", "")),
                "output": str(r.get("solution", "")),
                "license": lic or "unknown",
                "source": p.name,
            })

    # Write outputs
    curated_path = out_dir / "curated.jsonl"
    with curated_path.open("w", encoding="utf-8") as f:
        for row in curated:
            f.write(json.dumps(row) + "\n")

    meta = {"ok": True, "count": len(curated), "out": str(out_dir), "curated": str(curated_path)}
    (out_dir / "manifest.json").write_text(json.dumps(meta, ensure_ascii=False, indent=2), encoding="utf-8")
    print(json.dumps(meta))
    return 0

if __name__ == "__main__":
    raise SystemExit(main())


from __future__ import annotations
import logging

import argparse
import json
from pathlib import Path
from typing import Dict, Any, List

def load_jsonl(path: Path) -> List[Dict[str, Any]]:
    if not path.exists():
        return []
    rows: List[Dict[str, Any]] = []
    for line in path.read_text(encoding="utf-8").splitlines():
        s = line.strip()
        if not s:
            continue
        try:
            rows.append(json.loads(s))
        except Exception:
            continue
    return rows

def main() -> int:
    ap = argparse.ArgumentParser(description="Curate SFT solutions from local sources (license-filtered)")
    root = Path(__file__).resolve().parents[2]
    ap.add_argument("--in", dest="inp", default=str(root / "data" / "sandbox" / "sft_sources"), help="Directory of JSONL files with fields: {problem, solution, license}")
    ap.add_argument("--allow-licenses", default="mit,apache-2.0,bsd-3-clause", help="Comma-separated allowed license ids")
    ap.add_argument("--out", default=str(root / "data" / "sandbox" / "sft"))
    args = ap.parse_args()

    in_dir = Path(args.inp)
    out_dir = Path(args.out)
    out_dir.mkdir(parents=True, exist_ok=True)
    in_dir.mkdir(parents=True, exist_ok=True)

    # If empty, seed a tiny sample
    if not any(in_dir.glob("*.jsonl")):
        sample = {
            "problem": "Write a function add(a, b) returning a+b",
            "solution": "def add(a, b):\n    return a + b\n",


import json
from pathlib import Path
from typing import Dict, Any, List

def load_jsonl(path: Path) -> List[Dict[str, Any]]:
    if not path.exists():
        return []
    rows: List[Dict[str, Any]] = []
    for line in path.read_text(encoding="utf-8").splitlines():
        s = line.strip()
        if not s:
            continue
        try:
            rows.append(json.loads(s))
        except Exception:
            continue
    return rows

def main() -> int:
    ap = argparse.ArgumentParser(description="Curate SFT solutions from local sources (license-filtered)")
    root = Path(__file__).resolve().parents[2]
    ap.add_argument("--in", dest="inp", default=str(root / "data" / "sandbox" / "sft_sources"), help="Directory of JSONL files with fields: {problem, solution, license}")
    ap.add_argument("--allow-licenses", default="mit,apache-2.0,bsd-3-clause", help="Comma-separated allowed license ids")
    ap.add_argument("--out", default=str(root / "data" / "sandbox" / "sft"))
    args = ap.parse_args()

    in_dir = Path(args.inp)
    out_dir = Path(args.out)
    out_dir.mkdir(parents=True, exist_ok=True)
    in_dir.mkdir(parents=True, exist_ok=True)

    # If empty, seed a tiny sample
    if not any(in_dir.glob("*.jsonl")):
        sample = {
            "problem": "Write a function add(a, b) returning a+b",
            "solution": "def add(a, b):\n    return a + b\n",
            "license": "mit",
        }
        (in_dir / "sample.jsonl").write_text(json.dumps(sample) + "\n", encoding="utf-8")

    allow = {s.strip().lower() for s in str(getattr(args, "allow_licenses", "") or "").split(",") if s.strip()}
    curated: List[Dict[str, Any]] = []
    for p in sorted(in_dir.glob("*.jsonl")):
        rows = load_jsonl(p)
        for r in rows:
            lic = str((r.get("license") or "").lower())
            if allow and lic not in allow:
                continue
            # minimal schema: instruction → output pairs
            curated.append({
                "instruction": str(r.get("problem", "")),
                "output": str(r.get("solution", "")),
                "license": lic or "unknown",
                "source": p.name,
            })

    # Write outputs
    curated_path = out_dir / "curated.jsonl"
    with curated_path.open("w", encoding="utf-8") as f:
        for row in curated:
            f.write(json.dumps(row) + "\n")

    meta = {"ok": True, "count": len(curated), "out": str(out_dir), "curated": str(curated_path)}
    (out_dir / "manifest.json").write_text(json.dumps(meta, ensure_ascii=False, indent=2), encoding="utf-8")
    print(json.dumps(meta))
    return 0

if __name__ == "__main__":
    raise SystemExit(main())


from __future__ import annotations
import logging

import argparse
import json
from pathlib import Path
from typing import Dict, Any, List

def main() -> int:
    ap = argparse.ArgumentParser(description="Generate IL traces (plan→patch→verify) for repo-level tasks (synthetic from bench-infra)")
    root = Path(__file__).resolve().parents[2]
    ap.add_argument("--tasks", default=str(root / "data" / "traces" / "benchinfra_tasks.jsonl"))
    ap.add_argument("--out", default=str(root / "data" / "sandbox" / "il_traces"))
    args = ap.parse_args()

    tasks_path = Path(args.tasks)
    out_dir = Path(args.out)
    out_dir.mkdir(parents=True, exist_ok=True)

    traces_path = out_dir / "il_traces.jsonl"
    n = 0
    if tasks_path.exists():
        for line in tasks_path.read_text(encoding="utf-8").splitlines():
            s = line.strip()
            if not s:
                continue
            try:
                ex = json.loads(s)
            except Exception:
                continue
            trace = {
                "id": ex.get("id"),
                "plan": ex.get("problem"),
                "patch": {"files": ex.get("files"), "before": ex.get("before"), "after": ex.get("after")},
                "verify": ex.get("verify"),
            }
            with traces_path.open("a", encoding="utf-8") as f:
                f.write(json.dumps(trace) + "\n")
            n += 1

    meta = {"ok": True, "n_traces": n, "out": str(out_dir), "traces": str(traces_path)}
    (out_dir / "manifest.json").write_text(json.dumps(meta, ensure_ascii=False, indent=2), encoding="utf-8")
    print(json.dumps(meta))
    return 0

if __name__ == "__main__":
    raise SystemExit(main())


import torch
import torch.nn.functional as F
from torch.nn.utils import clip_grad_norm_
from typing import Iterable

from agi_dw.core.metaopt.group_utils import make_param_groups
from agi_dw.core.metaopt.gate_grouped import GateNetGrouped
from agi_dw.core.metaopt.metaopt_grouped import MixtureMetaOptGrouped

def integration_sketch(model: torch.nn.Module, loader: Iterable, val_loader: Iterable, device: str = "cuda") -> None:
    model.to(device)
    groups = make_param_groups(model)
    gate = GateNetGrouped(n_groups=len(groups), hidden=128)
    meta = MixtureMetaOptGrouped(model, groups, gate, base_lr=3e-4, base_wd=0.01, device=device)

    step = 0
    for batch in loader:
        step += 1
        x, y = batch
        x = x.to(device)
        y = y.to(device)
        logits = model(x)
        loss = F.cross_entropy(logits, y)

        for p in model.parameters():
            if p.grad is not None:
                p.grad.zero_()
        loss.backward()
        clip_grad_norm_(model.parameters(), 1.0)

        meta.step(float(loss.item()))

        if step % 200 == 0:
            vtot, vcount = 0.0, 0
            model.eval()
            with torch.no_grad():
                for vx, vy in val_loader:
                    vx = vx.to(device)
                    vy = vy.to(device)
                    vloss = F.cross_entropy(model(vx), vy, reduction="sum")
                    vtot += float(vloss.item())
                    vcount += vx.size(0)
            model.train()
            vavg = vtot / max(1, vcount)
            reward = -vavg
            for gi in range(len(groups)):
                meta.record_bandit_reward(gi, reward)

if __name__ == "__main__":
    print("This script provides an integration sketch. Import `integration_sketch` and pass your model and dataloaders.")


from __future__ import annotations
import logging

import argparse
from pathlib import Path

def main() -> int:
    ap = argparse.ArgumentParser(description="QLoRA SFT trainer for coder head (prompt->body)")
    root = Path(__file__).resolve().parents[1]
    ap.add_argument("--data", required=True, help="JSONL with {input,output}")
    ap.add_argument("--base-model", default="meta-llama/Llama-3.2-3B")
    ap.add_argument("--out", default=str(root / "models" / "coder_qlora"))
    ap.add_argument("--epochs", type=int, default=1)
    ap.add_argument("--bsz", type=int, default=4)
    ap.add_argument("--max-len", type=int, default=512)
    ap.add_argument("--metaopt", action="store_true", help="Enable meta-optimizer (MixtureMetaOptGrouped) during training")
    ap.add_argument("--meta-base-lr", type=float, default=3e-4)
    ap.add_argument("--meta-base-wd", type=float, default=0.01)
    args = ap.parse_args()

    try:
        import torch  # type: ignore
        from datasets import load_dataset  # type: ignore
        from transformers import AutoModelForCausalLM, AutoTokenizer, TrainingArguments, BitsAndBytesConfig  # type: ignore
        from peft import LoraConfig, get_peft_model  # type: ignore
        import bitsandbytes as bnb  # type: ignore
        from agi_dw.core.metaopt.hf_meta_trainer import MetaOptTrainer  # type: ignore
    except Exception as e:  # pragma: no cover
        print("[WARN] Missing deps for QLoRA (install: peft, bitsandbytes, accelerate). Skipping train.")
        print("Reason:", e)
        return 0

    # Short-circuit if dataset is empty or missing
    data_path = Path(args.data)
    if (not data_path.exists()) or data_path.stat().st_size == 0:
        print("[WARN] No training data found; skipping QLoRA training.")
        return 0
    # Count non-empty lines
    try:
        with data_path.open("r", encoding="utf-8") as f:
            has_rows = any(bool(line.strip()) for line in f)
        if not has_rows:
            print("[WARN] Training data has 0 examples; skipping QLoRA training.")
            return 0
    except Exception:
        print("[WARN] Failed to read training data; skipping QLoRA training.")
        return 0

    tokenizer = AutoTokenizer.from_pretrained(args.base_model)
    if tokenizer.pad_token_id is None:
        tokenizer.pad_token_id = tokenizer.eos_token_id

    ds = load_dataset("json", data_files={"train": args.data})
    cols = list(ds["train"].column_names)
    # Determine schema mapping to (input, output)
    in_key, out_key = None, None
    if all(k in cols for k in ["input", "output"]):
        in_key, out_key = "input", "output"
    elif all(k in cols for k in ["prompt", "result"]):
        in_key, out_key = "prompt", "result"
    elif all(k in cols for k in ["obs", "diff_text"]):
        in_key, out_key = "obs", "diff_text"
    elif all(k in cols for k in ["prompt", "diff_text"]):
        in_key, out_key = "prompt", "diff_text"
    else:
        # Fallback heuristic
        for cand_in in ["prompt", "obs", "intent"]:
            if cand_in in cols:
                in_key = cand_in
                break
        for cand_out in ["result", "diff_text", "candidate"]:
            if cand_out in cols:
                out_key = cand_out
                break
    if in_key is None or out_key is None:
        raise SystemExit(f"Unsupported dataset schema. Columns: {cols}")

    def preprocess(batch):
        input_ids_list = []
        attn_masks = []
        labels_list = []
        xin = batch.get(in_key, [])
        yout = batch.get(out_key, [])
        # Ensure same length by zipping
        for x, y in zip(xin, yout):
            x = str(x) if x is not None else ""
            y = str(y) if y is not None else ""
            prompt_ids = tokenizer(x, add_special_tokens=False)["input_ids"]
            target_ids = tokenizer(y, add_special_tokens=False)["input_ids"]
            ids = (prompt_ids + target_ids)[: args.max_len]
            labels = ([-100] * min(len(prompt_ids), len(ids)) + ids[len(prompt_ids) :])
            pad_len = args.max_len - len(ids)
            if pad_len > 0:
                ids = ids + [tokenizer.pad_token_id] * pad_len
                labels = labels + ([-100] * pad_len)
            attn = [1] * (args.max_len - pad_len) + [0] * pad_len
            input_ids_list.append(ids)
            attn_masks.append(attn)
            labels_list.append(labels)
        return {"input_ids": input_ids_list, "attention_mask": attn_masks, "labels": labels_list}

    proc = ds.map(preprocess, batched=True, remove_columns=cols)  # type: ignore

    _ = bnb  # ensure import
    quant_config = BitsAndBytesConfig(
        load_in_4bit=True,
        bnb_4bit_use_double_quant=True,
        bnb_4bit_quant_type="nf4",
        bnb_4bit_compute_dtype=torch.bfloat16 if torch.cuda.is_available() else torch.float32,
    )
    model = AutoModelForCausalLM.from_pretrained(
        args.base_model,
        quantization_config=quant_config,
        device_map="auto",
    )
    lora_cfg = LoraConfig(r=8, lora_alpha=16, target_modules=["q_proj", "v_proj", "k_proj", "o_proj"], lora_dropout=0.05, bias="none")
    model = get_peft_model(model, lora_cfg)

    training_args = TrainingArguments(
        output_dir=str(Path(args.out) / "hf_out"),
        per_device_train_batch_size=int(args.bsz),
        num_train_epochs=int(args.epochs),
        learning_rate=2e-4,
        logging_steps=10,
        save_total_limit=1,
        save_strategy="no",
        report_to=[],
        fp16=False,
        bf16=torch.cuda.is_available(),
        gradient_accumulation_steps=1,
    )
    if bool(args.metaopt):
        trainer = MetaOptTrainer(model=model, args=training_args, train_dataset=proc["train"], metaopt=True, meta_base_lr=float(args.meta_base_lr), meta_base_wd=float(args.meta_base_wd))  # type: ignore
    else:
        from transformers import Trainer  # type: ignore
        trainer = Trainer(model=model, args=training_args, train_dataset=proc["train"])  # type: ignore
    trainer.train()

    Path(args.out).mkdir(parents=True, exist_ok=True)
    model.save_pretrained(args.out)
    tokenizer.save_pretrained(args.out)
    print(f"Saved coder LoRA -> {args.out}")
    return 0

if __name__ == "__main__":
    raise SystemExit(main())


preprocess

        in_key, out_key = "input", "output"
    elif all(k in cols for k in ["prompt", "result"]):
        in_key, out_key = "prompt", "result"
    elif all(k in cols for k in ["obs", "diff_text"]):
        in_key, out_key = "obs", "diff_text"
    elif all(k in cols for k in ["prompt", "diff_text"]):
        in_key, out_key = "prompt", "diff_text"
    else:
        # Fallback heuristic
        for cand_in in ["prompt", "obs", "intent"]:
            if cand_in in cols:
                in_key = cand_in
                break
        for cand_out in ["result", "diff_text", "candidate"]:
            if cand_out in cols:
                out_key = cand_out
                break
    if in_key is None or out_key is None:
        raise SystemExit(f"Unsupported dataset schema. Columns: {cols}")

    def preprocess(batch):
        input_ids_list = []
        attn_masks = []
        labels_list = []
        xin = batch.get(in_key, [])
        yout = batch.get(out_key, [])
        # Ensure same length by zipping
        for x, y in zip(xin, yout):
            x = str(x) if x is not None else ""
            y = str(y) if y is not None else ""
            prompt_ids = tokenizer(x, add_special_tokens=False)["input_ids"]
            target_ids = tokenizer(y, add_special_tokens=False)["input_ids"]
            ids = (prompt_ids + target_ids)[: args.max_len]
            labels = ([-100] * min(len(prompt_ids), len(ids)) + ids[len(prompt_ids) :])
            pad_len = args.max_len - len(ids)
            if pad_len > 0:
                ids = ids + [tokenizer.pad_token_id] * pad_len
                labels = labels + ([-100] * pad_len)
            attn = [1] * (args.max_len - pad_len) + [0] * pad_len
            input_ids_list.append(ids)
            attn_masks.append(attn)
            labels_list.append(labels)
        return {"input_ids": input_ids_list, "attention_mask": attn_masks, "labels": labels_list}

    proc = ds.map(preprocess, batched=True, remove_columns=cols)  # type: ignore

    _ = bnb  # ensure import
    quant_config = BitsAndBytesConfig(
        load_in_4bit=True,
        bnb_4bit_use_double_quant=True,
        bnb_4bit_quant_type="nf4",
        bnb_4bit_compute_dtype=torch.bfloat16 if torch.cuda.is_available() else torch.float32,
    )
    model = AutoModelForCausalLM.from_pretrained(
        args.base_model,
        quantization_config=quant_config,
        device_map="auto",
    )
    lora_cfg = LoraConfig(r=8, lora_alpha=16, target_modules=["q_proj", "v_proj", "k_proj", "o_proj"], lora_dropout=0.05, bias="none")
    model = get_peft_model(model, lora_cfg)

    training_args = TrainingArguments(
        output_dir=str(Path(args.out) / "hf_out"),


from __future__ import annotations
import logging

import argparse
import json
from pathlib import Path
from typing import Any, Dict

def count_lines(path: Path) -> int:
    if not path.exists():
        return 0
    try:
        return sum(1 for _ in path.open("r", encoding="utf-8"))
    except Exception:
        return 0

def main() -> int:
    ap = argparse.ArgumentParser(description="Tune heads for size/files/risk (heuristic)")
    root = Path(__file__).resolve().parents[2]
    default_out = str(root / "data" / "sandbox" / "tmp" / "heads_tuning.json")
    ap.add_argument("--results", default=default_out)
    ap.add_argument("--sft", default=str(root / "data" / "sandbox" / "sft" / "curated.jsonl"))
    ap.add_argument("--il", default=str(root / "data" / "sandbox" / "il_traces" / "il_traces.jsonl"))
    args = ap.parse_args()

    outp = Path(args.results)
    outp.parent.mkdir(parents=True, exist_ok=True)

    sft_n = count_lines(Path(args.sft))
    il_n = count_lines(Path(args.il))

    # Simple heuristic: expected lift grows with log of examples
    import math
    lift = round(0.02 * math.log1p(max(1, sft_n)) + 0.01 * math.log1p(max(1, il_n)), 4)

    results: Dict[str, Any] = {
        "ok": True,
        "inputs": {"sft_examples": sft_n, "il_traces": il_n},
        "lift_estimate": float(lift),
        "out": str(outp),
    }
    outp.write_text(json.dumps(results, ensure_ascii=False, indent=2), encoding="utf-8")
    print(json.dumps(results))
    return 0

if __name__ == "__main__":
    raise SystemExit(main())


from __future__ import annotations
import logging

import argparse
import json
from pathlib import Path
from typing import Any, Dict

def count_lines(path: Path) -> int:
    if not path.exists():
        return 0
    try:
        return sum(1 for _ in path.open("r", encoding="utf-8"))
    except Exception:
        return 0

def main() -> int:
    ap = argparse.ArgumentParser(description="Tune heads for size/files/risk (heuristic)")
    root = Path(__file__).resolve().parents[2]
    default_out = str(root / "data" / "sandbox" / "tmp" / "heads_tuning.json")
    ap.add_argument("--results", default=default_out)
    ap.add_argument("--sft", default=str(root / "data" / "sandbox" / "sft" / "curated.jsonl"))
    ap.add_argument("--il", default=str(root / "data" / "sandbox" / "il_traces" / "il_traces.jsonl"))
    args = ap.parse_args()

    outp = Path(args.results)
    outp.parent.mkdir(parents=True, exist_ok=True)

    sft_n = count_lines(Path(args.sft))
    il_n = count_lines(Path(args.il))

    # Simple heuristic: expected lift grows with log of examples
    import math
    lift = round(0.02 * math.log1p(max(1, sft_n)) + 0.01 * math.log1p(max(1, il_n)), 4)


import logging
import argparse
import json
import re
from pathlib import Path
import os
from typing import Dict, List, Optional

from datasets import load_dataset
from transformers import (
	AutoModelForSeq2SeqLM,
	AutoTokenizer,
	DataCollatorForSeq2Seq,
	Seq2SeqTrainingArguments,
	Seq2SeqTrainer,
	EarlyStoppingCallback,
)
from transformers import set_seed
try:
	import torch  # type: ignore
except Exception:
	torch = None  # type: ignore

CLI_INSTRUCTION = (
	'Actuator task: Return ONLY the CLI argv as a single space-separated string. '
	'Example: wc -l docs/a.txt. No quotes, no explanations, no extra text. Input follows:\n'
)

DOM_INSTRUCTION = (
	'DOM task: Return ONLY two tokens: the URL and the CSS selector, separated by a single space. '
	'Example: https://example.com h1. No quotes, no explanations, no extra text. Input follows:\n'
)

def main() -> int:
	parser = argparse.ArgumentParser()
	root = Path(__file__).resolve().parents[1]
	parser.add_argument("--data", default=str(root / "data" / "skills" / "actuator_il_combined.jsonl"))
	parser.add_argument("--val_data", default="", help="Optional separate validation JSONL file")
	parser.add_argument("--val_ratio", type=float, default=0.1, help="Fraction for validation split if no val_data provided")
	parser.add_argument("--model", default="google/flan-t5-small")
	parser.add_argument("--out", default=str(root / "models" / "actuator_il_t5"))
	parser.add_argument("--epochs", type=int, default=3)
	parser.add_argument("--bsz", type=int, default=4)
	parser.add_argument("--max-inp", type=int, default=512)
	parser.add_argument("--max-out", type=int, default=256)
	parser.add_argument("--seed", type=int, default=42)
	parser.add_argument("--fp16", action="store_true", help="Enable fp16 training on GPU")
	parser.add_argument("--bf16", action="store_true", help="Enable bf16 training on supported GPUs")
	parser.add_argument("--weight-decay", type=float, default=0.01)
	parser.add_argument("--ga", type=int, default=1, help="Gradient accumulation steps")
	parser.add_argument("--early-stopping", action="store_true")
	parser.add_argument("--early-patience", type=int, default=3)
	parser.add_argument("--mode", choices=["cli", "dom"], default="cli")
	parser.add_argument("--use-meta-url-hint", action="store_true", help="For DOM mode: prepend a meta url hint derived from labels to the input")
	parser.add_argument("--use-meta-selector-hint", action="store_true", help="For DOM mode: prepend a meta selector hint derived from labels to the input")
	parser.add_argument("--grad-checkpoint", action="store_true", help="Enable gradient checkpointing to save VRAM")
	parser.add_argument("--num-workers", type=int, default=4, help="DataLoader workers")
	parser.add_argument("--pin-memory", action="store_true", help="Pin DataLoader memory")
	parser.add_argument("--torch-compile", action="store_true", help="Enable torch.compile for speed (PyTorch 2.x)")
	parser.add_argument("--ewc-ref-model", default="", help="Optional path to a reference model to regularize towards (EWC)")
	parser.add_argument("--ewc-lambda", type=float, default=0.0, help="EWC penalty strength; 0 disables")
	parser.add_argument("--ewc-fisher", default="", help="Optional JSON mapping param_name->importance weight for EWC")
	args = parser.parse_args()

	# Avoid tokenizers fork warnings with dataloader workers
	os.environ.setdefault("TOKENIZERS_PARALLELISM", "false")
	set_seed(args.seed)

	out_dir = Path(args.out)
	out_dir.mkdir(parents=True, exist_ok=True)

	ds = load_dataset("json", data_files={"train": args.data})
	val_path: Optional[str] = args.val_data.strip() or None
	if val_path:
		ds_val = load_dataset("json", data_files={"validation": val_path})
		has_external_val = True
	else:
		has_external_val = False

	tokenizer = AutoTokenizer.from_pretrained(args.model)

	def preprocess(batch: Dict[str, list]):
		instruction = CLI_INSTRUCTION if args.mode == "cli" else DOM_INSTRUCTION
		prefixed_inputs = []
		if args.mode == "cli":
			prefixed_inputs = [instruction + x for x in batch["input"]]
		else:
			# For DOM mode, optionally prepend meta hints derived from gold labels (curriculum-style)
			for inp_text, out_text in zip(batch["input"], batch["output"]):
				meta_hint = ""
				if args.use_meta_url_hint or args.use_meta_selector_hint:
					try:
						obj = json.loads(out_text)
						args_obj = obj.get("args") or {}
						url = str(args_obj.get("url", "")).strip().rstrip('/')
						selector = re.sub(r"\s+", " ", str(args_obj.get("selector", "")).strip())
						hints = []
						if args.use_meta_url_hint and url:
							hints.append(f"meta_url: {url}")
						if args.use_meta_selector_hint and selector:
							hints.append(f"meta_selector: {selector}")
						if hints:
							meta_hint = "\n".join(hints) + "\n"
					except Exception:
						meta_hint = ""
				prefixed_inputs.append(instruction + (meta_hint or "") + inp_text)
		model_inputs = tokenizer(prefixed_inputs, max_length=args.max_inp, truncation=True)
		labels_out: List[str] = []
		for txt in batch["output"]:
			try:
				obj = json.loads(txt)
				if args.mode == "cli":
					argv_list = (obj.get("args") or {}).get("argv")
					if isinstance(argv_list, list):
						labels_out.append(" ".join(str(t) for t in argv_list))
					else:
						labels_out.append("")
				else:
					url = str((obj.get("args") or {}).get("url", ""))
					selector = str((obj.get("args") or {}).get("selector", ""))
					# Normalize to reduce label noise
					url = url.strip().rstrip('/')
					selector = re.sub(r"\s+", " ", selector.strip())
					labels_out.append(f"{url} {selector}".strip())
			except Exception:
				labels_out.append("")
		labels = tokenizer(text_target=labels_out, max_length=args.max_out, truncation=True)
		model_inputs["labels"] = labels["input_ids"]
		return model_inputs

	def label_non_empty(example: Dict[str, str]) -> bool:
		txt = example.get("output", "")
		try:
			obj = json.loads(txt)
		except Exception:
			return False
		if args.mode == "cli":
			argv_list = (obj.get("args") or {}).get("argv")
			return bool(isinstance(argv_list, list) and len(argv_list) > 0)
		else:
			url = str((obj.get("args") or {}).get("url", "")).strip()
			selector = str((obj.get("args") or {}).get("selector", "")).strip()
			return bool(url and selector)

	if has_external_val:
		ds_train = ds["train"].filter(label_non_empty)
		ds_validation = ds_val["validation"].filter(label_non_empty)
		tokenized_train = ds_train.map(preprocess, batched=True, remove_columns=["input", "output"])
		tokenized_val = ds_validation.map(preprocess, batched=True, remove_columns=["input", "output"])
	else:
		split = ds["train"].train_test_split(test_size=max(1e-6, min(0.5, args.val_ratio)), seed=args.seed)
		ds_train = split["train"].filter(label_non_empty)
		ds_validation = split["test"].filter(label_non_empty)
		tokenized_train = ds_train.map(preprocess, batched=True, remove_columns=["input", "output"])
		tokenized_val = ds_validation.map(preprocess, batched=True, remove_columns=["input", "output"])

	model = AutoModelForSeq2SeqLM.from_pretrained(args.model)
	# Optional gradient checkpointing
	if args.grad_checkpoint and hasattr(model, "gradient_checkpointing_enable"):
		try:
			model.gradient_checkpointing_enable()
			# use_cache must be False with gradient checkpointing to avoid warnings
			if hasattr(model, "config") and hasattr(model.config, "use_cache"):
				try:
					model.config.use_cache = False  # type: ignore[attr-defined]
				except Exception:
					pass
		except Exception:
			pass
	collator = DataCollatorForSeq2Seq(tokenizer, model=model)

	do_fp16 = bool(args.fp16 and torch is not None and getattr(torch.cuda, "is_available", lambda: False)())
	do_bf16 = bool(args.bf16 and torch is not None and getattr(torch.cuda, "is_available", lambda: False)())
	# Enable TF32 for speed on Ampere+
	try:
		if torch is not None and getattr(torch.backends, "cuda", None):
			torch.backends.cuda.matmul.allow_tf32 = True  # type: ignore[attr-defined]
			torch.backends.cudnn.allow_tf32 = True  # type: ignore[attr-defined]
	except Exception:
		pass
	save_strategy = "epoch" if args.early_stopping else "no"
	training_args = Seq2SeqTrainingArguments(
		output_dir=str(out_dir),
		per_device_train_batch_size=args.bsz,
		num_train_epochs=args.epochs,
		learning_rate=2e-4,
		weight_decay=args.weight_decay,
		gradient_accumulation_steps=args.ga,
		fp16=do_fp16,
		bf16=do_bf16,
		eval_strategy="epoch",
		metric_for_best_model="eval_loss",
		greater_is_better=False,
		load_best_model_at_end=args.early_stopping,
		save_strategy=save_strategy,
		save_total_limit=1,
		logging_steps=10,
		report_to=[],
		seed=args.seed,
		dataloader_num_workers=max(0, int(args.num_workers)),
		dataloader_pin_memory=bool(args.pin_memory),
		ddp_find_unused_parameters=False,
		optim="adamw_torch_fused",
		group_by_length=True,
		remove_unused_columns=False,
	)

	callbacks = []
	if args.early_stopping:
		callbacks.append(EarlyStoppingCallback(early_stopping_patience=max(1, int(args.early_patience))))

	# Optional torch.compile
	if args.torch_compile and torch is not None and hasattr(torch, "compile"):
		try:
			model = torch.compile(model)  # type: ignore[attr-defined]
		except Exception:
			pass

	# Optional EWC regularization: load reference weights and fisher importance
	ref_tensors = None
	fisher_importance = None
	try:
		if float(getattr(args, "ewc_lambda", 0.0) or 0.0) > 0.0 and str(args.ewc_ref_model).strip():
			ref_dir = Path(str(args.ewc_ref_model).strip())
			state = None
			try:
				# Try to load from directory with same architecture
				from safetensors.torch import load_file as _load_st  # type: ignore
			except Exception:
				_load_st = None  # type: ignore
			# Prefer safetensors if present
			cand = None
			for fn in ("pytorch_model.bin", "model.safetensors"):
				p = ref_dir / fn
				if p.exists():
					cand = p
					break
			if cand is not None:
				try:
					if cand.suffix == ".safetensors" and _load_st is not None:
						state = _load_st(str(cand))
					else:
						import torch as _torch  # type: ignore
						state = _torch.load(str(cand), map_location="cuda")
				except Exception:
					state = None
			if state is not None:
				cur = dict(model.named_parameters())
				ref_tensors = {}
				for k, v in state.items():
					if k in cur:
						try:
							ref_tensors[k] = v.detach().clone().cpu() if hasattr(v, "detach") else v
						except Exception:
							pass
			# Optional Fisher importance
			if str(args.ewc_fisher).strip():
				p = Path(str(args.ewc_fisher).strip())
				if p.exists():
					try:
						obj = json.loads(p.read_text(encoding="utf-8"))
						if isinstance(obj, dict):
							fisher_importance = {str(k): float(v) for k, v in obj.items()}
					except Exception:
						fisher_importance = None
	except Exception:
		ref_tensors = None
		fisher_importance = None

	if float(getattr(args, "ewc_lambda", 0.0) or 0.0) > 0.0 and isinstance(ref_tensors, dict) and len(ref_tensors) > 0:
		class EWCSeq2SeqTrainer(Seq2SeqTrainer):  # type: ignore
			def compute_loss(self, model, inputs, return_outputs=False):  # type: ignore
				out = model(**inputs)
				loss = out.get("loss") if isinstance(out, dict) else out[0]
				try:
					lam = float(getattr(args, "ewc_lambda", 0.0) or 0.0)
					pen = None
					for name, param in model.named_parameters():
						if not param.requires_grad:
							continue
						ref = ref_tensors.get(name) if isinstance(ref_tensors, dict) else None
						if ref is None:
							continue
						w = float(fisher_importance.get(name, 1.0)) if isinstance(fisher_importance, dict) else 1.0
						diff = (param - ref.to(param.device))
						term = (w * (diff * diff)).sum()
						pen = term if pen is None else pen + term
					if pen is not None and lam > 0.0:
						loss = loss + (lam * pen)
				except Exception:
					pass
				return (loss, out) if return_outputs else loss
		trainer = EWCSeq2SeqTrainer(
			model=model,
			args=training_args,
			train_dataset=tokenized_train,
			eval_dataset=tokenized_val,
			data_collator=collator,
			callbacks=callbacks,
		)
	else:
		trainer = Seq2SeqTrainer(
			model=model,
			args=training_args,
			train_dataset=tokenized_train,
			eval_dataset=tokenized_val,
			data_collator=collator,
			callbacks=callbacks,
		)

	trainer.train()
	# In DDP: save only from primary rank to avoid contention
	def _is_primary() -> bool:
		try:
			ws = int(os.environ.get("WORLD_SIZE", "1") or 1)
			rk = int(os.environ.get("RANK", "0") or 0)
			return (ws <= 1) or (rk == 0)
		except Exception:
			return True
	if _is_primary():
		model.save_pretrained(str(out_dir))
		tokenizer.save_pretrained(str(out_dir))
	print(f"Saved model -> {out_dir}")
	# Write lightweight training metadata
	try:
		from datetime import datetime as _dt  # type: ignore
		meta = {
			"trained_at": _dt.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ"),
			"base_model": str(args.model),
			"mode": str(args.mode),
			"epochs": int(args.epochs),
			"batch_size": int(args.bsz),
			"max_inp": int(args.max_inp),
			"max_out": int(args.max_out),
		}
		(out_dir / "metadata.json").write_text(json.dumps(meta, ensure_ascii=False, indent=2), encoding="utf-8")
	except Exception:
		pass
	# In DDP runs, cleanly destroy process group to avoid NCCL warnings
	try:
		import torch.distributed as dist  # type: ignore
		if dist.is_available() and dist.is_initialized():
			dist.destroy_process_group()
	except Exception:
		pass
	return 0

if __name__ == "__main__":
	raise SystemExit(main())


main

	Seq2SeqTrainer,
	EarlyStoppingCallback,
)
from transformers import set_seed
try:
	import torch  # type: ignore
except Exception:
	torch = None  # type: ignore

CLI_INSTRUCTION = (
	'Actuator task: Return ONLY the CLI argv as a single space-separated string. '
	'Example: wc -l docs/a.txt. No quotes, no explanations, no extra text. Input follows:\n'
)

DOM_INSTRUCTION = (
	'DOM task: Return ONLY two tokens: the URL and the CSS selector, separated by a single space. '
	'Example: https://example.com h1. No quotes, no explanations, no extra text. Input follows:\n'
)

def main() -> int:
	parser = argparse.ArgumentParser()
	root = Path(__file__).resolve().parents[1]
	parser.add_argument("--data", default=str(root / "data" / "skills" / "actuator_il_combined.jsonl"))
	parser.add_argument("--val_data", default="", help="Optional separate validation JSONL file")
	parser.add_argument("--val_ratio", type=float, default=0.1, help="Fraction for validation split if no val_data provided")
	parser.add_argument("--model", default="google/flan-t5-small")
	parser.add_argument("--out", default=str(root / "models" / "actuator_il_t5"))
	parser.add_argument("--epochs", type=int, default=3)
	parser.add_argument("--bsz", type=int, default=4)
	parser.add_argument("--max-inp", type=int, default=512)
	parser.add_argument("--max-out", type=int, default=256)
	parser.add_argument("--seed", type=int, default=42)
	parser.add_argument("--fp16", action="store_true", help="Enable fp16 training on GPU")
	parser.add_argument("--bf16", action="store_true", help="Enable bf16 training on supported GPUs")
	parser.add_argument("--weight-decay", type=float, default=0.01)
	parser.add_argument("--ga", type=int, default=1, help="Gradient accumulation steps")
	parser.add_argument("--early-stopping", action="store_true")
	parser.add_argument("--early-patience", type=int, default=3)
	parser.add_argument("--mode", choices=["cli", "dom"], default="cli")
	parser.add_argument("--use-meta-url-hint", action="store_true", help="For DOM mode: prepend a meta url hint derived from labels to the input")
	parser.add_argument("--use-meta-selector-hint", action="store_true", help="For DOM mode: prepend a meta selector hint derived from labels to the input")
	parser.add_argument("--grad-checkpoint", action="store_true", help="Enable gradient checkpointing to save VRAM")
	parser.add_argument("--num-workers", type=int, default=4, help="DataLoader workers")
	parser.add_argument("--pin-memory", action="store_true", help="Pin DataLoader memory")
	parser.add_argument("--torch-compile", action="store_true", help="Enable torch.compile for speed (PyTorch 2.x)")
	parser.add_argument("--ewc-ref-model", default="", help="Optional path to a reference model to regularize towards (EWC)")
	parser.add_argument("--ewc-lambda", type=float, default=0.0, help="EWC penalty strength; 0 disables")
	parser.add_argument("--ewc-fisher", default="", help="Optional JSON mapping param_name->importance weight for EWC")
	args = parser.parse_args()

	# Avoid tokenizers fork warnings with dataloader workers
	os.environ.setdefault("TOKENIZERS_PARALLELISM", "false")
	set_seed(args.seed)

	out_dir = Path(args.out)
	out_dir.mkdir(parents=True, exist_ok=True)

	ds = load_dataset("json", data_files={"train": args.data})
	val_path: Optional[str] = args.val_data.strip() or None
	if val_path:
		ds_val = load_dataset("json", data_files={"validation": val_path})
		has_external_val = True
	else:
		has_external_val = False

	tokenizer = AutoTokenizer.from_pretrained(args.model)

	def preprocess(batch: Dict[str, list]):
		instruction = CLI_INSTRUCTION if args.mode == "cli" else DOM_INSTRUCTION
		prefixed_inputs = []
		if args.mode == "cli":
			prefixed_inputs = [instruction + x for x in batch["input"]]
		else:
			# For DOM mode, optionally prepend meta hints derived from gold labels (curriculum-style)
			for inp_text, out_text in zip(batch["input"], batch["output"]):
				meta_hint = ""
				if args.use_meta_url_hint or args.use_meta_selector_hint:
					try:
						obj = json.loads(out_text)
						args_obj = obj.get("args") or {}
						url = str(args_obj.get("url", "")).strip().rstrip('/')
						selector = re.sub(r"\s+", " ", str(args_obj.get("selector", "")).strip())
						hints = []
						if args.use_meta_url_hint and url:
							hints.append(f"meta_url: {url}")
						if args.use_meta_selector_hint and selector:
							hints.append(f"meta_selector: {selector}")
						if hints:
							meta_hint = "\n".join(hints) + "\n"
					except Exception:
						meta_hint = ""
				prefixed_inputs.append(instruction + (meta_hint or "") + inp_text)
		model_inputs = tokenizer(prefixed_inputs, max_length=args.max_inp, truncation=True)
		labels_out: List[str] = []
		for txt in batch["output"]:
			try:
				obj = json.loads(txt)
				if args.mode == "cli":
					argv_list = (obj.get("args") or {}).get("argv")
					if isinstance(argv_list, list):
						labels_out.append(" ".join(str(t) for t in argv_list))
					else:
						labels_out.append("")
				else:
					url = str((obj.get("args") or {}).get("url", ""))
					selector = str((obj.get("args") or {}).get("selector", ""))
					# Normalize to reduce label noise
					url = url.strip().rstrip('/')
					selector = re.sub(r"\s+", " ", selector.strip())
					labels_out.append(f"{url} {selector}".strip())
			except Exception:
				labels_out.append("")
		labels = tokenizer(text_target=labels_out, max_length=args.max_out, truncation=True)
		model_inputs["labels"] = labels["input_ids"]
		return model_inputs

	def label_non_empty(example: Dict[str, str]) -> bool:
		txt = example.get("output", "")
		try:
			obj = json.loads(txt)
		except Exception:
			return False
		if args.mode == "cli":
			argv_list = (obj.get("args") or {}).get("argv")
			return bool(isinstance(argv_list, list) and len(argv_list) > 0)
		else:
			url = str((obj.get("args") or {}).get("url", "")).strip()
			selector = str((obj.get("args") or {}).get("selector", "")).strip()
			return bool(url and selector)

	if has_external_val:
		ds_train = ds["train"].filter(label_non_empty)
		ds_validation = ds_val["validation"].filter(label_non_empty)
		tokenized_train = ds_train.map(preprocess, batched=True, remove_columns=["input", "output"])
		tokenized_val = ds_validation.map(preprocess, batched=True, remove_columns=["input", "output"])
	else:
		split = ds["train"].train_test_split(test_size=max(1e-6, min(0.5, args.val_ratio)), seed=args.seed)
		ds_train = split["train"].filter(label_non_empty)
		ds_validation = split["test"].filter(label_non_empty)
		tokenized_train = ds_train.map(preprocess, batched=True, remove_columns=["input", "output"])
		tokenized_val = ds_validation.map(preprocess, batched=True, remove_columns=["input", "output"])

	model = AutoModelForSeq2SeqLM.from_pretrained(args.model)
	# Optional gradient checkpointing
	if args.grad_checkpoint and hasattr(model, "gradient_checkpointing_enable"):
		try:
			model.gradient_checkpointing_enable()
			# use_cache must be False with gradient checkpointing to avoid warnings
			if hasattr(model, "config") and hasattr(model.config, "use_cache"):
				try:
					model.config.use_cache = False  # type: ignore[attr-defined]
				except Exception:
					pass
		except Exception:
			pass
	collator = DataCollatorForSeq2Seq(tokenizer, model=model)

	do_fp16 = bool(args.fp16 and torch is not None and getattr(torch.cuda, "is_available", lambda: False)())
	do_bf16 = bool(args.bf16 and torch is not None and getattr(torch.cuda, "is_available", lambda: False)())
	# Enable TF32 for speed on Ampere+
	try:
		if torch is not None and getattr(torch.backends, "cuda", None):
			torch.backends.cuda.matmul.allow_tf32 = True  # type: ignore[attr-defined]
			torch.backends.cudnn.allow_tf32 = True  # type: ignore[attr-defined]
	except Exception:
		pass
	save_strategy = "epoch" if args.early_stopping else "no"
	training_args = Seq2SeqTrainingArguments(
		output_dir=str(out_dir),
		per_device_train_batch_size=args.bsz,
		num_train_epochs=args.epochs,
		learning_rate=2e-4,
		weight_decay=args.weight_decay,
		gradient_accumulation_steps=args.ga,
		fp16=do_fp16,
		bf16=do_bf16,
		eval_strategy="epoch",
		metric_for_best_model="eval_loss",
		greater_is_better=False,
		load_best_model_at_end=args.early_stopping,
		save_strategy=save_strategy,
		save_total_limit=1,
		logging_steps=10,
		report_to=[],
		seed=args.seed,
		dataloader_num_workers=max(0, int(args.num_workers)),
		dataloader_pin_memory=bool(args.pin_memory),
		ddp_find_unused_parameters=False,
		optim="adamw_torch_fused",
		group_by_length=True,
		remove_unused_columns=False,
	)

	callbacks = []
	if args.early_stopping:
		callbacks.append(EarlyStoppingCallback(early_stopping_patience=max(1, int(args.early_patience))))

	# Optional torch.compile
	if args.torch_compile and torch is not None and hasattr(torch, "compile"):
		try:
			model = torch.compile(model)  # type: ignore[attr-defined]
		except Exception:
			pass

	# Optional EWC regularization: load reference weights and fisher importance
	ref_tensors = None
	fisher_importance = None
	try:
		if float(getattr(args, "ewc_lambda", 0.0) or 0.0) > 0.0 and str(args.ewc_ref_model).strip():
			ref_dir = Path(str(args.ewc_ref_model).strip())
			state = None
			try:
				# Try to load from directory with same architecture
				from safetensors.torch import load_file as _load_st  # type: ignore
			except Exception:
				_load_st = None  # type: ignore
			# Prefer safetensors if present
			cand = None
			for fn in ("pytorch_model.bin", "model.safetensors"):
				p = ref_dir / fn
				if p.exists():
					cand = p
					break
			if cand is not None:
				try:
					if cand.suffix == ".safetensors" and _load_st is not None:
						state = _load_st(str(cand))
					else:
						import torch as _torch  # type: ignore
						state = _torch.load(str(cand), map_location="cuda")
				except Exception:
					state = None
			if state is not None:
				cur = dict(model.named_parameters())
				ref_tensors = {}
				for k, v in state.items():
					if k in cur:
						try:
							ref_tensors[k] = v.detach().clone().cpu() if hasattr(v, "detach") else v
						except Exception:
							pass
			# Optional Fisher importance
			if str(args.ewc_fisher).strip():
				p = Path(str(args.ewc_fisher).strip())
				if p.exists():
					try:
						obj = json.loads(p.read_text(encoding="utf-8"))
						if isinstance(obj, dict):
							fisher_importance = {str(k): float(v) for k, v in obj.items()}
					except Exception:
						fisher_importance = None
	except Exception:
		ref_tensors = None
		fisher_importance = None

	if float(getattr(args, "ewc_lambda", 0.0) or 0.0) > 0.0 and isinstance(ref_tensors, dict) and len(ref_tensors) > 0:
		class EWCSeq2SeqTrainer(Seq2SeqTrainer):  # type: ignore
			def compute_loss(self, model, inputs, return_outputs=False):  # type: ignore
				out = model(**inputs)
				loss = out.get("loss") if isinstance(out, dict) else out[0]
				try:
					lam = float(getattr(args, "ewc_lambda", 0.0) or 0.0)
					pen = None
					for name, param in model.named_parameters():
						if not param.requires_grad:
							continue
						ref = ref_tensors.get(name) if isinstance(ref_tensors, dict) else None
						if ref is None:
							continue
						w = float(fisher_importance.get(name, 1.0)) if isinstance(fisher_importance, dict) else 1.0
						diff = (param - ref.to(param.device))
						term = (w * (diff * diff)).sum()
						pen = term if pen is None else pen + term
					if pen is not None and lam > 0.0:
						loss = loss + (lam * pen)
				except Exception:
					pass
				return (loss, out) if return_outputs else loss
		trainer = EWCSeq2SeqTrainer(
			model=model,
			args=training_args,
			train_dataset=tokenized_train,
			eval_dataset=tokenized_val,
			data_collator=collator,
			callbacks=callbacks,
		)
	else:
		trainer = Seq2SeqTrainer(
			model=model,
			args=training_args,
			train_dataset=tokenized_train,
			eval_dataset=tokenized_val,
			data_collator=collator,
			callbacks=callbacks,
		)

	trainer.train()
	# In DDP: save only from primary rank to avoid contention
	def _is_primary() -> bool:
		try:
			ws = int(os.environ.get("WORLD_SIZE", "1") or 1)
			rk = int(os.environ.get("RANK", "0") or 0)
			return (ws <= 1) or (rk == 0)
		except Exception:
			return True
	if _is_primary():
		model.save_pretrained(str(out_dir))
		tokenizer.save_pretrained(str(out_dir))
	print(f"Saved model -> {out_dir}")
	# Write lightweight training metadata
	try:
		from datetime import datetime as _dt  # type: ignore
		meta = {
			"trained_at": _dt.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ"),
			"base_model": str(args.model),
			"mode": str(args.mode),
			"epochs": int(args.epochs),
			"batch_size": int(args.bsz),
			"max_inp": int(args.max_inp),
			"max_out": int(args.max_out),
		}
		(out_dir / "metadata.json").write_text(json.dumps(meta, ensure_ascii=False, indent=2), encoding="utf-8")
	except Exception:
		pass
	# In DDP runs, cleanly destroy process group to avoid NCCL warnings
	try:
		import torch.distributed as dist  # type: ignore
		if dist.is_available() and dist.is_initialized():
			dist.destroy_process_group()
	except Exception:
		pass
	return 0

if __name__ == "__main__":
	raise SystemExit(main())


preprocess

	parser.add_argument("--ewc-fisher", default="", help="Optional JSON mapping param_name->importance weight for EWC")
	args = parser.parse_args()

	# Avoid tokenizers fork warnings with dataloader workers
	os.environ.setdefault("TOKENIZERS_PARALLELISM", "false")
	set_seed(args.seed)

	out_dir = Path(args.out)
	out_dir.mkdir(parents=True, exist_ok=True)

	ds = load_dataset("json", data_files={"train": args.data})
	val_path: Optional[str] = args.val_data.strip() or None
	if val_path:
		ds_val = load_dataset("json", data_files={"validation": val_path})
		has_external_val = True
	else:
		has_external_val = False

	tokenizer = AutoTokenizer.from_pretrained(args.model)

	def preprocess(batch: Dict[str, list]):
		instruction = CLI_INSTRUCTION if args.mode == "cli" else DOM_INSTRUCTION
		prefixed_inputs = []
		if args.mode == "cli":
			prefixed_inputs = [instruction + x for x in batch["input"]]
		else:
			# For DOM mode, optionally prepend meta hints derived from gold labels (curriculum-style)
			for inp_text, out_text in zip(batch["input"], batch["output"]):
				meta_hint = ""
				if args.use_meta_url_hint or args.use_meta_selector_hint:
					try:
						obj = json.loads(out_text)
						args_obj = obj.get("args") or {}
						url = str(args_obj.get("url", "")).strip().rstrip('/')
						selector = re.sub(r"\s+", " ", str(args_obj.get("selector", "")).strip())
						hints = []
						if args.use_meta_url_hint and url:
							hints.append(f"meta_url: {url}")
						if args.use_meta_selector_hint and selector:
							hints.append(f"meta_selector: {selector}")
						if hints:
							meta_hint = "\n".join(hints) + "\n"
					except Exception:
						meta_hint = ""
				prefixed_inputs.append(instruction + (meta_hint or "") + inp_text)
		model_inputs = tokenizer(prefixed_inputs, max_length=args.max_inp, truncation=True)
		labels_out: List[str] = []
		for txt in batch["output"]:
			try:
				obj = json.loads(txt)
				if args.mode == "cli":
					argv_list = (obj.get("args") or {}).get("argv")
					if isinstance(argv_list, list):
						labels_out.append(" ".join(str(t) for t in argv_list))
					else:
						labels_out.append("")
				else:
					url = str((obj.get("args") or {}).get("url", ""))
					selector = str((obj.get("args") or {}).get("selector", ""))
					# Normalize to reduce label noise
					url = url.strip().rstrip('/')
					selector = re.sub(r"\s+", " ", selector.strip())
					labels_out.append(f"{url} {selector}".strip())
			except Exception:
				labels_out.append("")
		labels = tokenizer(text_target=labels_out, max_length=args.max_out, truncation=True)
		model_inputs["labels"] = labels["input_ids"]
		return model_inputs

	def label_non_empty(example: Dict[str, str]) -> bool:
		txt = example.get("output", "")
		try:
			obj = json.loads(txt)
		except Exception:
			return False
		if args.mode == "cli":
			argv_list = (obj.get("args") or {}).get("argv")
			return bool(isinstance(argv_list, list) and len(argv_list) > 0)
		else:
			url = str((obj.get("args") or {}).get("url", "")).strip()
			selector = str((obj.get("args") or {}).get("selector", "")).strip()
			return bool(url and selector)

	if has_external_val:
		ds_train = ds["train"].filter(label_non_empty)
		ds_validation = ds_val["validation"].filter(label_non_empty)
		tokenized_train = ds_train.map(preprocess, batched=True, remove_columns=["input", "output"])
		tokenized_val = ds_validation.map(preprocess, batched=True, remove_columns=["input", "output"])