from pathlib import Path
from typing import Any, Dict, Iterable, List

from agi_dw.bench.common.trace import write_jsonl

def iter_jsonl(path: Path) -> Iterable[Dict[str, Any]]:
	if not path.exists():
		return
	with path.open("r", encoding="utf-8") as f:
		for line in f:
			line = line.strip()
			if not line:
				continue
			try:
				yield json.loads(line)
			except Exception:
				continue

def main() -> int:
	ap = argparse.ArgumentParser()
	root = Path(__file__).resolve().parents[1]
	ap.add_argument("--inputs", nargs="*", default=[
		str(root / "data" / "traces" / "seed_os_cli.jsonl"),
		str(root / "data" / "traces" / "seed_os_cli.verified.llm.jsonl"),
		str(root / "data" / "traces" / "loop_run.jsonl"),
		str(root / "data" / "traces" / "loop_web_dom.jsonl"),
		str(root / "data" / "traces" / "web_dom.verified.jsonl"),
	])
	ap.add_argument("--out", default=str(root / "data" / "traces" / "all.jsonl"))
	args = ap.parse_args()

	out = Path(args.out)
	out.parent.mkdir(parents=True, exist_ok=True)

	seen_keys: set[str] = set()
	count = 0
	for raw in args.inputs:
		p = Path(raw)
		for obj in iter_jsonl(p):
			# Dedup by coarse key: (task_id, result.status, critique.risk rounded)
			tid = str(obj.get("task_id", ""))
			status = str(((obj.get("result") or {}).get("status", "")))
			try:
				risk = float(((obj.get("critique") or {}).get("risk", 0.0)))
			except Exception:
				risk = 0.0
			key = f"{tid}|{status}|{round(risk, 2)}"
			if key in seen_keys:
				continue
			seen_keys.add(key)
			write_jsonl(args.out, obj)
			count += 1
	print(json.dumps({"merged": count, "inputs": len(args.inputs), "out": str(out)}))
	return 0

if __name__ == "__main__":
	import sys
	sys.exit(main())


#!/usr/bin/env python3
import logging
import argparse
import json
import sys
from pathlib import Path
from typing import Any, Dict, List, Tuple

def load_json_with_dupe_detection(path: Path) -> Tuple[Dict[str, Any], List[str]]:
	dups: List[str] = []

	def hook(pairs: List[Tuple[str, Any]]) -> Dict[str, Any]:
		seen: Dict[str, Any] = {}
		for k, v in pairs:
			if k in seen:
				dups.append(k)
			seen[k] = v  # keep last
		return seen

	obj = json.loads(path.read_text(encoding="utf-8"), object_pairs_hook=hook)
	return obj, dups

def dedupe_json_file(path: Path, out: Path | None, sort_keys: bool) -> Dict[str, Any]:
	obj, dups = load_json_with_dupe_detection(path)
	out_path = out or path
	out_path.write_text(json.dumps(obj, ensure_ascii=False, indent=2, sort_keys=sort_keys) + "\n", encoding="utf-8")
	print(json.dumps({"ok": True, "path": str(out_path), "dupe_keys_removed": sorted(set(dups))}))
	return obj

def stable_dumps(o: Any) -> str:
	return json.dumps(o, ensure_ascii=False, sort_keys=True, separators=(",", ":"))

def dedupe_jsonl_file(path: Path, out: Path | None, by_key: str | None) -> int:
	lines = path.read_text(encoding="utf-8").splitlines()
	# Keep last occurrence per key
	index: Dict[str, str] = {}
	order: List[str] = []
	for ln in lines:
		ln = ln.strip()
		if not ln:
			continue
		try:
			obj = json.loads(ln)
		except Exception:
			# treat as raw string key
			key = ln
		else:
			if by_key and isinstance(obj, dict) and by_key in obj:
				key = f"key::{obj[by_key]!r}"
			else:
				key = f"hash::{stable_dumps(obj)}"
		# If first time seeing key, record order
		if key not in index:
			order.append(key)
		index[key] = ln
	# Build output keeping last occurrence for each key
	out_lines: List[str] = [index[k] for k in order if k in index]
	out_path = out or path
	out_path.write_text("\n".join(out_lines) + ("\n" if out_lines else ""), encoding="utf-8")
	print(json.dumps({"ok": True, "path": str(out_path), "kept": len(out_lines), "dropped": len(lines) - len(out_lines)}))
	return len(out_lines)

def parse_args() -> argparse.Namespace:
	ap = argparse.ArgumentParser(description="Deduplicate JSON and JSONL files")
	ap.add_argument("paths", nargs="+", help="One or more .json or .jsonl files")
	ap.add_argument("--out", default=None, help="Optional output file (only valid with single input)")
	ap.add_argument("--by-key", default=None, help="For JSONL, field name to dedupe by (default: content hash; tries 'id' if present)")
	ap.add_argument("--sort-keys", action="store_true", help="Sort keys when rewriting JSON")
	return ap.parse_args()

def main() -> int:
	args = parse_args()
	out_path = Path(args.out) if args.out else None
	if out_path and len(args.paths) != 1:
		print(json.dumps({"ok": False, "error": "out_requires_single_input"}))
		return 2
	rc = 0
	for p in args.paths:
		path = Path(p)
		if not path.exists():
			print(json.dumps({"ok": False, "error": "missing", "path": str(path)}))
			rc = 2
			continue
		if path.suffix.lower() == ".json":
			try:
				dedupe_json_file(path, out_path, bool(args.sort_keys))
			except Exception as e:
				print(json.dumps({"ok": False, "error": "json_dedupe_failed", "path": str(path), "detail": str(e)}))
				rc = 1
		elif path.suffix.lower() == ".jsonl":
			try:
				by_key = args.by_key
				if by_key is None:
					# try 'id' if present in any line
					try:
						for ln in path.read_text(encoding="utf-8").splitlines():
							if not ln.strip():
								continue
							obj = json.loads(ln)
							if isinstance(obj, dict) and "id" in obj:
								by_key = "id"
								break
					except Exception:
						by_key = None
				dedupe_jsonl_file(path, out_path, by_key)
			except Exception as e:
				print(json.dumps({"ok": False, "error": "jsonl_dedupe_failed", "path": str(path), "detail": str(e)}))
				rc = 1
		else:
			print(json.dumps({"ok": False, "error": "unsupported_extension", "path": str(path)}))
			rc = 2
	return rc

if __name__ == "__main__":
	raise SystemExit(main())


#!/usr/bin/env python3
import logging
import argparse
import json
import sys
from pathlib import Path
from typing import Any, Dict, List, Tuple

def load_json_with_dupe_detection(path: Path) -> Tuple[Dict[str, Any], List[str]]:
	dups: List[str] = []

	def hook(pairs: List[Tuple[str, Any]]) -> Dict[str, Any]:
		seen: Dict[str, Any] = {}
		for k, v in pairs:
			if k in seen:
				dups.append(k)
			seen[k] = v  # keep last
		return seen

	obj = json.loads(path.read_text(encoding="utf-8"), object_pairs_hook=hook)
	return obj, dups

def dedupe_json_file(path: Path, out: Path | None, sort_keys: bool) -> Dict[str, Any]:
	obj, dups = load_json_with_dupe_detection(path)
	out_path = out or path
	out_path.write_text(json.dumps(obj, ensure_ascii=False, indent=2, sort_keys=sort_keys) + "\n", encoding="utf-8")
	print(json.dumps({"ok": True, "path": str(out_path), "dupe_keys_removed": sorted(set(dups))}))
	return obj

def stable_dumps(o: Any) -> str:
	return json.dumps(o, ensure_ascii=False, sort_keys=True, separators=(",", ":"))

def dedupe_jsonl_file(path: Path, out: Path | None, by_key: str | None) -> int:
	lines = path.read_text(encoding="utf-8").splitlines()
	# Keep last occurrence per key
	index: Dict[str, str] = {}
	order: List[str] = []
	for ln in lines:


import sys
from pathlib import Path
from typing import Any, Dict, List, Tuple

def load_json_with_dupe_detection(path: Path) -> Tuple[Dict[str, Any], List[str]]:
	dups: List[str] = []

	def hook(pairs: List[Tuple[str, Any]]) -> Dict[str, Any]:
		seen: Dict[str, Any] = {}
		for k, v in pairs:
			if k in seen:
				dups.append(k)
			seen[k] = v  # keep last
		return seen

	obj = json.loads(path.read_text(encoding="utf-8"), object_pairs_hook=hook)
	return obj, dups

def dedupe_json_file(path: Path, out: Path | None, sort_keys: bool) -> Dict[str, Any]:
	obj, dups = load_json_with_dupe_detection(path)
	out_path = out or path
	out_path.write_text(json.dumps(obj, ensure_ascii=False, indent=2, sort_keys=sort_keys) + "\n", encoding="utf-8")
	print(json.dumps({"ok": True, "path": str(out_path), "dupe_keys_removed": sorted(set(dups))}))
	return obj

def stable_dumps(o: Any) -> str:
	return json.dumps(o, ensure_ascii=False, sort_keys=True, separators=(",", ":"))

def dedupe_jsonl_file(path: Path, out: Path | None, by_key: str | None) -> int:
	lines = path.read_text(encoding="utf-8").splitlines()
	# Keep last occurrence per key
	index: Dict[str, str] = {}
	order: List[str] = []
	for ln in lines:
		ln = ln.strip()
		if not ln:
			continue
		try:
			obj = json.loads(ln)
		except Exception:
			# treat as raw string key
			key = ln


	def hook(pairs: List[Tuple[str, Any]]) -> Dict[str, Any]:
		seen: Dict[str, Any] = {}
		for k, v in pairs:
			if k in seen:
				dups.append(k)
			seen[k] = v  # keep last
		return seen

	obj = json.loads(path.read_text(encoding="utf-8"), object_pairs_hook=hook)
	return obj, dups

def dedupe_json_file(path: Path, out: Path | None, sort_keys: bool) -> Dict[str, Any]:
	obj, dups = load_json_with_dupe_detection(path)
	out_path = out or path
	out_path.write_text(json.dumps(obj, ensure_ascii=False, indent=2, sort_keys=sort_keys) + "\n", encoding="utf-8")
	print(json.dumps({"ok": True, "path": str(out_path), "dupe_keys_removed": sorted(set(dups))}))
	return obj

def stable_dumps(o: Any) -> str:
	return json.dumps(o, ensure_ascii=False, sort_keys=True, separators=(",", ":"))

def dedupe_jsonl_file(path: Path, out: Path | None, by_key: str | None) -> int:
	lines = path.read_text(encoding="utf-8").splitlines()
	# Keep last occurrence per key
	index: Dict[str, str] = {}
	order: List[str] = []
	for ln in lines:
		ln = ln.strip()
		if not ln:
			continue
		try:
			obj = json.loads(ln)
		except Exception:
			# treat as raw string key
			key = ln
		else:
			if by_key and isinstance(obj, dict) and by_key in obj:
				key = f"key::{obj[by_key]!r}"
			else:


				dups.append(k)
			seen[k] = v  # keep last
		return seen

	obj = json.loads(path.read_text(encoding="utf-8"), object_pairs_hook=hook)
	return obj, dups

def dedupe_json_file(path: Path, out: Path | None, sort_keys: bool) -> Dict[str, Any]:
	obj, dups = load_json_with_dupe_detection(path)
	out_path = out or path
	out_path.write_text(json.dumps(obj, ensure_ascii=False, indent=2, sort_keys=sort_keys) + "\n", encoding="utf-8")
	print(json.dumps({"ok": True, "path": str(out_path), "dupe_keys_removed": sorted(set(dups))}))
	return obj

def stable_dumps(o: Any) -> str:
	return json.dumps(o, ensure_ascii=False, sort_keys=True, separators=(",", ":"))

def dedupe_jsonl_file(path: Path, out: Path | None, by_key: str | None) -> int:
	lines = path.read_text(encoding="utf-8").splitlines()
	# Keep last occurrence per key
	index: Dict[str, str] = {}
	order: List[str] = []
	for ln in lines:
		ln = ln.strip()
		if not ln:
			continue
		try:
			obj = json.loads(ln)
		except Exception:
			# treat as raw string key
			key = ln
		else:
			if by_key and isinstance(obj, dict) and by_key in obj:
				key = f"key::{obj[by_key]!r}"
			else:
				key = f"hash::{stable_dumps(obj)}"
		# If first time seeing key, record order
		if key not in index:
			order.append(key)
		index[key] = ln
	# Build output keeping last occurrence for each key
	out_lines: List[str] = [index[k] for k in order if k in index]
	out_path = out or path
	out_path.write_text("\n".join(out_lines) + ("\n" if out_lines else ""), encoding="utf-8")
	print(json.dumps({"ok": True, "path": str(out_path), "kept": len(out_lines), "dropped": len(lines) - len(out_lines)}))
	return len(out_lines)

def parse_args() -> argparse.Namespace:
	ap = argparse.ArgumentParser(description="Deduplicate JSON and JSONL files")
	ap.add_argument("paths", nargs="+", help="One or more .json or .jsonl files")
	ap.add_argument("--out", default=None, help="Optional output file (only valid with single input)")
	ap.add_argument("--by-key", default=None, help="For JSONL, field name to dedupe by (default: content hash; tries 'id' if present)")
	ap.add_argument("--sort-keys", action="store_true", help="Sort keys when rewriting JSON")
	return ap.parse_args()

def main() -> int:
	args = parse_args()
	out_path = Path(args.out) if args.out else None
	if out_path and len(args.paths) != 1:
		print(json.dumps({"ok": False, "error": "out_requires_single_input"}))
		return 2
	rc = 0
	for p in args.paths:
		path = Path(p)


		except Exception:
			# treat as raw string key
			key = ln
		else:
			if by_key and isinstance(obj, dict) and by_key in obj:
				key = f"key::{obj[by_key]!r}"
			else:
				key = f"hash::{stable_dumps(obj)}"
		# If first time seeing key, record order
		if key not in index:
			order.append(key)
		index[key] = ln
	# Build output keeping last occurrence for each key
	out_lines: List[str] = [index[k] for k in order if k in index]
	out_path = out or path
	out_path.write_text("\n".join(out_lines) + ("\n" if out_lines else ""), encoding="utf-8")
	print(json.dumps({"ok": True, "path": str(out_path), "kept": len(out_lines), "dropped": len(lines) - len(out_lines)}))
	return len(out_lines)

def parse_args() -> argparse.Namespace:
	ap = argparse.ArgumentParser(description="Deduplicate JSON and JSONL files")
	ap.add_argument("paths", nargs="+", help="One or more .json or .jsonl files")
	ap.add_argument("--out", default=None, help="Optional output file (only valid with single input)")
	ap.add_argument("--by-key", default=None, help="For JSONL, field name to dedupe by (default: content hash; tries 'id' if present)")
	ap.add_argument("--sort-keys", action="store_true", help="Sort keys when rewriting JSON")
	return ap.parse_args()

def main() -> int:
	args = parse_args()
	out_path = Path(args.out) if args.out else None
	if out_path and len(args.paths) != 1:
		print(json.dumps({"ok": False, "error": "out_requires_single_input"}))
		return 2
	rc = 0
	for p in args.paths:
		path = Path(p)
		if not path.exists():
			print(json.dumps({"ok": False, "error": "missing", "path": str(path)}))
			rc = 2
			continue
		if path.suffix.lower() == ".json":
			try:
				dedupe_json_file(path, out_path, bool(args.sort_keys))
			except Exception as e:
				print(json.dumps({"ok": False, "error": "json_dedupe_failed", "path": str(path), "detail": str(e)}))


main

		if key not in index:
			order.append(key)
		index[key] = ln
	# Build output keeping last occurrence for each key
	out_lines: List[str] = [index[k] for k in order if k in index]
	out_path = out or path
	out_path.write_text("\n".join(out_lines) + ("\n" if out_lines else ""), encoding="utf-8")
	print(json.dumps({"ok": True, "path": str(out_path), "kept": len(out_lines), "dropped": len(lines) - len(out_lines)}))
	return len(out_lines)

def parse_args() -> argparse.Namespace:
	ap = argparse.ArgumentParser(description="Deduplicate JSON and JSONL files")
	ap.add_argument("paths", nargs="+", help="One or more .json or .jsonl files")
	ap.add_argument("--out", default=None, help="Optional output file (only valid with single input)")
	ap.add_argument("--by-key", default=None, help="For JSONL, field name to dedupe by (default: content hash; tries 'id' if present)")
	ap.add_argument("--sort-keys", action="store_true", help="Sort keys when rewriting JSON")
	return ap.parse_args()

def main() -> int:
	args = parse_args()
	out_path = Path(args.out) if args.out else None
	if out_path and len(args.paths) != 1:
		print(json.dumps({"ok": False, "error": "out_requires_single_input"}))
		return 2
	rc = 0
	for p in args.paths:
		path = Path(p)
		if not path.exists():
			print(json.dumps({"ok": False, "error": "missing", "path": str(path)}))
			rc = 2
			continue
		if path.suffix.lower() == ".json":
			try:
				dedupe_json_file(path, out_path, bool(args.sort_keys))
			except Exception as e:
				print(json.dumps({"ok": False, "error": "json_dedupe_failed", "path": str(path), "detail": str(e)}))
				rc = 1
		elif path.suffix.lower() == ".jsonl":
			try:
				by_key = args.by_key
				if by_key is None:
					# try 'id' if present in any line
					try:
						for ln in path.read_text(encoding="utf-8").splitlines():
							if not ln.strip():
								continue
							obj = json.loads(ln)
							if isinstance(obj, dict) and "id" in obj:
								by_key = "id"
								break
					except Exception:
						by_key = None
				dedupe_jsonl_file(path, out_path, by_key)
			except Exception as e:
				print(json.dumps({"ok": False, "error": "jsonl_dedupe_failed", "path": str(path), "detail": str(e)}))
				rc = 1
		else:
			print(json.dumps({"ok": False, "error": "unsupported_extension", "path": str(path)}))
			rc = 2
	return rc

if __name__ == "__main__":
	raise SystemExit(main())


hook

#!/usr/bin/env python3
import logging
import argparse
import json
import sys
from pathlib import Path
from typing import Any, Dict, List, Tuple

def load_json_with_dupe_detection(path: Path) -> Tuple[Dict[str, Any], List[str]]:
	dups: List[str] = []

	def hook(pairs: List[Tuple[str, Any]]) -> Dict[str, Any]:
		seen: Dict[str, Any] = {}
		for k, v in pairs:
			if k in seen:
				dups.append(k)
			seen[k] = v  # keep last
		return seen

	obj = json.loads(path.read_text(encoding="utf-8"), object_pairs_hook=hook)
	return obj, dups

def dedupe_json_file(path: Path, out: Path | None, sort_keys: bool) -> Dict[str, Any]:
	obj, dups = load_json_with_dupe_detection(path)
	out_path = out or path
	out_path.write_text(json.dumps(obj, ensure_ascii=False, indent=2, sort_keys=sort_keys) + "\n", encoding="utf-8")
	print(json.dumps({"ok": True, "path": str(out_path), "dupe_keys_removed": sorted(set(dups))}))
	return obj

def stable_dumps(o: Any) -> str:
	return json.dumps(o, ensure_ascii=False, sort_keys=True, separators=(",", ":"))

def dedupe_jsonl_file(path: Path, out: Path | None, by_key: str | None) -> int:
	lines = path.read_text(encoding="utf-8").splitlines()
	# Keep last occurrence per key


from __future__ import annotations
import logging

import json
import os
import subprocess
from datetime import datetime
from pathlib import Path
import importlib

def write_text(path: Path, text: str) -> None:
	path.parent.mkdir(parents=True, exist_ok=True)
	path.write_text(text, encoding="utf-8")

def run(cmd: list[str]) -> str:
	try:
		p = subprocess.run(cmd, capture_output=True, text=True, check=False)
		return (p.stdout or "") + ("\n" + p.stderr if p.stderr else "")
	except Exception as e:
		return f"<error: {e}>\n"

def main() -> int:
	root = Path(__file__).resolve().parents[1]
	out_dir = root / "ENV"
	out_dir.mkdir(parents=True, exist_ok=True)

	ts = datetime.utcnow().strftime("%Y%m%dT%H%M%SZ")
	brief = [f"# Snapshot at {ts} (UTC)", "", "## Python", run(["python3", "-V"]).strip(), ""]
	write_text(out_dir / "ENVIRONMENT_BRIEF.md", "\n".join(brief) + "\n")

	# Key versions
	mods = "torch transformers datasets sentence_transformers faiss peft bitsandbytes jsonschema sklearn accelerate".split()
	out: dict[str, str | None] = {}
	for m in mods:
		name = m.replace("-", "_")
		try:
			mod = importlib.import_module(name)
			out[m] = getattr(mod, "__version__", "?")
		except Exception:
			out[m] = None
	write_text(out_dir / "key_versions.json", json.dumps(out, indent=2))

	# Pip/conda
	write_text(out_dir / "pip_freeze.txt", run(["pip", "freeze"]))
	write_text(out_dir / "conda_env.yaml", run(["bash", "-lc", "conda env export || true"]))
	write_text(out_dir / "conda_explicit.txt", run(["bash", "-lc", "conda list --explicit || true"]))

	# System
	sys_info = ["# System", run(["uname", "-a"]).strip(), "", "# NVIDIA", run(["bash", "-lc", "nvidia-smi || true"]).strip()]
	write_text(out_dir / "system.txt", "\n".join(sys_info) + "\n")

	print(f"Saved environment snapshot -> {out_dir}")
	return 0

if __name__ == "__main__":
	raise SystemExit(main())


from __future__ import annotations
import logging

import json
import os
import subprocess
from datetime import datetime
from pathlib import Path
import importlib

def write_text(path: Path, text: str) -> None:
	path.parent.mkdir(parents=True, exist_ok=True)
	path.write_text(text, encoding="utf-8")

def run(cmd: list[str]) -> str:
	try:
		p = subprocess.run(cmd, capture_output=True, text=True, check=False)
		return (p.stdout or "") + ("\n" + p.stderr if p.stderr else "")
	except Exception as e:
		return f"<error: {e}>\n"

def main() -> int:
	root = Path(__file__).resolve().parents[1]
	out_dir = root / "ENV"
	out_dir.mkdir(parents=True, exist_ok=True)

	ts = datetime.utcnow().strftime("%Y%m%dT%H%M%SZ")
	brief = [f"# Snapshot at {ts} (UTC)", "", "## Python", run(["python3", "-V"]).strip(), ""]
	write_text(out_dir / "ENVIRONMENT_BRIEF.md", "\n".join(brief) + "\n")

	# Key versions


run

from __future__ import annotations
import logging

import json
import os
import subprocess
from datetime import datetime
from pathlib import Path
import importlib

def write_text(path: Path, text: str) -> None:
	path.parent.mkdir(parents=True, exist_ok=True)
	path.write_text(text, encoding="utf-8")

def run(cmd: list[str]) -> str:
	try:
		p = subprocess.run(cmd, capture_output=True, text=True, check=False)
		return (p.stdout or "") + ("\n" + p.stderr if p.stderr else "")
	except Exception as e:
		return f"<error: {e}>\n"

def main() -> int:
	root = Path(__file__).resolve().parents[1]
	out_dir = root / "ENV"
	out_dir.mkdir(parents=True, exist_ok=True)

	ts = datetime.utcnow().strftime("%Y%m%dT%H%M%SZ")
	brief = [f"# Snapshot at {ts} (UTC)", "", "## Python", run(["python3", "-V"]).strip(), ""]
	write_text(out_dir / "ENVIRONMENT_BRIEF.md", "\n".join(brief) + "\n")

	# Key versions
	mods = "torch transformers datasets sentence_transformers faiss peft bitsandbytes jsonschema sklearn accelerate".split()
	out: dict[str, str | None] = {}
	for m in mods:
		name = m.replace("-", "_")
		try:
			mod = importlib.import_module(name)
			out[m] = getattr(mod, "__version__", "?")
		except Exception:


main

import os
import subprocess
from datetime import datetime
from pathlib import Path
import importlib

def write_text(path: Path, text: str) -> None:
	path.parent.mkdir(parents=True, exist_ok=True)
	path.write_text(text, encoding="utf-8")

def run(cmd: list[str]) -> str:
	try:
		p = subprocess.run(cmd, capture_output=True, text=True, check=False)
		return (p.stdout or "") + ("\n" + p.stderr if p.stderr else "")
	except Exception as e:
		return f"<error: {e}>\n"

def main() -> int:
	root = Path(__file__).resolve().parents[1]
	out_dir = root / "ENV"
	out_dir.mkdir(parents=True, exist_ok=True)

	ts = datetime.utcnow().strftime("%Y%m%dT%H%M%SZ")
	brief = [f"# Snapshot at {ts} (UTC)", "", "## Python", run(["python3", "-V"]).strip(), ""]
	write_text(out_dir / "ENVIRONMENT_BRIEF.md", "\n".join(brief) + "\n")

	# Key versions
	mods = "torch transformers datasets sentence_transformers faiss peft bitsandbytes jsonschema sklearn accelerate".split()
	out: dict[str, str | None] = {}
	for m in mods:
		name = m.replace("-", "_")
		try:
			mod = importlib.import_module(name)
			out[m] = getattr(mod, "__version__", "?")
		except Exception:
			out[m] = None
	write_text(out_dir / "key_versions.json", json.dumps(out, indent=2))

	# Pip/conda
	write_text(out_dir / "pip_freeze.txt", run(["pip", "freeze"]))
	write_text(out_dir / "conda_env.yaml", run(["bash", "-lc", "conda env export || true"]))
	write_text(out_dir / "conda_explicit.txt", run(["bash", "-lc", "conda list --explicit || true"]))

	# System
	sys_info = ["# System", run(["uname", "-a"]).strip(), "", "# NVIDIA", run(["bash", "-lc", "nvidia-smi || true"]).strip()]
	write_text(out_dir / "system.txt", "\n".join(sys_info) + "\n")

	print(f"Saved environment snapshot -> {out_dir}")
	return 0

if __name__ == "__main__":
	raise SystemExit(main())


def add(x, y):
	# Bug: should be x + y
	return x - y


from foo import add
def test_add():
	assert add(2, 3) == 5


#!/usr/bin/env python3
from __future__ import annotations
import logging

import argparse
import json
import math
import re
from pathlib import Path
from typing import Dict, Any, List, Tuple
import json
import sys
from pathlib import Path

TOKEN_RE = re.compile(r"[A-Za-z0-9_]+")

def tokenize(q: str) -> List[str]:
	return [t.lower() for t in TOKEN_RE.findall(q or "")]

def load_bm25(path: Path) -> Dict[str, Any]:
	try:
		return json.loads(path.read_text(encoding="utf-8"))
	except Exception:
		return {}

def bm25_score(query: List[str], bm: Dict[str, Any], doc: str, k1: float = 1.2, b: float = 0.75) -> float:
	postings = bm.get("postings", {})
	df = bm.get("df", {})
	docs = bm.get("docs", {})
	avgdl = float(bm.get("avgdl", 1.0) or 1.0)
	N = int(bm.get("N", max(1, len(docs))))
	dl = float(docs.get(doc, 0) or 0)
	score = 0.0
	for term in query:
		dft = int(df.get(term, 0) or 0)
		if dft <= 0:
			continue
		idf = math.log(1 + (N - dft + 0.5) / (dft + 0.5))
		tf = float(postings.get(term, {}).get(doc, 0) or 0)
		if tf <= 0:
			continue
		num = tf * (k1 + 1)
		den = tf + k1 * (1 - b + b * (dl / max(1.0, avgdl)))
		score += idf * (num / max(1e-9, den))
	return float(score)

def search_bm25(index_path: Path, query_text: str, k: int = 20) -> List[Dict[str, Any]]:
	bm = load_bm25(index_path)
	terms = tokenize(query_text)
	docs = list((bm.get("docs") or {}).keys())
	scored: List[Tuple[str, float]] = []
	for d in docs:
		s = bm25_score(terms, bm, d)
		if s > 0:
			scored.append((d, s))
	scored.sort(key=lambda x: x[1], reverse=True)
	results = [{"path": d, "score": float(s)} for d, s in scored[:k]]
	# Fallback: path token match if BM25 yields nothing
	if not results:
		path_scores: List[Tuple[str, float]] = []
		ltokens = [t for t in terms if t]
		for d in docs:
			p = d.lower()
			s = 0.0
			for t in ltokens:
				if t and t in p:
					s += 1.0
			if s > 0:
				path_scores.append((d, s))
		# Prefer markdown if querying for readme/docs
		if path_scores:
			path_scores.sort(key=lambda x: (0 if x[0].lower().endswith('.md') else 1, -x[1]))
			results = [{"path": d, "score": float(s)} for d, s in path_scores[:k]]
	# Last-resort heuristics for README/summary/overview queries
	if not results:
		q = " ".join(terms)
		want_readme = any(tok in ("readme", "intro", "introduction", "overview") for tok in terms)
		want_summary = any(tok in ("summary", "summarize", "synopsis") for tok in terms)
		candidates: List[Tuple[str, float]] = []
		for d in docs:
			name = d.rsplit("/", 1)[-1].lower()
			score = 0.0
			if want_readme and name in ("readme.md", "readme"):
				score = 3.0
			elif want_summary and name in ("summary.md", "summary"):
				score = 3.0
			elif name in ("index.md", "overview.md", "introduction.md", "intro.md"):
				score = 2.0
			elif name.endswith(".md") and ("docs/" in d or "/docs/" in d):
				score = 1.0
			if score > 0.0:
				candidates.append((d, score))
		if candidates:
			candidates.sort(key=lambda x: (-x[1], x[0]))
			results = [{"path": d, "score": float(s)} for d, s in candidates[:k]]
	return results

def _cosine(a: List[float], b: List[float]) -> float:
	# inputs assumed normalized
	return float(sum(x*y for x, y in zip(a, b)))

def _load_embed(path: Path) -> Dict[str, Any]:
	try:
		return json.loads(path.read_text(encoding="utf-8"))
	except Exception:
		return {}

def _encode_query(q: str, model_name: str) -> List[float]:
	try:
		from sentence_transformers import SentenceTransformer  # type: ignore
		model = SentenceTransformer(model_name)
		vec = model.encode([q], normalize_embeddings=True)
		return [float(x) for x in (vec[0] if hasattr(vec, "__getitem__") else vec)]
	except Exception:
		return []

def main() -> int:
	ap = argparse.ArgumentParser(description="Search BM25 index for relevant files")
	root = Path(__file__).resolve().parents[2]
	ap.add_argument("--index", default=str(root / "data" / "sandbox" / "tmp" / "bm25_index.json"))
	ap.add_argument("--question", required=True)
	ap.add_argument("--k", type=int, default=20)
	ap.add_argument("--embed-index", default="")
	ap.add_argument("--embed-model", default="sentence-transformers/all-MiniLM-L6-v2")
	ap.add_argument("--debug", action="store_true")
	args = ap.parse_args()

	def log(msg: str) -> None:
		if bool(args.debug):
			print(f"[qa.search] {msg}", file=sys.stderr, flush=True)

	try:
		bm_path = Path(args.index)
		log(f"bm25_path={bm_path} exists={bm_path.exists()}")
		res = search_bm25(bm_path, args.question, k=int(args.k))
		log(f"bm25_results={len(res)}")
		# Optional embedding rerank/merge
		if str(args.embed_index).strip():
			emb_path = Path(args.embed_index)
			log(f"embed_path={emb_path} exists={emb_path.exists()}")
			emb = _load_embed(emb_path)
			paths = emb.get("paths", [])
			vecs = emb.get("vectors", [])
			log(f"embed_docs={len(paths)} vectors_len={len(vecs)}")
			qv = _encode_query(args.question, str(args.embed_model))
			log(f"qv_dim={len(qv)}")
			if qv and paths and vecs and len(paths) == len(vecs):
				# Score all docs by cosine; merge with bm25 by normalized sum
				cos_scores: Dict[str, float] = {}
				for p, v in zip(paths, vecs):
					try:
						cos_scores[p] = _cosine(qv, [float(x) for x in v])
					except Exception:
						continue
				if res:
					max_bm25 = max((r.get("score", 0.0) for r in res), default=1.0) or 1.0
				else:
					max_bm25 = 1.0
				merged: Dict[str, float] = {}
				for r in res:
					p = r.get("path")
					merged[p] = merged.get(p, 0.0) + float(r.get("score", 0.0)) / max_bm25
				for p, s in cos_scores.items():
					merged[p] = merged.get(p, 0.0) + float(s)
				res = [{"path": p, "score": float(s)} for p, s in merged.items()]
				res.sort(key=lambda x: x[1], reverse=True)
				res = res[: int(args.k)]
		print(json.dumps({"ok": True, "results": res}))
		return 0
	except Exception as e:
		# Emit machine-readable error to stdout for upstream resilience
		print(json.dumps({"ok": False, "error": str(e)}))
		return 0

if __name__ == "__main__":
	raise SystemExit(main())


tokenize

#!/usr/bin/env python3
from __future__ import annotations
import logging

import argparse
import json
import math
import re
from pathlib import Path
from typing import Dict, Any, List, Tuple
import json
import sys
from pathlib import Path

TOKEN_RE = re.compile(r"[A-Za-z0-9_]+")

def tokenize(q: str) -> List[str]:
	return [t.lower() for t in TOKEN_RE.findall(q or "")]

def load_bm25(path: Path) -> Dict[str, Any]:
	try:
		return json.loads(path.read_text(encoding="utf-8"))
	except Exception:
		return {}

def bm25_score(query: List[str], bm: Dict[str, Any], doc: str, k1: float = 1.2, b: float = 0.75) -> float:
	postings = bm.get("postings", {})
	df = bm.get("df", {})
	docs = bm.get("docs", {})
	avgdl = float(bm.get("avgdl", 1.0) or 1.0)
	N = int(bm.get("N", max(1, len(docs))))
	dl = float(docs.get(doc, 0) or 0)
	score = 0.0
	for term in query:
		dft = int(df.get(term, 0) or 0)
		if dft <= 0:


import logging

import argparse
import json
import math
import re
from pathlib import Path
from typing import Dict, Any, List, Tuple
import json
import sys
from pathlib import Path

TOKEN_RE = re.compile(r"[A-Za-z0-9_]+")

def tokenize(q: str) -> List[str]:
	return [t.lower() for t in TOKEN_RE.findall(q or "")]

def load_bm25(path: Path) -> Dict[str, Any]:
	try:
		return json.loads(path.read_text(encoding="utf-8"))
	except Exception:
		return {}

def bm25_score(query: List[str], bm: Dict[str, Any], doc: str, k1: float = 1.2, b: float = 0.75) -> float:
	postings = bm.get("postings", {})
	df = bm.get("df", {})
	docs = bm.get("docs", {})
	avgdl = float(bm.get("avgdl", 1.0) or 1.0)
	N = int(bm.get("N", max(1, len(docs))))
	dl = float(docs.get(doc, 0) or 0)
	score = 0.0
	for term in query:
		dft = int(df.get(term, 0) or 0)
		if dft <= 0:
			continue
		idf = math.log(1 + (N - dft + 0.5) / (dft + 0.5))
		tf = float(postings.get(term, {}).get(doc, 0) or 0)
		if tf <= 0:
			continue
		num = tf * (k1 + 1)
		den = tf + k1 * (1 - b + b * (dl / max(1.0, avgdl)))


from typing import Dict, Any, List, Tuple
import json
import sys
from pathlib import Path

TOKEN_RE = re.compile(r"[A-Za-z0-9_]+")

def tokenize(q: str) -> List[str]:
	return [t.lower() for t in TOKEN_RE.findall(q or "")]

def load_bm25(path: Path) -> Dict[str, Any]:
	try:
		return json.loads(path.read_text(encoding="utf-8"))
	except Exception:
		return {}

def bm25_score(query: List[str], bm: Dict[str, Any], doc: str, k1: float = 1.2, b: float = 0.75) -> float:
	postings = bm.get("postings", {})
	df = bm.get("df", {})
	docs = bm.get("docs", {})
	avgdl = float(bm.get("avgdl", 1.0) or 1.0)
	N = int(bm.get("N", max(1, len(docs))))
	dl = float(docs.get(doc, 0) or 0)
	score = 0.0
	for term in query:
		dft = int(df.get(term, 0) or 0)
		if dft <= 0:
			continue
		idf = math.log(1 + (N - dft + 0.5) / (dft + 0.5))
		tf = float(postings.get(term, {}).get(doc, 0) or 0)
		if tf <= 0:
			continue
		num = tf * (k1 + 1)
		den = tf + k1 * (1 - b + b * (dl / max(1.0, avgdl)))
		score += idf * (num / max(1e-9, den))
	return float(score)

def search_bm25(index_path: Path, query_text: str, k: int = 20) -> List[Dict[str, Any]]:
	bm = load_bm25(index_path)
	terms = tokenize(query_text)
	docs = list((bm.get("docs") or {}).keys())
	scored: List[Tuple[str, float]] = []
	for d in docs:
		s = bm25_score(terms, bm, d)
		if s > 0:
			scored.append((d, s))
	scored.sort(key=lambda x: x[1], reverse=True)
	results = [{"path": d, "score": float(s)} for d, s in scored[:k]]
	# Fallback: path token match if BM25 yields nothing
	if not results:
		path_scores: List[Tuple[str, float]] = []
		ltokens = [t for t in terms if t]
		for d in docs:
			p = d.lower()
			s = 0.0


	df = bm.get("df", {})
	docs = bm.get("docs", {})
	avgdl = float(bm.get("avgdl", 1.0) or 1.0)
	N = int(bm.get("N", max(1, len(docs))))
	dl = float(docs.get(doc, 0) or 0)
	score = 0.0
	for term in query:
		dft = int(df.get(term, 0) or 0)
		if dft <= 0:
			continue
		idf = math.log(1 + (N - dft + 0.5) / (dft + 0.5))
		tf = float(postings.get(term, {}).get(doc, 0) or 0)
		if tf <= 0:
			continue
		num = tf * (k1 + 1)
		den = tf + k1 * (1 - b + b * (dl / max(1.0, avgdl)))
		score += idf * (num / max(1e-9, den))
	return float(score)

def search_bm25(index_path: Path, query_text: str, k: int = 20) -> List[Dict[str, Any]]:
	bm = load_bm25(index_path)
	terms = tokenize(query_text)
	docs = list((bm.get("docs") or {}).keys())
	scored: List[Tuple[str, float]] = []
	for d in docs:
		s = bm25_score(terms, bm, d)
		if s > 0:
			scored.append((d, s))
	scored.sort(key=lambda x: x[1], reverse=True)
	results = [{"path": d, "score": float(s)} for d, s in scored[:k]]
	# Fallback: path token match if BM25 yields nothing
	if not results:
		path_scores: List[Tuple[str, float]] = []
		ltokens = [t for t in terms if t]
		for d in docs:
			p = d.lower()
			s = 0.0
			for t in ltokens:
				if t and t in p:
					s += 1.0
			if s > 0:
				path_scores.append((d, s))
		# Prefer markdown if querying for readme/docs
		if path_scores:
			path_scores.sort(key=lambda x: (0 if x[0].lower().endswith('.md') else 1, -x[1]))
			results = [{"path": d, "score": float(s)} for d, s in path_scores[:k]]
	# Last-resort heuristics for README/summary/overview queries
	if not results:
		q = " ".join(terms)
		want_readme = any(tok in ("readme", "intro", "introduction", "overview") for tok in terms)
		want_summary = any(tok in ("summary", "summarize", "synopsis") for tok in terms)
		candidates: List[Tuple[str, float]] = []
		for d in docs:
			name = d.rsplit("/", 1)[-1].lower()
			score = 0.0
			if want_readme and name in ("readme.md", "readme"):
				score = 3.0
			elif want_summary and name in ("summary.md", "summary"):
				score = 3.0
			elif name in ("index.md", "overview.md", "introduction.md", "intro.md"):
				score = 2.0
			elif name.endswith(".md") and ("docs/" in d or "/docs/" in d):
				score = 1.0
			if score > 0.0:
				candidates.append((d, score))
		if candidates:
			candidates.sort(key=lambda x: (-x[1], x[0]))
			results = [{"path": d, "score": float(s)} for d, s in candidates[:k]]
	return results

def _cosine(a: List[float], b: List[float]) -> float:
	# inputs assumed normalized
	return float(sum(x*y for x, y in zip(a, b)))

def _load_embed(path: Path) -> Dict[str, Any]:
	try:
		return json.loads(path.read_text(encoding="utf-8"))
	except Exception:
		return {}

def _encode_query(q: str, model_name: str) -> List[float]:
	try:
		from sentence_transformers import SentenceTransformer  # type: ignore
		model = SentenceTransformer(model_name)
		vec = model.encode([q], normalize_embeddings=True)
		return [float(x) for x in (vec[0] if hasattr(vec, "__getitem__") else vec)]


		candidates: List[Tuple[str, float]] = []
		for d in docs:
			name = d.rsplit("/", 1)[-1].lower()
			score = 0.0
			if want_readme and name in ("readme.md", "readme"):
				score = 3.0
			elif want_summary and name in ("summary.md", "summary"):
				score = 3.0
			elif name in ("index.md", "overview.md", "introduction.md", "intro.md"):
				score = 2.0
			elif name.endswith(".md") and ("docs/" in d or "/docs/" in d):
				score = 1.0
			if score > 0.0:
				candidates.append((d, score))
		if candidates:
			candidates.sort(key=lambda x: (-x[1], x[0]))
			results = [{"path": d, "score": float(s)} for d, s in candidates[:k]]
	return results

def _cosine(a: List[float], b: List[float]) -> float:
	# inputs assumed normalized
	return float(sum(x*y for x, y in zip(a, b)))

def _load_embed(path: Path) -> Dict[str, Any]:
	try:
		return json.loads(path.read_text(encoding="utf-8"))
	except Exception:
		return {}

def _encode_query(q: str, model_name: str) -> List[float]:
	try:
		from sentence_transformers import SentenceTransformer  # type: ignore
		model = SentenceTransformer(model_name)
		vec = model.encode([q], normalize_embeddings=True)
		return [float(x) for x in (vec[0] if hasattr(vec, "__getitem__") else vec)]
	except Exception:
		return []

def main() -> int:


				score = 3.0
			elif want_summary and name in ("summary.md", "summary"):
				score = 3.0
			elif name in ("index.md", "overview.md", "introduction.md", "intro.md"):
				score = 2.0
			elif name.endswith(".md") and ("docs/" in d or "/docs/" in d):
				score = 1.0
			if score > 0.0:
				candidates.append((d, score))
		if candidates:
			candidates.sort(key=lambda x: (-x[1], x[0]))
			results = [{"path": d, "score": float(s)} for d, s in candidates[:k]]
	return results

def _cosine(a: List[float], b: List[float]) -> float:
	# inputs assumed normalized
	return float(sum(x*y for x, y in zip(a, b)))

def _load_embed(path: Path) -> Dict[str, Any]:
	try:
		return json.loads(path.read_text(encoding="utf-8"))
	except Exception:
		return {}

def _encode_query(q: str, model_name: str) -> List[float]:
	try:
		from sentence_transformers import SentenceTransformer  # type: ignore
		model = SentenceTransformer(model_name)
		vec = model.encode([q], normalize_embeddings=True)
		return [float(x) for x in (vec[0] if hasattr(vec, "__getitem__") else vec)]
	except Exception:
		return []

def main() -> int:
	ap = argparse.ArgumentParser(description="Search BM25 index for relevant files")
	root = Path(__file__).resolve().parents[2]
	ap.add_argument("--index", default=str(root / "data" / "sandbox" / "tmp" / "bm25_index.json"))
	ap.add_argument("--question", required=True)
	ap.add_argument("--k", type=int, default=20)
	ap.add_argument("--embed-index", default="")
	ap.add_argument("--embed-model", default="sentence-transformers/all-MiniLM-L6-v2")


			if score > 0.0:
				candidates.append((d, score))
		if candidates:
			candidates.sort(key=lambda x: (-x[1], x[0]))
			results = [{"path": d, "score": float(s)} for d, s in candidates[:k]]
	return results

def _cosine(a: List[float], b: List[float]) -> float:
	# inputs assumed normalized
	return float(sum(x*y for x, y in zip(a, b)))

def _load_embed(path: Path) -> Dict[str, Any]:
	try:
		return json.loads(path.read_text(encoding="utf-8"))
	except Exception:
		return {}

def _encode_query(q: str, model_name: str) -> List[float]:
	try:
		from sentence_transformers import SentenceTransformer  # type: ignore
		model = SentenceTransformer(model_name)
		vec = model.encode([q], normalize_embeddings=True)
		return [float(x) for x in (vec[0] if hasattr(vec, "__getitem__") else vec)]
	except Exception:
		return []

def main() -> int:
	ap = argparse.ArgumentParser(description="Search BM25 index for relevant files")
	root = Path(__file__).resolve().parents[2]
	ap.add_argument("--index", default=str(root / "data" / "sandbox" / "tmp" / "bm25_index.json"))
	ap.add_argument("--question", required=True)
	ap.add_argument("--k", type=int, default=20)
	ap.add_argument("--embed-index", default="")
	ap.add_argument("--embed-model", default="sentence-transformers/all-MiniLM-L6-v2")
	ap.add_argument("--debug", action="store_true")
	args = ap.parse_args()

	def log(msg: str) -> None:
		if bool(args.debug):
			print(f"[qa.search] {msg}", file=sys.stderr, flush=True)

	try:
		bm_path = Path(args.index)
		log(f"bm25_path={bm_path} exists={bm_path.exists()}")


main

	return float(sum(x*y for x, y in zip(a, b)))

def _load_embed(path: Path) -> Dict[str, Any]:
	try:
		return json.loads(path.read_text(encoding="utf-8"))
	except Exception:
		return {}

def _encode_query(q: str, model_name: str) -> List[float]:
	try:
		from sentence_transformers import SentenceTransformer  # type: ignore
		model = SentenceTransformer(model_name)
		vec = model.encode([q], normalize_embeddings=True)
		return [float(x) for x in (vec[0] if hasattr(vec, "__getitem__") else vec)]
	except Exception:
		return []

def main() -> int:
	ap = argparse.ArgumentParser(description="Search BM25 index for relevant files")
	root = Path(__file__).resolve().parents[2]
	ap.add_argument("--index", default=str(root / "data" / "sandbox" / "tmp" / "bm25_index.json"))
	ap.add_argument("--question", required=True)
	ap.add_argument("--k", type=int, default=20)
	ap.add_argument("--embed-index", default="")
	ap.add_argument("--embed-model", default="sentence-transformers/all-MiniLM-L6-v2")
	ap.add_argument("--debug", action="store_true")
	args = ap.parse_args()

	def log(msg: str) -> None:
		if bool(args.debug):
			print(f"[qa.search] {msg}", file=sys.stderr, flush=True)

	try:
		bm_path = Path(args.index)
		log(f"bm25_path={bm_path} exists={bm_path.exists()}")
		res = search_bm25(bm_path, args.question, k=int(args.k))
		log(f"bm25_results={len(res)}")
		# Optional embedding rerank/merge
		if str(args.embed_index).strip():
			emb_path = Path(args.embed_index)
			log(f"embed_path={emb_path} exists={emb_path.exists()}")
			emb = _load_embed(emb_path)
			paths = emb.get("paths", [])
			vecs = emb.get("vectors", [])
			log(f"embed_docs={len(paths)} vectors_len={len(vecs)}")
			qv = _encode_query(args.question, str(args.embed_model))
			log(f"qv_dim={len(qv)}")
			if qv and paths and vecs and len(paths) == len(vecs):
				# Score all docs by cosine; merge with bm25 by normalized sum
				cos_scores: Dict[str, float] = {}
				for p, v in zip(paths, vecs):
					try:
						cos_scores[p] = _cosine(qv, [float(x) for x in v])
					except Exception:
						continue
				if res:
					max_bm25 = max((r.get("score", 0.0) for r in res), default=1.0) or 1.0
				else:
					max_bm25 = 1.0
				merged: Dict[str, float] = {}
				for r in res:
					p = r.get("path")
					merged[p] = merged.get(p, 0.0) + float(r.get("score", 0.0)) / max_bm25
				for p, s in cos_scores.items():
					merged[p] = merged.get(p, 0.0) + float(s)
				res = [{"path": p, "score": float(s)} for p, s in merged.items()]
				res.sort(key=lambda x: x[1], reverse=True)
				res = res[: int(args.k)]
		print(json.dumps({"ok": True, "results": res}))
		return 0
	except Exception as e:
		# Emit machine-readable error to stdout for upstream resilience
		print(json.dumps({"ok": False, "error": str(e)}))
		return 0

if __name__ == "__main__":
	raise SystemExit(main())


log

	try:
		from sentence_transformers import SentenceTransformer  # type: ignore
		model = SentenceTransformer(model_name)
		vec = model.encode([q], normalize_embeddings=True)
		return [float(x) for x in (vec[0] if hasattr(vec, "__getitem__") else vec)]
	except Exception:
		return []

def main() -> int:
	ap = argparse.ArgumentParser(description="Search BM25 index for relevant files")
	root = Path(__file__).resolve().parents[2]
	ap.add_argument("--index", default=str(root / "data" / "sandbox" / "tmp" / "bm25_index.json"))
	ap.add_argument("--question", required=True)
	ap.add_argument("--k", type=int, default=20)
	ap.add_argument("--embed-index", default="")
	ap.add_argument("--embed-model", default="sentence-transformers/all-MiniLM-L6-v2")
	ap.add_argument("--debug", action="store_true")
	args = ap.parse_args()

	def log(msg: str) -> None:
		if bool(args.debug):
			print(f"[qa.search] {msg}", file=sys.stderr, flush=True)

	try:
		bm_path = Path(args.index)
		log(f"bm25_path={bm_path} exists={bm_path.exists()}")
		res = search_bm25(bm_path, args.question, k=int(args.k))
		log(f"bm25_results={len(res)}")
		# Optional embedding rerank/merge
		if str(args.embed_index).strip():
			emb_path = Path(args.embed_index)
			log(f"embed_path={emb_path} exists={emb_path.exists()}")
			emb = _load_embed(emb_path)
			paths = emb.get("paths", [])
			vecs = emb.get("vectors", [])
			log(f"embed_docs={len(paths)} vectors_len={len(vecs)}")
			qv = _encode_query(args.question, str(args.embed_model))
			log(f"qv_dim={len(qv)}")
			if qv and paths and vecs and len(paths) == len(vecs):
				# Score all docs by cosine; merge with bm25 by normalized sum
				cos_scores: Dict[str, float] = {}
				for p, v in zip(paths, vecs):


#!/usr/bin/env python3
from __future__ import annotations
import logging

import argparse
import json
from pathlib import Path
from typing import Dict, Any

def read_range(path: Path, start: int, end: int, max_bytes: int = 20000) -> Dict[str, Any]:
	if not path.exists() or not path.is_file():
		return {"ok": False, "error": "not_found", "path": str(path)}
	text = path.read_text(encoding="utf-8", errors="ignore")
	# Boundaries (1-based lines)
	lines = text.splitlines()
	n = len(lines)
	lo = max(1, int(start))
	hi = min(max(lo, int(end)), n)
	selected = "\n".join(lines[lo - 1:hi])
	if len(selected.encode("utf-8")) > max_bytes:
		selected = selected.encode("utf-8")[:max_bytes].decode("utf-8", errors="ignore")
	return {"ok": True, "path": str(path), "start": lo, "end": hi, "content": selected}

def main() -> int:
	ap = argparse.ArgumentParser(description="Read bounded file range (line-based)")
	root = Path(__file__).resolve().parents[2]
	ap.add_argument("path", help="Absolute or repo-relative path")
	ap.add_argument("--start", type=int, default=1)
	ap.add_argument("--end", type=int, default=200)
	ap.add_argument("--repo-root", default=str(root))
	args = ap.parse_args()

	repo = Path(args.repo_root).resolve()
	pp = Path(args.path)
	if not pp.is_absolute():
		pp = (repo / pp).resolve()
	res = read_range(pp, int(args.start), int(args.end))
	print(json.dumps(res, ensure_ascii=False))
	return 0

if __name__ == "__main__":
	raise SystemExit(main())


#!/usr/bin/env python3
from __future__ import annotations
import logging

import argparse
import json
from pathlib import Path
from typing import Dict, Any

def read_range(path: Path, start: int, end: int, max_bytes: int = 20000) -> Dict[str, Any]:
	if not path.exists() or not path.is_file():
		return {"ok": False, "error": "not_found", "path": str(path)}
	text = path.read_text(encoding="utf-8", errors="ignore")
	# Boundaries (1-based lines)
	lines = text.splitlines()
	n = len(lines)
	lo = max(1, int(start))
	hi = min(max(lo, int(end)), n)
	selected = "\n".join(lines[lo - 1:hi])
	if len(selected.encode("utf-8")) > max_bytes:
		selected = selected.encode("utf-8")[:max_bytes].decode("utf-8", errors="ignore")
	return {"ok": True, "path": str(path), "start": lo, "end": hi, "content": selected}

def main() -> int:
	ap = argparse.ArgumentParser(description="Read bounded file range (line-based)")
	root = Path(__file__).resolve().parents[2]
	ap.add_argument("path", help="Absolute or repo-relative path")
	ap.add_argument("--start", type=int, default=1)
	ap.add_argument("--end", type=int, default=200)
	ap.add_argument("--repo-root", default=str(root))
	args = ap.parse_args()

	repo = Path(args.repo_root).resolve()
	pp = Path(args.path)
	if not pp.is_absolute():
		pp = (repo / pp).resolve()
	res = read_range(pp, int(args.start), int(args.end))
	print(json.dumps(res, ensure_ascii=False))
	return 0


main

import json
from pathlib import Path
from typing import Dict, Any

def read_range(path: Path, start: int, end: int, max_bytes: int = 20000) -> Dict[str, Any]:
	if not path.exists() or not path.is_file():
		return {"ok": False, "error": "not_found", "path": str(path)}
	text = path.read_text(encoding="utf-8", errors="ignore")
	# Boundaries (1-based lines)
	lines = text.splitlines()
	n = len(lines)
	lo = max(1, int(start))
	hi = min(max(lo, int(end)), n)
	selected = "\n".join(lines[lo - 1:hi])
	if len(selected.encode("utf-8")) > max_bytes:
		selected = selected.encode("utf-8")[:max_bytes].decode("utf-8", errors="ignore")
	return {"ok": True, "path": str(path), "start": lo, "end": hi, "content": selected}

def main() -> int:
	ap = argparse.ArgumentParser(description="Read bounded file range (line-based)")
	root = Path(__file__).resolve().parents[2]
	ap.add_argument("path", help="Absolute or repo-relative path")
	ap.add_argument("--start", type=int, default=1)
	ap.add_argument("--end", type=int, default=200)
	ap.add_argument("--repo-root", default=str(root))
	args = ap.parse_args()

	repo = Path(args.repo_root).resolve()
	pp = Path(args.path)
	if not pp.is_absolute():
		pp = (repo / pp).resolve()
	res = read_range(pp, int(args.start), int(args.end))
	print(json.dumps(res, ensure_ascii=False))
	return 0

if __name__ == "__main__":
	raise SystemExit(main())


#!/usr/bin/env python3
from __future__ import annotations
import logging

import argparse
import json
from pathlib import Path
from typing import Dict, Any, List

def summarize(question: str, snippets: List[Dict[str, Any]]) -> Dict[str, Any]:
	# Deterministic heuristic summary with basic de-dup and README preference
	parts: List[str] = []
	citations: List[Dict[str, Any]] = []
	seen: set[str] = set()
	# Prefer README/summary files first
	def rank(sn: Dict[str, Any]) -> tuple:
		p = str(sn.get("path", "")).lower()
		is_md = p.endswith(".md")
		is_readme = p.endswith("readme.md") or p.endswith("/readme")
		is_summary = p.endswith("summary.md")
		return (
			0 if is_readme else (1 if is_summary else (2 if is_md else 3)),
			len(str(sn.get("content", ""))) * -1,
		)
	for sn in sorted(snippets, key=rank)[:10]:
		path = str(sn.get("path", ""))
		start = int(sn.get("start", 1))
		end = int(sn.get("end", start))
		content = str(sn.get("content", ""))
		if not content:
			continue
		key = (path + "::" + content[:80]).strip()
		if key in seen:
			continue
		seen.add(key)
		parts.append(content.strip())
		citations.append({"path": path, "start": start, "end": end})
	answer = "\n\n".join(parts)[:2000]
	return {"answer": answer, "citations": citations}

def main() -> int:
	ap = argparse.ArgumentParser(description="Summarize snippets into an answer with citations")
	ap.add_argument("--question", required=False, default="")
	ap.add_argument("--snippets", required=False, help="Path to JSON file with snippets list")
	ap.add_argument("--emit-only", action="store_true", help="No summarization; just echo snippets as citations")
	ap.add_argument("--debug", action="store_true")
	args = ap.parse_args()

	if args.emit_only:
		# Emit empty answer and pass-through citations (for tooling)
		try:
			lst = json.loads(Path(args.snippets).read_text(encoding="utf-8")) if args.snippets else []
		except Exception:
			lst = []
		cits = [{"path": str(x.get("path", "")), "start": int(x.get("start", 1)), "end": int(x.get("end", 1))} for x in (lst or [])]
		print(json.dumps({"answer": "", "citations": cits}))
		return 0

	try:
		snippets = json.loads(Path(args.snippets).read_text(encoding="utf-8")) if args.snippets else []
	except Exception as e:
		if bool(args.debug):
			print(f"[qa.summarize] failed to load snippets: {e}", flush=True)
		snippets = []
	res = summarize(str(args.question or ""), snippets or [])
	print(json.dumps(res, ensure_ascii=False))
	return 0

if __name__ == "__main__":
	raise SystemExit(main())


summarize

#!/usr/bin/env python3
from __future__ import annotations
import logging

import argparse
import json
from pathlib import Path
from typing import Dict, Any, List

def summarize(question: str, snippets: List[Dict[str, Any]]) -> Dict[str, Any]:
	# Deterministic heuristic summary with basic de-dup and README preference
	parts: List[str] = []
	citations: List[Dict[str, Any]] = []
	seen: set[str] = set()
	# Prefer README/summary files first
	def rank(sn: Dict[str, Any]) -> tuple:
		p = str(sn.get("path", "")).lower()
		is_md = p.endswith(".md")
		is_readme = p.endswith("readme.md") or p.endswith("/readme")
		is_summary = p.endswith("summary.md")
		return (
			0 if is_readme else (1 if is_summary else (2 if is_md else 3)),
			len(str(sn.get("content", ""))) * -1,
		)
	for sn in sorted(snippets, key=rank)[:10]:
		path = str(sn.get("path", ""))
		start = int(sn.get("start", 1))
		end = int(sn.get("end", start))
		content = str(sn.get("content", ""))
		if not content:
			continue
		key = (path + "::" + content[:80]).strip()
		if key in seen:
			continue
		seen.add(key)
		parts.append(content.strip())
		citations.append({"path": path, "start": start, "end": end})
	answer = "\n\n".join(parts)[:2000]
	return {"answer": answer, "citations": citations}

def main() -> int:
	ap = argparse.ArgumentParser(description="Summarize snippets into an answer with citations")
	ap.add_argument("--question", required=False, default="")
	ap.add_argument("--snippets", required=False, help="Path to JSON file with snippets list")
	ap.add_argument("--emit-only", action="store_true", help="No summarization; just echo snippets as citations")
	ap.add_argument("--debug", action="store_true")
	args = ap.parse_args()

	if args.emit_only:
		# Emit empty answer and pass-through citations (for tooling)
		try:
			lst = json.loads(Path(args.snippets).read_text(encoding="utf-8")) if args.snippets else []
		except Exception:
			lst = []
		cits = [{"path": str(x.get("path", "")), "start": int(x.get("start", 1)), "end": int(x.get("end", 1))} for x in (lst or [])]
		print(json.dumps({"answer": "", "citations": cits}))
		return 0


main

			0 if is_readme else (1 if is_summary else (2 if is_md else 3)),
			len(str(sn.get("content", ""))) * -1,
		)
	for sn in sorted(snippets, key=rank)[:10]:
		path = str(sn.get("path", ""))
		start = int(sn.get("start", 1))
		end = int(sn.get("end", start))
		content = str(sn.get("content", ""))
		if not content:
			continue
		key = (path + "::" + content[:80]).strip()
		if key in seen:
			continue
		seen.add(key)
		parts.append(content.strip())
		citations.append({"path": path, "start": start, "end": end})
	answer = "\n\n".join(parts)[:2000]
	return {"answer": answer, "citations": citations}

def main() -> int:
	ap = argparse.ArgumentParser(description="Summarize snippets into an answer with citations")
	ap.add_argument("--question", required=False, default="")
	ap.add_argument("--snippets", required=False, help="Path to JSON file with snippets list")
	ap.add_argument("--emit-only", action="store_true", help="No summarization; just echo snippets as citations")
	ap.add_argument("--debug", action="store_true")
	args = ap.parse_args()

	if args.emit_only:
		# Emit empty answer and pass-through citations (for tooling)
		try:
			lst = json.loads(Path(args.snippets).read_text(encoding="utf-8")) if args.snippets else []
		except Exception:
			lst = []
		cits = [{"path": str(x.get("path", "")), "start": int(x.get("start", 1)), "end": int(x.get("end", 1))} for x in (lst or [])]
		print(json.dumps({"answer": "", "citations": cits}))
		return 0

	try:
		snippets = json.loads(Path(args.snippets).read_text(encoding="utf-8")) if args.snippets else []
	except Exception as e:
		if bool(args.debug):
			print(f"[qa.summarize] failed to load snippets: {e}", flush=True)
		snippets = []
	res = summarize(str(args.question or ""), snippets or [])
	print(json.dumps(res, ensure_ascii=False))
	return 0

if __name__ == "__main__":
	raise SystemExit(main())


rank

#!/usr/bin/env python3
from __future__ import annotations
import logging

import argparse
import json
from pathlib import Path
from typing import Dict, Any, List

def summarize(question: str, snippets: List[Dict[str, Any]]) -> Dict[str, Any]:
	# Deterministic heuristic summary with basic de-dup and README preference
	parts: List[str] = []
	citations: List[Dict[str, Any]] = []
	seen: set[str] = set()
	# Prefer README/summary files first
	def rank(sn: Dict[str, Any]) -> tuple:
		p = str(sn.get("path", "")).lower()
		is_md = p.endswith(".md")
		is_readme = p.endswith("readme.md") or p.endswith("/readme")
		is_summary = p.endswith("summary.md")
		return (
			0 if is_readme else (1 if is_summary else (2 if is_md else 3)),
			len(str(sn.get("content", ""))) * -1,
		)
	for sn in sorted(snippets, key=rank)[:10]:
		path = str(sn.get("path", ""))
		start = int(sn.get("start", 1))
		end = int(sn.get("end", start))
		content = str(sn.get("content", ""))
		if not content:
			continue
		key = (path + "::" + content[:80]).strip()
		if key in seen:
			continue
		seen.add(key)
		parts.append(content.strip())
		citations.append({"path": path, "start": start, "end": end})
	answer = "\n\n".join(parts)[:2000]
	return {"answer": answer, "citations": citations}

def main() -> int:
	ap = argparse.ArgumentParser(description="Summarize snippets into an answer with citations")
	ap.add_argument("--question", required=False, default="")


#!/usr/bin/env python3
from __future__ import annotations
import logging

import argparse
import json
from pathlib import Path
from typing import Any, Dict, List

TEMPLATE = (
	"You are a codebase QA model. Answer the user's QUESTION strictly based on the provided SNIPPETS.\n"
	"Rules:\n"
	"- Output ONLY a single JSON object with keys: answer (string), citations (list of objects with path,start,end).\n"
	"- The answer must be concise and faithful to the snippets; do not invent facts.\n"
	"- Every factual statement must be supported by at least one citation.\n"
	"- Citations must reference the given snippets' file paths and line ranges.\n"
	"- If evidence is insufficient, set answer to 'insufficient evidence' and citations to [].\n"
	"JSON schema:\n{\n  \"answer\": \"string\",\n  \"citations\": [{\"path\": \"/abs/path\", \"start\": 1, \"end\": 10}]\n}\n"
)

def build_prompt(question: str, snippets: List[Dict[str, Any]]) -> str:
	parts: List[str] = [TEMPLATE, f"QUESTION: {question}\n", "SNIPPETS:\n"]
	for i, sn in enumerate(snippets[:20]):
		p = str(sn.get("path", ""))
		start = int(sn.get("start", 1))
		end = int(sn.get("end", start))
		content = str(sn.get("content", ""))
		parts.append(f"--- snippet_{i+1} {p}:{start}-{end} ---\n{content}\n")
	parts.append("Return ONLY JSON.\n")
	return "".join(parts)

def safe_parse_json(text: str) -> Dict[str, Any]:
	text = text.strip()
	# Heuristics: find first and last brace
	lo = text.find("{")
	hi = text.rfind("}")
	if lo >= 0 and hi > lo:
		cand = text[lo:hi + 1]
		try:
			obj = json.loads(cand)
			if isinstance(obj, dict) and "answer" in obj and "citations" in obj:
				return obj
		except Exception:
			pass
	return {"answer": "", "citations": []}

def main() -> int:
	ap = argparse.ArgumentParser(description="LLM answerer for codebase QA (constrained JSON)")
	root = Path(__file__).resolve().parents[2]
	ap.add_argument("--question", required=True)
	ap.add_argument("--snippets", required=True, help="Path to JSON file with snippets list")
	ap.add_argument("--model", default="meta-llama/Llama-3.2-3B")
	ap.add_argument("--backend", choices=["hf"], default="hf")
	ap.add_argument("--max-new", type=int, default=300)
	args = ap.parse_args()

	try:
		snippets = json.loads(Path(args.snippets).read_text(encoding="utf-8"))
	except Exception:
		snippets = []
	prompt = build_prompt(str(args.question), snippets or [])

	# Backends: HF local
	try:
		from agi_dw.core.llm.hf_client import HFClient  # type: ignore
		client = HFClient.get_cached(args.model)
		text = client.generate(prompt, max_new_tokens=int(args.max_new), temperature=0.0)
	except Exception:
		text = "{\n  \"answer\": \"\", \"citations\": []\n}"

	obj = safe_parse_json(text)
	# Sanitize citations types
	cits_out: List[Dict[str, Any]] = []
	for c in (obj.get("citations") or []):
		try:
			path = str(c.get("path"))
			start = int(c.get("start"))
			end = int(c.get("end"))
			cits_out.append({"path": path, "start": start, "end": end})
		except Exception:
			continue
	res = {"answer": str(obj.get("answer", "")), "citations": cits_out}
	print(json.dumps(res, ensure_ascii=False))
	return 0

if __name__ == "__main__":
	raise SystemExit(main())


import logging

import argparse
import json
from pathlib import Path
from typing import Any, Dict, List

TEMPLATE = (
	"You are a codebase QA model. Answer the user's QUESTION strictly based on the provided SNIPPETS.\n"
	"Rules:\n"
	"- Output ONLY a single JSON object with keys: answer (string), citations (list of objects with path,start,end).\n"
	"- The answer must be concise and faithful to the snippets; do not invent facts.\n"
	"- Every factual statement must be supported by at least one citation.\n"
	"- Citations must reference the given snippets' file paths and line ranges.\n"
	"- If evidence is insufficient, set answer to 'insufficient evidence' and citations to [].\n"
	"JSON schema:\n{\n  \"answer\": \"string\",\n  \"citations\": [{\"path\": \"/abs/path\", \"start\": 1, \"end\": 10}]\n}\n"
)

def build_prompt(question: str, snippets: List[Dict[str, Any]]) -> str:
	parts: List[str] = [TEMPLATE, f"QUESTION: {question}\n", "SNIPPETS:\n"]
	for i, sn in enumerate(snippets[:20]):
		p = str(sn.get("path", ""))
		start = int(sn.get("start", 1))
		end = int(sn.get("end", start))
		content = str(sn.get("content", ""))
		parts.append(f"--- snippet_{i+1} {p}:{start}-{end} ---\n{content}\n")
	parts.append("Return ONLY JSON.\n")
	return "".join(parts)

def safe_parse_json(text: str) -> Dict[str, Any]:
	text = text.strip()
	# Heuristics: find first and last brace
	lo = text.find("{")
	hi = text.rfind("}")
	if lo >= 0 and hi > lo:
		cand = text[lo:hi + 1]
		try:
			obj = json.loads(cand)
			if isinstance(obj, dict) and "answer" in obj and "citations" in obj:
				return obj
		except Exception:
			pass
	return {"answer": "", "citations": []}

def main() -> int:
	ap = argparse.ArgumentParser(description="LLM answerer for codebase QA (constrained JSON)")


	"- The answer must be concise and faithful to the snippets; do not invent facts.\n"
	"- Every factual statement must be supported by at least one citation.\n"
	"- Citations must reference the given snippets' file paths and line ranges.\n"
	"- If evidence is insufficient, set answer to 'insufficient evidence' and citations to [].\n"
	"JSON schema:\n{\n  \"answer\": \"string\",\n  \"citations\": [{\"path\": \"/abs/path\", \"start\": 1, \"end\": 10}]\n}\n"
)

def build_prompt(question: str, snippets: List[Dict[str, Any]]) -> str:
	parts: List[str] = [TEMPLATE, f"QUESTION: {question}\n", "SNIPPETS:\n"]
	for i, sn in enumerate(snippets[:20]):
		p = str(sn.get("path", ""))
		start = int(sn.get("start", 1))
		end = int(sn.get("end", start))
		content = str(sn.get("content", ""))
		parts.append(f"--- snippet_{i+1} {p}:{start}-{end} ---\n{content}\n")
	parts.append("Return ONLY JSON.\n")
	return "".join(parts)

def safe_parse_json(text: str) -> Dict[str, Any]:
	text = text.strip()
	# Heuristics: find first and last brace
	lo = text.find("{")
	hi = text.rfind("}")
	if lo >= 0 and hi > lo:
		cand = text[lo:hi + 1]
		try:
			obj = json.loads(cand)
			if isinstance(obj, dict) and "answer" in obj and "citations" in obj:
				return obj
		except Exception:
			pass
	return {"answer": "", "citations": []}

def main() -> int:
	ap = argparse.ArgumentParser(description="LLM answerer for codebase QA (constrained JSON)")
	root = Path(__file__).resolve().parents[2]
	ap.add_argument("--question", required=True)
	ap.add_argument("--snippets", required=True, help="Path to JSON file with snippets list")
	ap.add_argument("--model", default="meta-llama/Llama-3.2-3B")
	ap.add_argument("--backend", choices=["hf"], default="hf")
	ap.add_argument("--max-new", type=int, default=300)
	args = ap.parse_args()

	try:
		snippets = json.loads(Path(args.snippets).read_text(encoding="utf-8"))
	except Exception:
		snippets = []
	prompt = build_prompt(str(args.question), snippets or [])

	# Backends: HF local
	try:


main

	parts.append("Return ONLY JSON.\n")
	return "".join(parts)

def safe_parse_json(text: str) -> Dict[str, Any]:
	text = text.strip()
	# Heuristics: find first and last brace
	lo = text.find("{")
	hi = text.rfind("}")
	if lo >= 0 and hi > lo:
		cand = text[lo:hi + 1]
		try:
			obj = json.loads(cand)
			if isinstance(obj, dict) and "answer" in obj and "citations" in obj:
				return obj
		except Exception:
			pass
	return {"answer": "", "citations": []}

def main() -> int:
	ap = argparse.ArgumentParser(description="LLM answerer for codebase QA (constrained JSON)")
	root = Path(__file__).resolve().parents[2]
	ap.add_argument("--question", required=True)
	ap.add_argument("--snippets", required=True, help="Path to JSON file with snippets list")
	ap.add_argument("--model", default="meta-llama/Llama-3.2-3B")
	ap.add_argument("--backend", choices=["hf"], default="hf")
	ap.add_argument("--max-new", type=int, default=300)
	args = ap.parse_args()

	try:
		snippets = json.loads(Path(args.snippets).read_text(encoding="utf-8"))
	except Exception:
		snippets = []
	prompt = build_prompt(str(args.question), snippets or [])

	# Backends: HF local
	try:
		from agi_dw.core.llm.hf_client import HFClient  # type: ignore
		client = HFClient.get_cached(args.model)
		text = client.generate(prompt, max_new_tokens=int(args.max_new), temperature=0.0)
	except Exception:
		text = "{\n  \"answer\": \"\", \"citations\": []\n}"

	obj = safe_parse_json(text)
	# Sanitize citations types
	cits_out: List[Dict[str, Any]] = []
	for c in (obj.get("citations") or []):
		try:
			path = str(c.get("path"))
			start = int(c.get("start"))
			end = int(c.get("end"))
			cits_out.append({"path": path, "start": start, "end": end})
		except Exception:
			continue
	res = {"answer": str(obj.get("answer", "")), "citations": cits_out}
	print(json.dumps(res, ensure_ascii=False))
	return 0

if __name__ == "__main__":
	raise SystemExit(main())


#!/usr/bin/env python3
from __future__ import annotations
import logging

import argparse
import json
import os
import subprocess
import sys
import tempfile
from pathlib import Path
from typing import Any, Dict, List

def run_cmd(argv: List[str], cwd: Path | None = None) -> Dict[str, Any]:
	try:
		p = subprocess.run(argv, cwd=str(cwd) if cwd else None, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, check=False)
		return {"argv": argv, "rc": int(p.returncode), "stdout": p.stdout, "stderr": p.stderr}
	except Exception as e:
		return {"argv": argv, "rc": -1, "stdout": "", "stderr": str(e)}

def load_json_safe(p: Path) -> Dict[str, Any]:
	try:
		return json.loads(p.read_text(encoding="utf-8"))
	except Exception:
		return {}

def main() -> int:
	ap = argparse.ArgumentParser(description="Ask a question about a sandboxed codebase and emit answer+citations")
	root = Path(__file__).resolve().parents[2]
	ap.add_argument("--question", required=True)
	ap.add_argument("--world", default=str(root / "data" / "sandbox" / "tmp" / "world.json"))
	ap.add_argument("--code-index", default=str(root / "data" / "sandbox" / "tmp" / "code_index.json"))
	ap.add_argument("--bm25-index", default=str(root / "data" / "sandbox" / "tmp" / "bm25_index.json"))
	ap.add_argument("--max-files", type=int, default=20)
	ap.add_argument("--lines", type=int, default=80, help="Lines per read window")
	ap.add_argument("--out-traces", default=str(root / "data" / "devtools" / "qa.traces.jsonl"))
	ap.add_argument("--validate", action="store_true")
	ap.add_argument("--debug", action="store_true")
	args = ap.parse_args()

	def log(msg: str) -> None:
		if bool(args.debug):
			print(f"[qa.ask] {msg}", file=sys.stderr, flush=True)

	world = load_json_safe(Path(args.world))
	repo_root = Path(world.get("repo_root", root.as_posix()))
	log(f"repo_root={repo_root}")

	# 1) Ensure BM25 index exists (prefer world path)
	bm25_path = Path(world.get("bm25_index_path", args.bm25_index))
	log(f"bm25_index={bm25_path} exists={bm25_path.exists()}")
	if not bm25_path.exists():
		res = run_cmd(["python3", "scripts/qa/bm25_index.py", "--root", str(repo_root), "--out", str(bm25_path)])
		log(f"bm25_build rc={res.get('rc')} stderr={res.get('stderr')[:200]}")
		if int(res.get("rc", 1)) != 0:
			print(json.dumps({"ok": False, "error": "bm25_build_failed", "detail": res}))
			return 1

	# 2) Search for candidate files
	# Prefer hybrid search if embed index exists
	embed_path = Path(world.get("embed_index_path", repo_root / "data" / "sandbox" / "tmp" / "embed_index.json"))
	argv = ["python3", "scripts/qa/search.py", "--index", str(bm25_path), "--question", str(args.question), "--k", str(int(args.max_files))]
	if embed_path.exists():
		argv += ["--embed-index", str(embed_path)]
	log(f"search_argv={' '.join(argv)} (embed_exists={embed_path.exists()})")
	res = run_cmd(argv, cwd=repo_root)
	log(f"search_rc={res.get('rc')} stdout_len={len(res.get('stdout',''))} stderr={res.get('stderr')[:200]}")
	try:
		results = json.loads(res.get("stdout", "{}"))
	except Exception:
		results = {"results": []}
	cands = results.get("results", [])
	log(f"candidates={len(cands)} top3={[str(x.get('path','')) for x in cands[:3]]}")
	# Fallback: if no candidates, try common root docs
	if not cands:
		fallbacks = ["README.md", "readme.md", "ROADMAP.md", "SUMMARY.md", "docs/README.md", "docs/index.md"]
		picked: list[str] = []
		for rel in fallbacks:
			pp = (repo_root / rel).resolve()
			if pp.exists() and pp.is_file():
				picked.append(pp.as_posix())
				if len(picked) >= int(args.max_files):
					break
		cands = [{"path": p, "score": 1.0} for p in picked]
		log(f"fallback_candidates={len(cands)} from root docs")

	# 3) Read top snippets (prefer README/docs when relevant; take first 2 windows)
	snippets: List[Dict[str, Any]] = []
	for c in cands:
		p = Path(c.get("path", ""))
		if not p.is_absolute():
			p = (repo_root / p).resolve()
		# window 1
		read1 = run_cmd(["python3", "scripts/qa/read_file_range.py", str(p), "--start", "1", "--end", str(int(args.lines))])
		# window 2
		read2 = run_cmd(["python3", "scripts/qa/read_file_range.py", str(p), "--start", str(int(args.lines) + 1), "--end", str(int(args.lines) * 2)])
		for read in (read1, read2):
			try:
				obj = json.loads(read.get("stdout", "{}"))
			except Exception:
				obj = {"ok": False}
			if obj.get("ok"):
				obj["path"] = str(Path(obj.get("path")).resolve().as_posix())
				snippets.append(obj)
			else:
				log(f"read_fail path={p} rc={read.get('rc')} err={read.get('stderr')[:120]}")
	log(f"snippets_count={len(snippets)}")

	# 4) Summarize
	with tempfile.NamedTemporaryFile("w", delete=False, suffix=".json") as f:
		f.write(json.dumps(snippets, ensure_ascii=False))
		tmp_path = f.name
	res_sum = run_cmd(["python3", "scripts/qa/answer_summarize.py", "--question", str(args.question), "--snippets", str(tmp_path), "--debug"])
	log(f"summarize_rc={res_sum.get('rc')} out_len={len(res_sum.get('stdout',''))}")
	try:
		ans = json.loads(res_sum.get("stdout", "{}"))
	except Exception:
		ans = {"answer": "", "citations": []}
	log(f"answer_len={len(ans.get('answer',''))} citations={len(ans.get('citations',[]))}")

	# 5) Optional validation
	val = {"ok": True}
	if bool(args.validate):
		with tempfile.NamedTemporaryFile("w", delete=False, suffix=".json") as cfile:
			cfile.write(json.dumps(ans.get("citations", []), ensure_ascii=False))
			cpath = cfile.name
		# Use lenient defaults to allow short READMEs to pass; CI can enforce stricter gate
		val_run = run_cmd(["python3", "scripts/qa/validate_citations.py", "--world", str(args.world), "--citations", str(cpath), "--min-files", "1", "--min-lines", "5"], cwd=repo_root)
		try:
			val = json.loads(val_run.get("stdout", "{}"))
		except Exception:
			val = {"ok": False}
		log(f"validator_ok={val.get('ok')} coverage={val.get('coverage')}")

	# 6) Trace write
	trace = {
		"question": str(args.question),
		"candidates": cands,
		"snippets_count": len(snippets),
		"citations": ans.get("citations", []),
		"answer": ans.get("answer", ""),
		"validator": val,
	}
	outp = Path(args.out_traces)
	outp.parent.mkdir(parents=True, exist_ok=True)
	with outp.open("a", encoding="utf-8") as f:
		f.write(json.dumps(trace, ensure_ascii=False) + "\n")

	ok = bool(val.get("ok", True)) and bool(ans.get("answer", "").strip())
	print(json.dumps({"ok": ok, "answer": ans.get("answer", ""), "citations": ans.get("citations", [])}))
	return 0

if __name__ == "__main__":
	raise SystemExit(main())


#!/usr/bin/env python3
from __future__ import annotations
import logging

import argparse
import json
import os
import subprocess
import sys
import tempfile
from pathlib import Path
from typing import Any, Dict, List

def run_cmd(argv: List[str], cwd: Path | None = None) -> Dict[str, Any]:
	try:
		p = subprocess.run(argv, cwd=str(cwd) if cwd else None, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, check=False)
		return {"argv": argv, "rc": int(p.returncode), "stdout": p.stdout, "stderr": p.stderr}
	except Exception as e:
		return {"argv": argv, "rc": -1, "stdout": "", "stderr": str(e)}

def load_json_safe(p: Path) -> Dict[str, Any]:
	try:
		return json.loads(p.read_text(encoding="utf-8"))
	except Exception:
		return {}

def main() -> int:
	ap = argparse.ArgumentParser(description="Ask a question about a sandboxed codebase and emit answer+citations")
	root = Path(__file__).resolve().parents[2]
	ap.add_argument("--question", required=True)
	ap.add_argument("--world", default=str(root / "data" / "sandbox" / "tmp" / "world.json"))
	ap.add_argument("--code-index", default=str(root / "data" / "sandbox" / "tmp" / "code_index.json"))
	ap.add_argument("--bm25-index", default=str(root / "data" / "sandbox" / "tmp" / "bm25_index.json"))
	ap.add_argument("--max-files", type=int, default=20)
	ap.add_argument("--lines", type=int, default=80, help="Lines per read window")
	ap.add_argument("--out-traces", default=str(root / "data" / "devtools" / "qa.traces.jsonl"))
	ap.add_argument("--validate", action="store_true")


import logging

import argparse
import json
import os
import subprocess
import sys
import tempfile
from pathlib import Path
from typing import Any, Dict, List

def run_cmd(argv: List[str], cwd: Path | None = None) -> Dict[str, Any]:
	try:
		p = subprocess.run(argv, cwd=str(cwd) if cwd else None, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, check=False)
		return {"argv": argv, "rc": int(p.returncode), "stdout": p.stdout, "stderr": p.stderr}
	except Exception as e:
		return {"argv": argv, "rc": -1, "stdout": "", "stderr": str(e)}

def load_json_safe(p: Path) -> Dict[str, Any]:
	try:
		return json.loads(p.read_text(encoding="utf-8"))
	except Exception:
		return {}

def main() -> int:
	ap = argparse.ArgumentParser(description="Ask a question about a sandboxed codebase and emit answer+citations")
	root = Path(__file__).resolve().parents[2]
	ap.add_argument("--question", required=True)
	ap.add_argument("--world", default=str(root / "data" / "sandbox" / "tmp" / "world.json"))
	ap.add_argument("--code-index", default=str(root / "data" / "sandbox" / "tmp" / "code_index.json"))
	ap.add_argument("--bm25-index", default=str(root / "data" / "sandbox" / "tmp" / "bm25_index.json"))
	ap.add_argument("--max-files", type=int, default=20)
	ap.add_argument("--lines", type=int, default=80, help="Lines per read window")
	ap.add_argument("--out-traces", default=str(root / "data" / "devtools" / "qa.traces.jsonl"))
	ap.add_argument("--validate", action="store_true")
	ap.add_argument("--debug", action="store_true")
	args = ap.parse_args()

	def log(msg: str) -> None:
		if bool(args.debug):
			print(f"[qa.ask] {msg}", file=sys.stderr, flush=True)


import tempfile
from pathlib import Path
from typing import Any, Dict, List

def run_cmd(argv: List[str], cwd: Path | None = None) -> Dict[str, Any]:
	try:
		p = subprocess.run(argv, cwd=str(cwd) if cwd else None, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, check=False)
		return {"argv": argv, "rc": int(p.returncode), "stdout": p.stdout, "stderr": p.stderr}
	except Exception as e:
		return {"argv": argv, "rc": -1, "stdout": "", "stderr": str(e)}

def load_json_safe(p: Path) -> Dict[str, Any]:
	try:
		return json.loads(p.read_text(encoding="utf-8"))
	except Exception:
		return {}

def main() -> int:
	ap = argparse.ArgumentParser(description="Ask a question about a sandboxed codebase and emit answer+citations")
	root = Path(__file__).resolve().parents[2]
	ap.add_argument("--question", required=True)
	ap.add_argument("--world", default=str(root / "data" / "sandbox" / "tmp" / "world.json"))
	ap.add_argument("--code-index", default=str(root / "data" / "sandbox" / "tmp" / "code_index.json"))
	ap.add_argument("--bm25-index", default=str(root / "data" / "sandbox" / "tmp" / "bm25_index.json"))
	ap.add_argument("--max-files", type=int, default=20)
	ap.add_argument("--lines", type=int, default=80, help="Lines per read window")
	ap.add_argument("--out-traces", default=str(root / "data" / "devtools" / "qa.traces.jsonl"))
	ap.add_argument("--validate", action="store_true")
	ap.add_argument("--debug", action="store_true")
	args = ap.parse_args()

	def log(msg: str) -> None:
		if bool(args.debug):
			print(f"[qa.ask] {msg}", file=sys.stderr, flush=True)

	world = load_json_safe(Path(args.world))
	repo_root = Path(world.get("repo_root", root.as_posix()))
	log(f"repo_root={repo_root}")

	# 1) Ensure BM25 index exists (prefer world path)
	bm25_path = Path(world.get("bm25_index_path", args.bm25_index))
	log(f"bm25_index={bm25_path} exists={bm25_path.exists()}")
	if not bm25_path.exists():
		res = run_cmd(["python3", "scripts/qa/bm25_index.py", "--root", str(repo_root), "--out", str(bm25_path)])
		log(f"bm25_build rc={res.get('rc')} stderr={res.get('stderr')[:200]}")
		if int(res.get("rc", 1)) != 0:
			print(json.dumps({"ok": False, "error": "bm25_build_failed", "detail": res}))
			return 1

	# 2) Search for candidate files
	# Prefer hybrid search if embed index exists
	embed_path = Path(world.get("embed_index_path", repo_root / "data" / "sandbox" / "tmp" / "embed_index.json"))
	argv = ["python3", "scripts/qa/search.py", "--index", str(bm25_path), "--question", str(args.question), "--k", str(int(args.max_files))]
	if embed_path.exists():
		argv += ["--embed-index", str(embed_path)]
	log(f"search_argv={' '.join(argv)} (embed_exists={embed_path.exists()})")
	res = run_cmd(argv, cwd=repo_root)
	log(f"search_rc={res.get('rc')} stdout_len={len(res.get('stdout',''))} stderr={res.get('stderr')[:200]}")
	try:
		results = json.loads(res.get("stdout", "{}"))
	except Exception:
		results = {"results": []}
	cands = results.get("results", [])
	log(f"candidates={len(cands)} top3={[str(x.get('path','')) for x in cands[:3]]}")
	# Fallback: if no candidates, try common root docs
	if not cands:
		fallbacks = ["README.md", "readme.md", "ROADMAP.md", "SUMMARY.md", "docs/README.md", "docs/index.md"]
		picked: list[str] = []
		for rel in fallbacks:
			pp = (repo_root / rel).resolve()
			if pp.exists() and pp.is_file():
				picked.append(pp.as_posix())
				if len(picked) >= int(args.max_files):
					break
		cands = [{"path": p, "score": 1.0} for p in picked]
		log(f"fallback_candidates={len(cands)} from root docs")

	# 3) Read top snippets (prefer README/docs when relevant; take first 2 windows)
	snippets: List[Dict[str, Any]] = []
	for c in cands:
		p = Path(c.get("path", ""))
		if not p.is_absolute():
			p = (repo_root / p).resolve()
		# window 1
		read1 = run_cmd(["python3", "scripts/qa/read_file_range.py", str(p), "--start", "1", "--end", str(int(args.lines))])
		# window 2
		read2 = run_cmd(["python3", "scripts/qa/read_file_range.py", str(p), "--start", str(int(args.lines) + 1), "--end", str(int(args.lines) * 2)])
		for read in (read1, read2):
			try:
				obj = json.loads(read.get("stdout", "{}"))
			except Exception:
				obj = {"ok": False}
			if obj.get("ok"):
				obj["path"] = str(Path(obj.get("path")).resolve().as_posix())
				snippets.append(obj)
			else:
				log(f"read_fail path={p} rc={read.get('rc')} err={read.get('stderr')[:120]}")
	log(f"snippets_count={len(snippets)}")

	# 4) Summarize
	with tempfile.NamedTemporaryFile("w", delete=False, suffix=".json") as f:
		f.write(json.dumps(snippets, ensure_ascii=False))
		tmp_path = f.name
	res_sum = run_cmd(["python3", "scripts/qa/answer_summarize.py", "--question", str(args.question), "--snippets", str(tmp_path), "--debug"])
	log(f"summarize_rc={res_sum.get('rc')} out_len={len(res_sum.get('stdout',''))}")
	try:
		ans = json.loads(res_sum.get("stdout", "{}"))
	except Exception:
		ans = {"answer": "", "citations": []}
	log(f"answer_len={len(ans.get('answer',''))} citations={len(ans.get('citations',[]))}")

	# 5) Optional validation
	val = {"ok": True}
	if bool(args.validate):
		with tempfile.NamedTemporaryFile("w", delete=False, suffix=".json") as cfile:
			cfile.write(json.dumps(ans.get("citations", []), ensure_ascii=False))
			cpath = cfile.name
		# Use lenient defaults to allow short READMEs to pass; CI can enforce stricter gate
		val_run = run_cmd(["python3", "scripts/qa/validate_citations.py", "--world", str(args.world), "--citations", str(cpath), "--min-files", "1", "--min-lines", "5"], cwd=repo_root)
		try:
			val = json.loads(val_run.get("stdout", "{}"))
		except Exception:
			val = {"ok": False}
		log(f"validator_ok={val.get('ok')} coverage={val.get('coverage')}")

	# 6) Trace write
	trace = {
		"question": str(args.question),
		"candidates": cands,
		"snippets_count": len(snippets),
		"citations": ans.get("citations", []),
		"answer": ans.get("answer", ""),
		"validator": val,
	}
	outp = Path(args.out_traces)
	outp.parent.mkdir(parents=True, exist_ok=True)
	with outp.open("a", encoding="utf-8") as f:
		f.write(json.dumps(trace, ensure_ascii=False) + "\n")

	ok = bool(val.get("ok", True)) and bool(ans.get("answer", "").strip())
	print(json.dumps({"ok": ok, "answer": ans.get("answer", ""), "citations": ans.get("citations", [])}))
	return 0

if __name__ == "__main__":
	raise SystemExit(main())


log

	try:
		return json.loads(p.read_text(encoding="utf-8"))
	except Exception:
		return {}

def main() -> int:
	ap = argparse.ArgumentParser(description="Ask a question about a sandboxed codebase and emit answer+citations")
	root = Path(__file__).resolve().parents[2]
	ap.add_argument("--question", required=True)
	ap.add_argument("--world", default=str(root / "data" / "sandbox" / "tmp" / "world.json"))
	ap.add_argument("--code-index", default=str(root / "data" / "sandbox" / "tmp" / "code_index.json"))
	ap.add_argument("--bm25-index", default=str(root / "data" / "sandbox" / "tmp" / "bm25_index.json"))
	ap.add_argument("--max-files", type=int, default=20)
	ap.add_argument("--lines", type=int, default=80, help="Lines per read window")
	ap.add_argument("--out-traces", default=str(root / "data" / "devtools" / "qa.traces.jsonl"))
	ap.add_argument("--validate", action="store_true")
	ap.add_argument("--debug", action="store_true")
	args = ap.parse_args()

	def log(msg: str) -> None:
		if bool(args.debug):
			print(f"[qa.ask] {msg}", file=sys.stderr, flush=True)

	world = load_json_safe(Path(args.world))
	repo_root = Path(world.get("repo_root", root.as_posix()))
	log(f"repo_root={repo_root}")

	# 1) Ensure BM25 index exists (prefer world path)
	bm25_path = Path(world.get("bm25_index_path", args.bm25_index))
	log(f"bm25_index={bm25_path} exists={bm25_path.exists()}")
	if not bm25_path.exists():
		res = run_cmd(["python3", "scripts/qa/bm25_index.py", "--root", str(repo_root), "--out", str(bm25_path)])
		log(f"bm25_build rc={res.get('rc')} stderr={res.get('stderr')[:200]}")
		if int(res.get("rc", 1)) != 0:
			print(json.dumps({"ok": False, "error": "bm25_build_failed", "detail": res}))
			return 1

	# 2) Search for candidate files
	# Prefer hybrid search if embed index exists
	embed_path = Path(world.get("embed_index_path", repo_root / "data" / "sandbox" / "tmp" / "embed_index.json"))
	argv = ["python3", "scripts/qa/search.py", "--index", str(bm25_path), "--question", str(args.question), "--k", str(int(args.max_files))]
	if embed_path.exists():


#!/usr/bin/env python3
from __future__ import annotations
import logging

import argparse
import json
import os
from pathlib import Path
from typing import Dict, Any, List, Tuple

def _try_imports():
	try:
		from sentence_transformers import SentenceTransformer  # type: ignore
	except Exception as e:
		raise RuntimeError("sentence-transformers is required. pip install sentence-transformers") from e
	return SentenceTransformer

def iter_repo_files(root: Path, include_exts: List[str]) -> List[Path]:
	files: List[Path] = []
	for p in root.rglob("*"):
		if not p.is_file():
			continue
		try:
			rel = p.relative_to(root)
		except Exception:
			rel = p
		parts = rel.parts
		if any(seg.startswith(".") for seg in parts):
			continue
		if any(seg in (".git", "models") for seg in parts):
			continue
		if include_exts and p.suffix not in include_exts:
			continue
		files.append(p)
	return files

def load_text_excerpt(p: Path, max_chars: int = 4000) -> str:
	try:
		text = p.read_text(encoding="utf-8", errors="ignore")
	except Exception:
		text = ""
	return text[:max_chars]

def build_embed_index(root: Path, out_path: Path, model_name: str, include_exts: List[str]) -> Dict[str, Any]:
	SentenceTransformer = _try_imports()
	root = root.resolve()
	out_path.parent.mkdir(parents=True, exist_ok=True)
	files = iter_repo_files(root, include_exts)
	# Prepare texts
	paths: List[str] = []
	texts: List[str] = []
	for fp in files:
		paths.append(fp.as_posix())
		texts.append(load_text_excerpt(fp))
	model = SentenceTransformer(model_name)
	embs = model.encode(texts or [""], normalize_embeddings=True)
	# Convert to plain list for JSON portability
	vectors: List[List[float]] = [list(map(float, v)) for v in embs]
	index = {
		"version": "0.1",
		"model": model_name,
		"root": root.as_posix(),
		"paths": paths,
		"vectors": vectors,
	}
	out_path.write_text(json.dumps(index, ensure_ascii=False), encoding="utf-8")
	print(json.dumps({"ok": True, "wrote": out_path.as_posix(), "docs": len(paths)}))
	return index

def main() -> int:
	ap = argparse.ArgumentParser(description="Build embedding index for code/docs using sentence-transformers")
	root = Path(__file__).resolve().parents[2]
	ap.add_argument("--root", default=str(root))
	ap.add_argument("--out", default=str(root / "data" / "sandbox" / "tmp" / "embed_index.json"))
	ap.add_argument("--model", default=os.environ.get("QA_EMBED_MODEL", "sentence-transformers/all-MiniLM-L6-v2"))
	ap.add_argument("--exts", default=".py,.md,.mk,.txt,.yaml,.yml,.toml,.ini")
	args = ap.parse_args()

	include_exts = [e.strip() for e in str(args.exts).split(",") if e.strip()]
	build_embed_index(Path(args.root), Path(args.out), str(args.model), include_exts)
	return 0

if __name__ == "__main__":
	raise SystemExit(main())


#!/usr/bin/env python3
from __future__ import annotations
import logging

import argparse
import json
import os
from pathlib import Path
from typing import Dict, Any, List, Tuple

def _try_imports():
	try:
		from sentence_transformers import SentenceTransformer  # type: ignore
	except Exception as e:
		raise RuntimeError("sentence-transformers is required. pip install sentence-transformers") from e
	return SentenceTransformer

def iter_repo_files(root: Path, include_exts: List[str]) -> List[Path]:
	files: List[Path] = []
	for p in root.rglob("*"):
		if not p.is_file():
			continue
		try:
			rel = p.relative_to(root)
		except Exception:
			rel = p
		parts = rel.parts
		if any(seg.startswith(".") for seg in parts):
			continue
		if any(seg in (".git", "models") for seg in parts):
			continue
		if include_exts and p.suffix not in include_exts:
			continue
		files.append(p)
	return files


#!/usr/bin/env python3
from __future__ import annotations
import logging

import argparse
import json
import os
from pathlib import Path
from typing import Dict, Any, List, Tuple

def _try_imports():
	try:
		from sentence_transformers import SentenceTransformer  # type: ignore
	except Exception as e:
		raise RuntimeError("sentence-transformers is required. pip install sentence-transformers") from e
	return SentenceTransformer

def iter_repo_files(root: Path, include_exts: List[str]) -> List[Path]:
	files: List[Path] = []
	for p in root.rglob("*"):
		if not p.is_file():
			continue
		try:
			rel = p.relative_to(root)
		except Exception:
			rel = p
		parts = rel.parts
		if any(seg.startswith(".") for seg in parts):
			continue
		if any(seg in (".git", "models") for seg in parts):
			continue
		if include_exts and p.suffix not in include_exts:
			continue
		files.append(p)
	return files

def load_text_excerpt(p: Path, max_chars: int = 4000) -> str:
	try:
		text = p.read_text(encoding="utf-8", errors="ignore")
	except Exception:
		text = ""
	return text[:max_chars]

def build_embed_index(root: Path, out_path: Path, model_name: str, include_exts: List[str]) -> Dict[str, Any]:
	SentenceTransformer = _try_imports()
	root = root.resolve()
	out_path.parent.mkdir(parents=True, exist_ok=True)
	files = iter_repo_files(root, include_exts)
	# Prepare texts
	paths: List[str] = []
	texts: List[str] = []
	for fp in files:
		paths.append(fp.as_posix())


def iter_repo_files(root: Path, include_exts: List[str]) -> List[Path]:
	files: List[Path] = []
	for p in root.rglob("*"):
		if not p.is_file():
			continue
		try:
			rel = p.relative_to(root)
		except Exception:
			rel = p
		parts = rel.parts
		if any(seg.startswith(".") for seg in parts):
			continue
		if any(seg in (".git", "models") for seg in parts):
			continue
		if include_exts and p.suffix not in include_exts:
			continue
		files.append(p)
	return files

def load_text_excerpt(p: Path, max_chars: int = 4000) -> str:
	try:
		text = p.read_text(encoding="utf-8", errors="ignore")
	except Exception:
		text = ""
	return text[:max_chars]

def build_embed_index(root: Path, out_path: Path, model_name: str, include_exts: List[str]) -> Dict[str, Any]:
	SentenceTransformer = _try_imports()
	root = root.resolve()
	out_path.parent.mkdir(parents=True, exist_ok=True)
	files = iter_repo_files(root, include_exts)
	# Prepare texts
	paths: List[str] = []
	texts: List[str] = []
	for fp in files:
		paths.append(fp.as_posix())
		texts.append(load_text_excerpt(fp))
	model = SentenceTransformer(model_name)
	embs = model.encode(texts or [""], normalize_embeddings=True)
	# Convert to plain list for JSON portability
	vectors: List[List[float]] = [list(map(float, v)) for v in embs]
	index = {
		"version": "0.1",
		"model": model_name,


			rel = p
		parts = rel.parts
		if any(seg.startswith(".") for seg in parts):
			continue
		if any(seg in (".git", "models") for seg in parts):
			continue
		if include_exts and p.suffix not in include_exts:
			continue
		files.append(p)
	return files

def load_text_excerpt(p: Path, max_chars: int = 4000) -> str:
	try:
		text = p.read_text(encoding="utf-8", errors="ignore")
	except Exception:
		text = ""
	return text[:max_chars]

def build_embed_index(root: Path, out_path: Path, model_name: str, include_exts: List[str]) -> Dict[str, Any]:
	SentenceTransformer = _try_imports()
	root = root.resolve()
	out_path.parent.mkdir(parents=True, exist_ok=True)
	files = iter_repo_files(root, include_exts)
	# Prepare texts
	paths: List[str] = []
	texts: List[str] = []
	for fp in files:
		paths.append(fp.as_posix())
		texts.append(load_text_excerpt(fp))
	model = SentenceTransformer(model_name)
	embs = model.encode(texts or [""], normalize_embeddings=True)
	# Convert to plain list for JSON portability
	vectors: List[List[float]] = [list(map(float, v)) for v in embs]
	index = {
		"version": "0.1",
		"model": model_name,
		"root": root.as_posix(),
		"paths": paths,
		"vectors": vectors,
	}
	out_path.write_text(json.dumps(index, ensure_ascii=False), encoding="utf-8")
	print(json.dumps({"ok": True, "wrote": out_path.as_posix(), "docs": len(paths)}))
	return index

def main() -> int:
	ap = argparse.ArgumentParser(description="Build embedding index for code/docs using sentence-transformers")
	root = Path(__file__).resolve().parents[2]
	ap.add_argument("--root", default=str(root))
	ap.add_argument("--out", default=str(root / "data" / "sandbox" / "tmp" / "embed_index.json"))
	ap.add_argument("--model", default=os.environ.get("QA_EMBED_MODEL", "sentence-transformers/all-MiniLM-L6-v2"))
	ap.add_argument("--exts", default=".py,.md,.mk,.txt,.yaml,.yml,.toml,.ini")
	args = ap.parse_args()

	include_exts = [e.strip() for e in str(args.exts).split(",") if e.strip()]
	build_embed_index(Path(args.root), Path(args.out), str(args.model), include_exts)
	return 0

if __name__ == "__main__":
	raise SystemExit(main())


main

	texts: List[str] = []
	for fp in files:
		paths.append(fp.as_posix())
		texts.append(load_text_excerpt(fp))
	model = SentenceTransformer(model_name)
	embs = model.encode(texts or [""], normalize_embeddings=True)
	# Convert to plain list for JSON portability
	vectors: List[List[float]] = [list(map(float, v)) for v in embs]
	index = {
		"version": "0.1",
		"model": model_name,
		"root": root.as_posix(),
		"paths": paths,
		"vectors": vectors,
	}
	out_path.write_text(json.dumps(index, ensure_ascii=False), encoding="utf-8")
	print(json.dumps({"ok": True, "wrote": out_path.as_posix(), "docs": len(paths)}))
	return index

def main() -> int:
	ap = argparse.ArgumentParser(description="Build embedding index for code/docs using sentence-transformers")
	root = Path(__file__).resolve().parents[2]
	ap.add_argument("--root", default=str(root))
	ap.add_argument("--out", default=str(root / "data" / "sandbox" / "tmp" / "embed_index.json"))
	ap.add_argument("--model", default=os.environ.get("QA_EMBED_MODEL", "sentence-transformers/all-MiniLM-L6-v2"))
	ap.add_argument("--exts", default=".py,.md,.mk,.txt,.yaml,.yml,.toml,.ini")
	args = ap.parse_args()

	include_exts = [e.strip() for e in str(args.exts).split(",") if e.strip()]
	build_embed_index(Path(args.root), Path(args.out), str(args.model), include_exts)
	return 0

if __name__ == "__main__":
	raise SystemExit(main())


#!/usr/bin/env python3
from __future__ import annotations
import logging

import argparse
import json
import math
import os
import re
from pathlib import Path
from typing import Dict, Any, List, Tuple

TOKEN_RE = re.compile(r"[A-Za-z0-9_]+")

def iter_repo_files(root: Path, include_exts: List[str]) -> List[Path]:
	files: List[Path] = []
	for p in root.rglob("*"):
		if not p.is_file():
			continue
		try:
			rel = p.relative_to(root)
		except Exception:
			rel = p
		parts = rel.parts
		if any(seg.startswith(".") for seg in parts):
			continue
		# Avoid indexing repo-internal artifacts directories
		if any(seg in ("data", "models", ".git") for seg in parts):
			continue
		if include_exts and p.suffix not in include_exts:
			continue
		files.append(p)
	return files

def tokenize(text: str) -> List[str]:
	return [t.lower() for t in TOKEN_RE.findall(text)]

def build_index(root: Path, out_path: Path, include_exts: List[str]) -> Dict[str, Any]:
	root = root.resolve()
	out_path.parent.mkdir(parents=True, exist_ok=True)
	files = iter_repo_files(root, include_exts)
	# Document statistics
	doc_len: Dict[str, int] = {}
	term_df: Dict[str, int] = {}
	postings: Dict[str, Dict[str, int]] = {}
	for fp in files:
		try:
			text = fp.read_text(encoding="utf-8", errors="ignore")
		except Exception:
			text = ""
		tokens = tokenize(text)
		doc_len[fp.as_posix()] = len(tokens)
		seen: Dict[str, int] = {}
		for tok in tokens:
			seen[tok] = seen.get(tok, 0) + 1
		for tok, tf in seen.items():
			postings.setdefault(tok, {})[fp.as_posix()] = tf
			term_df[tok] = term_df.get(tok, 0) + 1
	N = max(1, len(doc_len))
	avgdl = sum(doc_len.values()) / float(N)
	index = {
		"version": "0.1",
		"root": root.as_posix(),
		"docs": doc_len,
		"df": term_df,
		"postings": postings,
		"avgdl": avgdl,
		"N": N,
	}
	out_path.write_text(json.dumps(index, ensure_ascii=False, indent=2), encoding="utf-8")
	return index

def main() -> int:
	ap = argparse.ArgumentParser(description="Build simple BM25-like index for code/docs")
	root = Path(__file__).resolve().parents[2]
	ap.add_argument("--root", default=str(root))
	default_out = Path(__file__).resolve().parents[1] / "data" / "sandbox" / "tmp" / "bm25_index.json"
	ap.add_argument("--out", default=str(default_out))
	ap.add_argument("--exts", default=".py,.md,.mk,.txt,.yaml,.yml,.toml,.ini")
	args = ap.parse_args()

	include_exts = [e.strip() for e in str(args.exts).split(",") if e.strip()]
	idx = build_index(Path(args.root), Path(args.out), include_exts)
	print(json.dumps({"ok": True, "wrote": str(args.out), "docs": len(idx.get("docs", {}))}))
	return 0

if __name__ == "__main__":
	raise SystemExit(main())


#!/usr/bin/env python3
from __future__ import annotations
import logging

import argparse
import json
import math
import os
import re
from pathlib import Path
from typing import Dict, Any, List, Tuple

TOKEN_RE = re.compile(r"[A-Za-z0-9_]+")

def iter_repo_files(root: Path, include_exts: List[str]) -> List[Path]:
	files: List[Path] = []
	for p in root.rglob("*"):
		if not p.is_file():
			continue
		try:
			rel = p.relative_to(root)
		except Exception:
			rel = p
		parts = rel.parts
		if any(seg.startswith(".") for seg in parts):
			continue
		# Avoid indexing repo-internal artifacts directories
		if any(seg in ("data", "models", ".git") for seg in parts):
			continue
		if include_exts and p.suffix not in include_exts:
			continue
		files.append(p)
	return files

def tokenize(text: str) -> List[str]:
	return [t.lower() for t in TOKEN_RE.findall(text)]

def build_index(root: Path, out_path: Path, include_exts: List[str]) -> Dict[str, Any]:
	root = root.resolve()
	out_path.parent.mkdir(parents=True, exist_ok=True)
	files = iter_repo_files(root, include_exts)
	# Document statistics
	doc_len: Dict[str, int] = {}
	term_df: Dict[str, int] = {}
	postings: Dict[str, Dict[str, int]] = {}
	for fp in files:
		try:
			text = fp.read_text(encoding="utf-8", errors="ignore")
		except Exception:
			text = ""
		tokens = tokenize(text)


tokenize

	files: List[Path] = []
	for p in root.rglob("*"):
		if not p.is_file():
			continue
		try:
			rel = p.relative_to(root)
		except Exception:
			rel = p
		parts = rel.parts
		if any(seg.startswith(".") for seg in parts):
			continue
		# Avoid indexing repo-internal artifacts directories
		if any(seg in ("data", "models", ".git") for seg in parts):
			continue
		if include_exts and p.suffix not in include_exts:
			continue
		files.append(p)
	return files

def tokenize(text: str) -> List[str]:
	return [t.lower() for t in TOKEN_RE.findall(text)]

def build_index(root: Path, out_path: Path, include_exts: List[str]) -> Dict[str, Any]:
	root = root.resolve()
	out_path.parent.mkdir(parents=True, exist_ok=True)
	files = iter_repo_files(root, include_exts)
	# Document statistics
	doc_len: Dict[str, int] = {}
	term_df: Dict[str, int] = {}
	postings: Dict[str, Dict[str, int]] = {}
	for fp in files:
		try:
			text = fp.read_text(encoding="utf-8", errors="ignore")
		except Exception:
			text = ""
		tokens = tokenize(text)
		doc_len[fp.as_posix()] = len(tokens)
		seen: Dict[str, int] = {}
		for tok in tokens:
			seen[tok] = seen.get(tok, 0) + 1