| |
| """Mutation check for the reviewer-R2 fixes: each mutant reverts ONE fix in a private copy of the tree and |
| tests/test_p2_r2_review.py must go red. |
| |
| .venv/bin/python scripts/p2/mutation_check_r2.py |
| |
| Same method as scripts/p2/mutation_check_integration.py (scratch tree under .tmp; the working tree is never |
| modified). Exit status 1 if any mutant survives or any mutation does not apply exactly once. |
| """ |
|
|
| from __future__ import annotations |
|
|
| import json |
| import os |
| import shutil |
| import subprocess |
| import sys |
| import tempfile |
| from pathlib import Path |
|
|
| REPO = Path(__file__).resolve().parents[2] |
|
|
| MUTANTS = [ |
| |
| ("D1 guard: borrowed lock not tied to the holder", "align/utils/guard.py", |
| " _die_with_parent(int(parent_pid))\n", "", "borrowed"), |
| ("D2 guard: pdeathsig failure ignored", "align/utils/guard.py", |
| " if rc != 0:\n raise PreflightRefused(\n f\"inherit_heavy_job_lock: prctl", " if False:\n raise PreflightRefused(\n f\"inherit_heavy_job_lock: prctl", "pdeathsig_failure"), |
| ("D3 guard: parent death before prctl ignored", "align/utils/guard.py", |
| " if os.getppid() != parent_pid:\n raise PreflightRefused(f\"inherit_heavy_job_lock: parent", " if False:\n raise PreflightRefused(f\"inherit_heavy_job_lock: parent", "parent_that_died"), |
| ("S1 supervisor: stop signal before child recorded ignored", "scripts/p2/launch_baseline.py", |
| " if self.stop:\n # A stop signal that landed between", " if False:\n # A stop signal that landed between", "stop_signal"), |
| ("S2 supervisor: zombie leader counted as alive", "scripts/p2/launch_baseline.py", |
| "if int(rest[2]) == pgid and rest[0] != \"Z\":", "if int(rest[2]) == pgid:", "zombie"), |
| ("S3 supervisor: group liveness ignores non-leader members", "scripts/p2/launch_baseline.py", |
| "if int(rest[2]) == pgid and rest[0] != \"Z\":", "if int(entry) == pgid and rest[0] != \"Z\":", "live_member"), |
| ("R1 lean_gen: model_rev not compared on resume", "extras/vllm_rollout/lean_gen.py", |
| "if r.get(\"model_rev\") != model_rev:", "if False:", "another_models_output"), |
| ("R2 lean_gen: seed not compared on resume", "extras/vllm_rollout/lean_gen.py", |
| "elif r.get(\"sampling\", {}).get(\"seed\") != want:", "elif False:", "base_seed"), |
| ("R3 lean_gen: sampling params not compared on resume", "extras/vllm_rollout/lean_gen.py", |
| "elif r.get(\"sampling\", {}).get(\"temperature\") != temperature or r.get(\"sampling\", {}).get(\"top_p\") != top_p:", "elif False:", "sampling_parameters"), |
| ("R4 lean_gen: conflict count truncated", "extras/vllm_rollout/lean_gen.py", |
| " if n_bad > len(bad):", " if False:", "bounded"), |
| ("R5 lean_gen: prompt module change ignored", "extras/vllm_rollout/lean_gen.py", |
| "if s.get(\"lean_prompts_sha256\") != prompts_sha256})", "if False})", "prompt_module_edited"), |
| ("R6 lean_gen: resume check not wired into main", "extras/vllm_rollout/lean_gen.py", |
| " conflicts = resume_conflicts(a.out, rev, a.seed)\n", " conflicts = []\n", "main_refuses"), |
| ("P1 sft.py: repo root not put on sys.path", "scripts/sft.py", |
| "sys.path.insert(0, str(Path(__file__).resolve().parent.parent))\n", "", "sft_script"), |
| ("H1 shm sweep: segments a live process holds are removed", "scripts/p2/launch_baseline.py", |
| "or path in held:", "or False:", "sweep_removes"), |
| ("H2 shm sweep: no time bound (other jobs' old segments removed)", "scripts/p2/launch_baseline.py", |
| "or st.st_mtime < since", "or False", "sweep_removes"), |
| ("H3 shm sweep: any file name removed", "scripts/p2/launch_baseline.py", |
| "if not re.fullmatch(SHM_NAME_RE, name):", "if False:", "sweep_removes"), |
| ("H4 shm sweep: not run after the child exits", "scripts/p2/launch_baseline.py", |
| " gone, nbytes = sweep_orphan_shm(started_at)\n", " gone, nbytes = [], 0\n", "run_child_sweeps"), |
| ("T1 conftest: a held heavy-job lock is not seen by gpu tests", "tests/conftest.py", |
| " return None if f\"pid={os.getpid()}\" in label.split() else (label or \"unknown holder\")", " return None", "gpu_test_guard"), |
| ("T2 conftest: own lock blocks own tests", "tests/conftest.py", |
| "None if f\"pid={os.getpid()}\" in label.split() else (label or \"unknown holder\")", "(label or \"unknown holder\")", "gpu_test_guard"), |
| ] |
|
|
|
|
| def main() -> int: |
| (REPO / ".tmp").mkdir(exist_ok=True) |
| work = Path(tempfile.mkdtemp(dir=REPO / ".tmp", prefix="mutr2_")) |
| results = [] |
| try: |
| for mid, rel, old, new, sel in MUTANTS: |
| tree = work / mid.split()[0] |
| shutil.copytree(REPO, tree, ignore=shutil.ignore_patterns(".tmp", ".venv", ".venv-rollout", ".cache", "__pycache__", ".git", "hf_cache"), dirs_exist_ok=True) |
| src = (tree / rel).read_text() |
| if src.count(old) != 1: |
| results.append({"mutant": mid, "status": "BAD_MUTANT", "count": src.count(old)}) |
| print(results[-1], flush=True) |
| continue |
| (tree / rel).write_text(src.replace(old, new)) |
| env = dict(os.environ, PYTHONPATH=str(tree)) |
| r = subprocess.run([str(REPO / ".venv/bin/python"), "-m", "pytest", "tests/test_p2_r2_review.py", "-q", "-x", "-k", sel, |
| "-p", "no:cacheprovider"], cwd=tree, capture_output=True, text=True, env=env, timeout=600) |
| results.append({"mutant": mid, "status": "killed" if r.returncode == 1 else ("SURVIVED" if r.returncode == 0 else f"BAD_SELECTOR_rc{r.returncode}"), "tail": r.stdout.strip().splitlines()[-1:]}) |
| print(results[-1], flush=True) |
| shutil.rmtree(tree, ignore_errors=True) |
| finally: |
| shutil.rmtree(work, ignore_errors=True) |
| bad = [r for r in results if r["status"] != "killed"] |
| print(json.dumps({"mutants": len(results), "killed": len(results) - len(bad), "not_killed": bad}, indent=1)) |
| return 1 if bad else 0 |
|
|
|
|
| if __name__ == "__main__": |
| raise SystemExit(main()) |
|
|
