File size: 6,152 Bytes
63f41a6 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 | #!/usr/bin/env python
"""Mutation check for the reviewer-R2 fixes: each mutant reverts ONE fix in a private copy of the tree and
tests/test_p2_r2_review.py must go red.
.venv/bin/python scripts/p2/mutation_check_r2.py
Same method as scripts/p2/mutation_check_integration.py (scratch tree under .tmp; the working tree is never
modified). Exit status 1 if any mutant survives or any mutation does not apply exactly once.
"""
from __future__ import annotations
import json
import os
import shutil
import subprocess
import sys
import tempfile
from pathlib import Path
REPO = Path(__file__).resolve().parents[2]
MUTANTS = [
# (id, file, old, new, test selector)
("D1 guard: borrowed lock not tied to the holder", "align/utils/guard.py",
" _die_with_parent(int(parent_pid))\n", "", "borrowed"),
("D2 guard: pdeathsig failure ignored", "align/utils/guard.py",
" if rc != 0:\n raise PreflightRefused(\n f\"inherit_heavy_job_lock: prctl", " if False:\n raise PreflightRefused(\n f\"inherit_heavy_job_lock: prctl", "pdeathsig_failure"),
("D3 guard: parent death before prctl ignored", "align/utils/guard.py",
" if os.getppid() != parent_pid:\n raise PreflightRefused(f\"inherit_heavy_job_lock: parent", " if False:\n raise PreflightRefused(f\"inherit_heavy_job_lock: parent", "parent_that_died"),
("S1 supervisor: stop signal before child recorded ignored", "scripts/p2/launch_baseline.py",
" if self.stop:\n # A stop signal that landed between", " if False:\n # A stop signal that landed between", "stop_signal"),
("S2 supervisor: zombie leader counted as alive", "scripts/p2/launch_baseline.py",
"if int(rest[2]) == pgid and rest[0] != \"Z\":", "if int(rest[2]) == pgid:", "zombie"),
("S3 supervisor: group liveness ignores non-leader members", "scripts/p2/launch_baseline.py",
"if int(rest[2]) == pgid and rest[0] != \"Z\":", "if int(entry) == pgid and rest[0] != \"Z\":", "live_member"),
("R1 lean_gen: model_rev not compared on resume", "extras/vllm_rollout/lean_gen.py",
"if r.get(\"model_rev\") != model_rev:", "if False:", "another_models_output"),
("R2 lean_gen: seed not compared on resume", "extras/vllm_rollout/lean_gen.py",
"elif r.get(\"sampling\", {}).get(\"seed\") != want:", "elif False:", "base_seed"),
("R3 lean_gen: sampling params not compared on resume", "extras/vllm_rollout/lean_gen.py",
"elif r.get(\"sampling\", {}).get(\"temperature\") != temperature or r.get(\"sampling\", {}).get(\"top_p\") != top_p:", "elif False:", "sampling_parameters"),
("R4 lean_gen: conflict count truncated", "extras/vllm_rollout/lean_gen.py",
" if n_bad > len(bad):", " if False:", "bounded"),
("R5 lean_gen: prompt module change ignored", "extras/vllm_rollout/lean_gen.py",
"if s.get(\"lean_prompts_sha256\") != prompts_sha256})", "if False})", "prompt_module_edited"),
("R6 lean_gen: resume check not wired into main", "extras/vllm_rollout/lean_gen.py",
" conflicts = resume_conflicts(a.out, rev, a.seed)\n", " conflicts = []\n", "main_refuses"),
("P1 sft.py: repo root not put on sys.path", "scripts/sft.py",
"sys.path.insert(0, str(Path(__file__).resolve().parent.parent))\n", "", "sft_script"),
("H1 shm sweep: segments a live process holds are removed", "scripts/p2/launch_baseline.py",
"or path in held:", "or False:", "sweep_removes"),
("H2 shm sweep: no time bound (other jobs' old segments removed)", "scripts/p2/launch_baseline.py",
"or st.st_mtime < since", "or False", "sweep_removes"),
("H3 shm sweep: any file name removed", "scripts/p2/launch_baseline.py",
"if not re.fullmatch(SHM_NAME_RE, name):", "if False:", "sweep_removes"),
("H4 shm sweep: not run after the child exits", "scripts/p2/launch_baseline.py",
" gone, nbytes = sweep_orphan_shm(started_at)\n", " gone, nbytes = [], 0\n", "run_child_sweeps"),
("T1 conftest: a held heavy-job lock is not seen by gpu tests", "tests/conftest.py",
" return None if f\"pid={os.getpid()}\" in label.split() else (label or \"unknown holder\")", " return None", "gpu_test_guard"),
("T2 conftest: own lock blocks own tests", "tests/conftest.py",
"None if f\"pid={os.getpid()}\" in label.split() else (label or \"unknown holder\")", "(label or \"unknown holder\")", "gpu_test_guard"),
]
def main() -> int:
(REPO / ".tmp").mkdir(exist_ok=True)
work = Path(tempfile.mkdtemp(dir=REPO / ".tmp", prefix="mutr2_"))
results = []
try:
for mid, rel, old, new, sel in MUTANTS:
tree = work / mid.split()[0]
shutil.copytree(REPO, tree, ignore=shutil.ignore_patterns(".tmp", ".venv", ".venv-rollout", ".cache", "__pycache__", ".git", "hf_cache"), dirs_exist_ok=True)
src = (tree / rel).read_text()
if src.count(old) != 1:
results.append({"mutant": mid, "status": "BAD_MUTANT", "count": src.count(old)})
print(results[-1], flush=True)
continue
(tree / rel).write_text(src.replace(old, new))
env = dict(os.environ, PYTHONPATH=str(tree))
r = subprocess.run([str(REPO / ".venv/bin/python"), "-m", "pytest", "tests/test_p2_r2_review.py", "-q", "-x", "-k", sel,
"-p", "no:cacheprovider"], cwd=tree, capture_output=True, text=True, env=env, timeout=600)
results.append({"mutant": mid, "status": "killed" if r.returncode == 1 else ("SURVIVED" if r.returncode == 0 else f"BAD_SELECTOR_rc{r.returncode}"), "tail": r.stdout.strip().splitlines()[-1:]})
print(results[-1], flush=True)
shutil.rmtree(tree, ignore_errors=True)
finally:
shutil.rmtree(work, ignore_errors=True)
bad = [r for r in results if r["status"] != "killed"]
print(json.dumps({"mutants": len(results), "killed": len(results) - len(bad), "not_killed": bad}, indent=1))
return 1 if bad else 0
if __name__ == "__main__":
raise SystemExit(main())
|