| |
| """Mutation check for the P2 integration seams: each mutant removes ONE guard from a private copy of the |
| file under test and tests/test_p2_integration.py must go red. |
| |
| .venv/bin/python scripts/p2/mutation_check_integration.py |
| |
| The mutated file is a copy placed in a scratch tree that mirrors the repo layout (scripts/p2/, align/utils/); |
| nothing in the working tree is modified, so a concurrent job importing the real files is never exposed to |
| a mutant. Exit status 1 if any mutant survives. |
| """ |
|
|
| from __future__ import annotations |
|
|
| import json |
| import os |
| import shutil |
| import subprocess |
| import sys |
| import tempfile |
| from pathlib import Path |
|
|
| REPO = Path(__file__).resolve().parents[2] |
|
|
| MUTANTS = [ |
| |
| ("S1 keys: unverified samples allowed", "scripts/p2/check_seams.py", "if missing or extra:", "if False:", "keys_seam"), |
| ("S2 keys: stale verdict allowed", "scripts/p2/check_seams.py", "if stale:", "if False:", "stale_verdict"), |
| ("S3 keys: accepted truncated allowed", "scripts/p2/check_seams.py", "if bad:\n raise SeamError(f\"{len(bad)} length", "if False:\n raise SeamError(f\"{len(bad)} length", "truncated"), |
| ("S4 sft: rejected sample may be trained on", "scripts/p2/check_seams.py", "if not (key in vers and vers[key][\"accepted\"]):", "if False:", "rejected_sample"), |
| ("S5 sft: completion not compared", "scripts/p2/check_seams.py", "if not np.array_equal(trained, np.asarray(g[\"completion_ids\"], dtype=trained.dtype)):", "if False:", "differs_by_one_token"), |
| ("S6 sft: prompt not compared", "scripts/p2/check_seams.py", "if p != len(g[\"prompt_ids\"]) or not np.array_equal(ids[:p], np.asarray(g[\"prompt_ids\"])):", "if False:", "different_prompt"), |
| ("S7 sft: empty shard set passes", "scripts/p2/check_seams.py", "if rows == 0:", "if False:", "vacuously"), |
| ("L1 launch: exit code trusted over completeness", "scripts/p2/launch_baseline.py", "if rc == 0 and ok:", "if rc == 0:", "exits_zero"), |
| ("L2 launch: retry without progress", "scripts/p2/launch_baseline.py", "if size <= last:", "if False:", "crash_loop"), |
| ("L3 launch: finished phases rerun", "scripts/p2/launch_baseline.py", "if marker.is_file():", "if False:", "skipped_on_rerun"), |
| ("L4 launch: orphans not reaped", "scripts/p2/launch_baseline.py", " self.kill_group(self.child, grace=15.0)\n self.child = None", " self.child = None", "reaped"), |
| ("L5 launch: torn tail counted", "scripts/p2/launch_baseline.py", "if not line.endswith(b\"\\n\"):\n break", "if False:\n break", "torn_last_line"), |
| ("L6 launch: children do not inherit the lock", "scripts/p2/launch_baseline.py", "\"--inherit-lock\", str(parent_pid),\n \"--label\"", "\"--label\"", "inherits_the_lock"), |
| ("L7 launch: ver complete on any non-empty file", "scripts/p2/launch_baseline.py", "return bool(gkeys) and keys == gkeys,", "return bool(gkeys) and bool(keys),", "key_set"), |
| |
| |
| ("G1 guard: parent pid unchecked", "align/utils/guard.py", |
| ["if os.getppid() != int(parent_pid):", " if os.getppid() != parent_pid:\n raise PreflightRefused(f\"inherit_heavy_job_lock: parent"], |
| ["if False:", " if False:\n raise PreflightRefused(f\"inherit_heavy_job_lock: parent"], "inherit"), |
| ("G2 guard: holder label unchecked", "align/utils/guard.py", "if f\"pid={int(parent_pid)}\" not in holder.split():", "if False:", "inherit"), |
| ("G3 guard: flock held not verified", "align/utils/guard.py", " raise PreflightRefused(\n f\"lock file names pid={parent_pid} but nobody", " return True\n raise PreflightRefused(\n f\"lock file names pid={parent_pid} but nobody", "inherit"), |
| ("E1 export: group not signalled after result", "scripts/qwen3_export_check.py", "os.killpg(proc.pid, sig)", "pass", "vllm_child"), |
| ] |
|
|
|
|
| def main() -> int: |
| work = Path(tempfile.mkdtemp(dir=REPO / ".tmp", prefix="mutint_")) |
| results = [] |
| try: |
| for mid, rel, old, new, sel in MUTANTS: |
| tree = work / mid.split()[0] |
| shutil.copytree(REPO, tree, ignore=shutil.ignore_patterns(".tmp", ".venv", ".venv-rollout", ".cache", "__pycache__", ".git", "hf_cache"), dirs_exist_ok=True) |
| src = (tree / rel).read_text() |
| pairs = list(zip(old, new)) if isinstance(old, (list, tuple)) else [(old, new)] |
| if any(src.count(o) != 1 for o, _ in pairs): |
| results.append({"mutant": mid, "status": "BAD_MUTANT", "count": [src.count(o) for o, _ in pairs]}) |
| continue |
| for o, n in pairs: |
| src = src.replace(o, n) |
| (tree / rel).write_text(src) |
| env = dict(os.environ, PYTHONPATH=str(tree)) |
| r = subprocess.run([str(REPO / ".venv/bin/python"), "-m", "pytest", "tests/test_p2_integration.py", "-q", "-x", "-k", sel, |
| "-p", "no:cacheprovider"], cwd=tree, capture_output=True, text=True, env=env, timeout=600) |
| results.append({"mutant": mid, "status": "killed" if r.returncode != 0 else "SURVIVED", "tail": r.stdout.strip().splitlines()[-1:]}) |
| print(results[-1], flush=True) |
| shutil.rmtree(tree, ignore_errors=True) |
| finally: |
| shutil.rmtree(work, ignore_errors=True) |
| bad = [r for r in results if r["status"] != "killed"] |
| print(json.dumps({"mutants": len(results), "killed": len(results) - len(bad), "not_killed": bad}, indent=1)) |
| return 1 if bad else 0 |
|
|
|
|
| if __name__ == "__main__": |
| raise SystemExit(main()) |
|
|
